diff --git a/.travis.yml b/.travis.yml index fbe1e0c9b..0e4f44d1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - GOARCH=amd64 - GOARCH=amd64 ENTERPRISE=1 install: - - make install-dep install-statik vendor generate-statik + - make install-dep vendor script: - make test before_deploy: @@ -27,9 +27,16 @@ deploy: go: "1.10" condition: $GOARCH = amd64 matrix: + # Excluding or allowing failures on non-primary matrix configurations due to long running times. + fast_finish: true allow_failures: - go: master - fast_finish: true + - go: 1.9 + exclude: + - go: 1.9 + env: GOARCH=386 + - go: 1.9 + env: GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/CHANGELOG.md b/CHANGELOG.md index 33969d4eb..4199a1763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,50 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v1.0.0] - 2018-07-05 + +### Added + +- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) +- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) + +### Changed + +- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304)) +- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) +- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) +- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) +- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) +- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) +- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296)) +- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) +- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) +- Modify PQL ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354)) +- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) +- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) +- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) +- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) +- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) + +### Removed + +- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) +- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393)) +- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) +- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) +- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) +- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) +- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) +- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) +- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) + +### Fixed + +- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) +- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) +- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) +- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) + ## [v0.10.0] - 2018-05-15 This version contains 93 contribution from 8 contributors. There are 93 files changed, 4,495 insertions, and 5,392 deletions. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c991caeab..2ad9ab13d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks: make install ``` -- Install build dependencies (dep, statik, and protoc): +- Install build dependencies (dep and protoc): ```sh make install-build-deps @@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks: make release ``` -- Generate static assets for the WebUI: - - ```sh - make generate-statik - ``` - - Regenerate protocol buffer files in `internal/`: ```sh diff --git a/Dockerfile b/Dockerfile index 7cdb93cc0..2b83c71c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + && CGO_ENABLED=0 make install-dep install FLAGS="-a" FROM scratch diff --git a/Gopkg.lock b/Gopkg.lock index 8bb1744bd..33187bfe2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - name = "github.com/BurntSushi/toml" - packages = ["."] - revision = "b26d9c308763d68093482582cea63d69be07a0f0" - version = "v0.3.0" - [[projects]] branch = "master" name = "github.com/CAFxX/gcnotifier" @@ -76,6 +70,18 @@ packages = ["proto"] revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/cmpopts", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] name = "github.com/gorilla/context" packages = ["."] @@ -191,12 +197,6 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" -[[projects]] - name = "github.com/rakyll/statik" - packages = ["fs"] - revision = "fd36b3595eb2ec8da4b8153b107f7ea08504899d" - version = "v0.1.1" - [[projects]] name = "github.com/satori/go.uuid" packages = ["."] @@ -212,14 +212,22 @@ [[projects]] name = "github.com/shirou/gopsutil" packages = [ + "cpu", "host", "internal/common", "mem", + "net", "process" ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] branch = "master" name = "github.com/spf13/afero" @@ -302,6 +310,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba" + inputs-digest = "da6d02118ca77527c4ff00e9522880032fc052fb39bc8efe6c76602857c8c84e" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 3f17f0b2e..90e4700df 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -48,8 +48,8 @@ build: vendor # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) - $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID)) + cp NOTICE README.md build/pilosa-$(VERSION_ID) + $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID)) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz @@ -88,16 +88,15 @@ install: vendor generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -# `go generate` statik assets (WebUI) -generate-statik: require-statik - go generate github.com/pilosa/pilosa/statik - # `go generate` stringers generate-stringer: go generate github.com/pilosa/pilosa +generate-pql: require-peg + cd pql && peg -inline pql.peg && cd .. + # `go generate` all needed packages -generate: generate-protoc generate-statik generate-stringer +generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile docker: @@ -126,23 +125,20 @@ endef require-dep: $(call require,dep) -require-statik: - $(call require,statik) - require-protoc-gen-gofast: $(call require,protoc-gen-gofast) require-protoc: $(call require,protoc) -install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer +require-peg: + $(call require,peg) + +install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg install-dep: go get -u github.com/golang/dep/cmd/dep -install-statik: - go get -u github.com/rakyll/statik - install-stringer: go get -u golang.org/x/tools/cmd/stringer @@ -151,3 +147,6 @@ install-protoc-gen-gofast: install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html + +install-peg: + go get github.com/pointlander/peg diff --git a/NOTES b/NOTES index 6b8e088ea..8da55fe11 100644 --- a/NOTES +++ b/NOTES @@ -14,13 +14,13 @@ │0000000000000000000000000000000000000000│ │────────────────────────────────────────┤ F ▶│0000000000000000000000000000000000000000│ - r ││0000000000000000000000000000000000000000│ - a ││0000000000000000000000000000000000000000│ - m ││0000000000000000000000000000000000000000│ - e ▶│0000000000000000000000000000000000000000│ + i ││0000000000000000000000000000000000000000│ + e ││0000000000000000000000000000000000000000│ + l ││0000000000000000000000000000000000000000│ + d ▶│0000000000000000000000000000000000000000│ └────────────────────────────────────────┘ ▲───────────▲ - Slice + Shard -Fragment=intersection of frame & slice +Fragment=intersection of field & shard diff --git a/api.go b/api.go index 252480d52..f646112b9 100644 --- a/api.go +++ b/api.go @@ -22,13 +22,10 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "strconv" "strings" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -36,27 +33,37 @@ import ( // API provides the top level programmatic interface to Pilosa. It is usually // wrapped by a handler which provides an external interface (e.g. HTTP). type API struct { - Holder *Holder - // The execution engine for running queries. - Executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + holder *Holder + cluster *cluster + server *Server + + Serializer Serializer +} + +// apiOption is a functional option type for pilosa.API +type apiOption func(*API) error + +func OptAPIServer(s *Server) apiOption { + return func(a *API) error { + a.server = s + a.holder = s.holder + a.cluster = s.cluster + a.Serializer = s.serializer + return nil } - Broadcaster Broadcaster - BroadcastHandler BroadcastHandler - StatusHandler StatusHandler - Cluster *Cluster - RemoteClient *http.Client - Logger Logger } // NewAPI returns a new API instance. -func NewAPI() *API { - return &API{ - Broadcaster: NopBroadcaster, - //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop - //StatusHandler: NopStatusHandler, // TODO: implement the nop - Logger: NopLogger, +func NewAPI(opts ...apiOption) (*API, error) { + api := &API{} + + for _, opt := range opts { + err := opt(api) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } } + return api, nil } // validAPIMethods specifies the api methods that are valid for each @@ -79,11 +86,11 @@ func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { } func (api *API) validate(f apiMethod) error { - state := api.Cluster.State() + state := api.cluster.State() if _, ok := validAPIMethods[state][f]; ok { return nil } - return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)} + return newApiMethodNotAllowedError(errors.Errorf("api method %s not allowed in state %s", f, state)) } // Query parses a PQL query out of the request and executes it. @@ -98,12 +105,12 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return resp, errors.Wrap(err, "parsing") } - execOpts := &ExecOptions{ + execOpts := &execOptions{ Remote: req.Remote, ExcludeRowAttrs: req.ExcludeRowAttrs, ExcludeColumns: req.ExcludeColumns, } - results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { return resp, errors.Wrap(err, "executing") } @@ -122,10 +129,22 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } // Retrieve column attributes across all calls. - columnAttrSets, err := api.readColumnAttrSets(api.Holder.Index(req.Index), columnIDs) + columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs) if err != nil { return resp, errors.Wrap(err, "reading column attrs") } + + // Translate column attributes, if necessary. + if api.server.primaryTranslateStore != nil { + for _, col := range resp.ColumnAttrSets { + v, err := api.server.primaryTranslateStore.TranslateColumnToString(req.Index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + resp.ColumnAttrSets = columnAttrSets } return resp, nil @@ -161,21 +180,20 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index } // Create index. - index, err := api.Holder.CreateIndex(indexName, options) + index, err := api.holder.CreateIndex(indexName, options) if err != nil { return nil, errors.Wrap(err, "creating index") } // Send the create index message to all nodes. - err = api.Broadcaster.SendSync( - &internal.CreateIndexMessage{ + err = api.server.SendSync( + &CreateIndexMessage{ Index: indexName, - Meta: options.Encode(), + Meta: &options, }) if err != nil { - api.Logger.Printf("problem sending CreateIndex message: %s", err) return nil, errors.Wrap(err, "sending CreateIndex message") } - api.Holder.Stats.Count("createIndex", 1, 1.0) + api.holder.Stats.Count("createIndex", 1, 1.0) return index, nil } @@ -185,9 +203,9 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { return nil, errors.Wrap(err, "validating api method") } - index := api.Holder.Index(indexName) + index := api.holder.Index(indexName) if index == nil { - return nil, ErrIndexNotFound + return nil, newNotFoundError(ErrIndexNotFound) } return index, nil } @@ -200,104 +218,128 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { } // Delete index from the holder. - err := api.Holder.DeleteIndex(indexName) + err := api.holder.DeleteIndex(indexName) if err != nil { return errors.Wrap(err, "deleting index") } // Send the delete index message to all nodes. - err = api.Broadcaster.SendSync( - &internal.DeleteIndexMessage{ + err = api.server.SendSync( + &DeleteIndexMessage{ Index: indexName, }) if err != nil { - api.Logger.Printf("problem sending DeleteIndex message: %s", err) + api.server.logger.Printf("problem sending DeleteIndex message: %s", err) return errors.Wrap(err, "sending DeleteIndex message") } - api.Holder.Stats.Count("deleteIndex", 1, 1.0) + api.holder.Stats.Count("deleteIndex", 1, 1.0) return nil } -// CreateFrame makes the named frame in the named index with the given options. -func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { - if err := api.validate(apiCreateFrame); err != nil { +// CreateField makes the named field in the named index with the given options. +// This method currently only takes a single functional option, but that may be +// changed in the future to support multiple options. +func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) { + if err := api.validate(apiCreateField); err != nil { return nil, errors.Wrap(err, "validating api method") } + // Apply functional options. + fo := FieldOptions{} + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + // Find index. - index := api.Holder.Index(indexName) + index := api.holder.Index(indexName) if index == nil { - return nil, ErrIndexNotFound + return nil, newNotFoundError(ErrIndexNotFound) } - // Create frame. - frame, err := index.CreateFrame(frameName, options) + // Create field. + field, err := index.CreateField(fieldName, opts...) if err != nil { - return nil, errors.Wrap(err, "creating frame") + return nil, errors.Wrap(err, "creating field") } - // Send the create frame message to all nodes. - err = api.Broadcaster.SendSync( - &internal.CreateFrameMessage{ + // Send the create field message to all nodes. + err = api.server.SendSync( + &CreateFieldMessage{ Index: indexName, - Frame: frameName, - Meta: options.Encode(), + Field: fieldName, + Meta: &fo, }) if err != nil { - api.Logger.Printf("problem sending CreateFrame message: %s", err) - return nil, errors.Wrap(err, "sending CreateFrame message") + api.server.logger.Printf("problem sending CreateField message: %s", err) + return nil, errors.Wrap(err, "sending CreateField message") } - api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) - return frame, nil + api.holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return field, nil } -// DeleteFrame removes the named frame from the named index. If the index is not -// found, an error is returned. If the frame is not found, it is ignored and no +// Field retrieves the named field. +func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, error) { + if err := api.validate(apiField); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + field := api.holder.Field(indexName, fieldName) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound) + } + return field, nil +} + +// DeleteField removes the named field from the named index. If the index is not +// found, an error is returned. If the field is not found, it is ignored and no // action is taken. -func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { - if err := api.validate(apiDeleteFrame); err != nil { +func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { return errors.Wrap(err, "validating api method") } // Find index. - index := api.Holder.Index(indexName) + index := api.holder.Index(indexName) if index == nil { - return ErrIndexNotFound + return newNotFoundError(ErrIndexNotFound) } - // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { - return errors.Wrap(err, "deleting frame") + // Delete field from the index. + if err := index.DeleteField(fieldName); err != nil { + return errors.Wrap(err, "deleting field") } - // Send the delete frame message to all nodes. - err := api.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ + // Send the delete field message to all nodes. + err := api.server.SendSync( + &DeleteFieldMessage{ Index: indexName, - Frame: frameName, + Field: fieldName, }) if err != nil { - api.Logger.Printf("problem sending DeleteFrame message: %s", err) - return errors.Wrap(err, "sending DeleteFrame message") + api.server.logger.Printf("problem sending DeleteField message: %s", err) + return errors.Wrap(err, "sending DeleteField message") } - api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } -// ExportCSV encodes the fragment designated by the index,frame,slice as +// ExportCSV encodes the fragment designated by the index,field,shard as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } - // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return ErrClusterDoesNotOwnSlice + // Validate that this handler owns the shard. + if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) + return ErrClusterDoesNotOwnShard } // Find the fragment. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.holder.fragment(indexName, fieldName, viewStandard, shard) if f == nil { return ErrFragmentNotFound } @@ -306,7 +348,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin cw := csv.NewWriter(w) // Iterate over each column. - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), strconv.FormatUint(columnID, 10), @@ -321,62 +363,13 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin return nil } -// SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { - if err := api.validate(apiSliceNodes); err != nil { +// ShardNodes returns the node and all replicas which should contain a shard's data. +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { + if err := api.validate(apiShardNodes); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.SliceNodes(indexName, slice), nil -} - -// MarshalFragment returns an object which can write the specified fragment's data -// to an io.Writer. The serialized data can be read back into a fragment with -// the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) { - if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) - if f == nil { - return nil, ErrFragmentNotFound - } - return f, nil -} - -// UnmarshalFragment creates a new fragment (if necessary) and reads data from a -// Reader which was previously written by MarshalFragment to populate the -// fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error { - if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Retrieve view. - view, err := f.CreateViewIfNotExists(ViewStandard) - if err != nil { - return errors.Wrap(err, "creating view") - } - - // Retrieve fragment from frame. - frag, err := view.CreateFragmentIfNotExists(slice) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - // Read fragment in from request body. - if _, err := frag.ReadFrom(reader); err != nil { - return errors.Wrap(err, "reading fragment") - } - return nil + return api.cluster.shardNodes(indexName, shard), nil } // FragmentBlockData is an endpoint for internal usage. It is not guaranteed to @@ -389,24 +382,24 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, reqBytes, err := ioutil.ReadAll(body) if err != nil { - return nil, BadRequestError{errors.Wrap(err, "read body error")} + return nil, NewBadRequestError(errors.Wrap(err, "read body error")) } - var req internal.BlockDataRequest - if err := proto.Unmarshal(reqBytes, &req); err != nil { - return nil, BadRequestError{errors.Wrap(err, "unmarshal body error")} + var req BlockDataRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice) + f := api.holder.fragment(req.Index, req.Field, viewStandard, req.Shard) if f == nil { return nil, ErrFragmentNotFound } - var resp = internal.BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + var resp = BlockDataResponse{} + resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. - buf, err := proto.Marshal(&resp) + buf, err := api.Serializer.Marshal(&resp) if err != nil { return nil, errors.Wrap(err, "merge block response encoding error") } @@ -415,13 +408,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, shard uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.holder.fragment(indexName, fieldName, viewStandard, shard) if f == nil { return nil, ErrFragmentNotFound } @@ -434,7 +427,13 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. func (api *API) Hosts(ctx context.Context) []*Node { - return api.Cluster.Nodes + return api.cluster.Nodes +} + +// Node gets the ID, URI and coordinator status for this particular node. +func (api *API) Node() *Node { + node := api.server.node() + return &node } // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. @@ -443,11 +442,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return errors.Wrap(err, "validating api method") } - err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) + err := api.server.SendSync(&RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") } - api.Holder.RecalculateCaches() + api.holder.recalculateCaches() return nil } @@ -464,155 +463,72 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { return errors.Wrap(err, "reading body") } - // Marshal into request object. - pb, err := UnmarshalMessage(body) + typ := body[0] + msg := getMessage(typ) + err = api.server.serializer.Unmarshal(body[1:], msg) if err != nil { - return errors.Wrap(err, "unmarshaling message") + return errors.Wrap(err, "deserializing cluster message") } // Forward the error message. - if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { + if err := api.server.receiveMessage(msg); err != nil { return errors.Wrap(err, "receiving message") } return nil } -// LocalID returns the current node's ID. -func (api *API) LocalID() string { - return api.Cluster.Node.ID -} - -// Schema returns information about each index in Pilosa including which frames -// and views they contain. +// Schema returns information about each index in Pilosa including which fields +// they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { - return api.Holder.Schema() + return api.holder.limitedSchema() } -// CreateField creates a new BSI field in the given index and frame. -func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { - if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Create new field. - if err := f.CreateField(field); err != nil { - return errors.Wrap(err, "creating field") - } - - // Send the create field message to all nodes. - err := api.Broadcaster.SendSync( - &internal.CreateFieldMessage{ - Index: indexName, - Frame: frameName, - Field: encodeField(field), - }) - if err != nil { - api.Logger.Printf("problem sending CreateField message: %s", err) - } - return errors.Wrap(err, "sending CreateField message") -} - -// DeleteField deletes the given field. -func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { - if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Delete field. - if err := f.DeleteField(fieldName); err != nil { - return errors.Wrap(err, "deleting field") - } - - // Send the delete field message to all nodes. - err := api.Broadcaster.SendSync( - &internal.DeleteFieldMessage{ - Index: indexName, - Frame: frameName, - Field: fieldName, - }) - if err != nil { - api.Logger.Printf("problem sending DeleteField message: %s", err) - } - return errors.Wrap(err, "sending DeleteField message") -} - -// Fields returns the fields in the given frame. -func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { - if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - index := api.Holder.index(indexName) - if index == nil { - return nil, ErrIndexNotFound - } - - frame := index.frame(frameName) - if frame == nil { - return nil, ErrFrameNotFound - } - - return frame.GetFields() -} - -// Views returns the views in the given frame. -func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { +// Views returns the views in the given field. +func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { if err := api.validate(apiViews); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve views. - f := api.Holder.Frame(indexName, frameName) + f := api.holder.Field(indexName, fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Fetch views. - views := f.Views() + views := f.views() return views, nil } // DeleteView removes the given view. -func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { +func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { return errors.Wrap(err, "validating api method") } - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) + // Retrieve field. + f := api.holder.Field(indexName, fieldName) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Delete the view. - if err := f.DeleteView(viewName); err != nil { - // Ignore this error because views do not exist on all nodes due to slice distribution. + if err := f.deleteView(viewName); err != nil { + // Ignore this error because views do not exist on all nodes due to shard distribution. if err != ErrInvalidView { return errors.Wrap(err, "deleting view") } } // Send the delete view message to all nodes. - err := api.Broadcaster.SendSync( - &internal.DeleteViewMessage{ + err := api.server.SendSync( + &DeleteViewMessage{ Index: indexName, - Frame: frameName, + Field: fieldName, View: viewName, }) if err != nil { - api.Logger.Printf("problem sending DeleteView message: %s", err) + api.server.logger.Printf("problem sending DeleteView message: %s", err) } return errors.Wrap(err, "sending DeleteView message") @@ -625,9 +541,9 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At } // Retrieve index from holder. - index := api.Holder.Index(indexName) + index := api.holder.Index(indexName) if index == nil { - return nil, ErrIndexNotFound + return nil, newNotFoundError(ErrIndexNotFound) } // Retrieve local blocks. @@ -638,7 +554,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At // Read all attributes from all mismatched blocks. attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { // Retrieve block data. m, err := index.ColumnAttrStore().BlockData(blockID) if err != nil { @@ -653,15 +569,15 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At return attrs, nil } -func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { - if err := api.validate(apiFrameAttrDiff); err != nil { +func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFieldAttrDiff); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve index from holder. - f := api.Holder.Frame(indexName, frameName) + f := api.holder.Field(indexName, fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Retrieve local blocks. @@ -672,7 +588,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Read all attributes from all mismatched blocks. attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + for _, blockID := range attrBlocks(localBlocks).Diff(blocks) { // Retrieve block data. m, err := f.RowAttrStore().BlockData(blockID) if err != nil { @@ -687,15 +603,15 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s return attrs, nil } -// Import bulk imports data into a particular index,frame,slice. -func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { +// Import bulk imports data into a particular index,field,shard. +func (api *API) Import(ctx context.Context, req *ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } // Convert timestamps to time.Time. @@ -709,77 +625,76 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // Import into fragment. - err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } // ImportValue bulk imports values into a particular field. -func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { +func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } - // Import into fragment. - err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + err = field.importValue(req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } -// MaxSlices returns the maximum slice number for each index in a map. -func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { - return api.Holder.MaxSlices() +// MaxShards returns the maximum shard number for each index in a map. +func (api *API) MaxShards(ctx context.Context) map[string]uint64 { + return api.holder.maxShards() } // StatsWithTags returns an instance of whatever implementation of StatsClient // pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) StatsClient { - if api.Holder == nil || api.Cluster == nil { + if api.holder == nil || api.cluster == nil { return nil } - return api.Holder.Stats.WithTags(tags...) + return api.holder.Stats.WithTags(tags...) } // LongQueryTime returns the configured threshold for logging/statting // long running queries. func (api *API) LongQueryTime() time.Duration { - if api.Cluster == nil { + if api.cluster == nil { return 0 } - return api.Cluster.LongQueryTime + return api.cluster.longQueryTime } -func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { - // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return nil, nil, ErrClusterDoesNotOwnSlice +func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) { + // Validate that this handler owns the shard. + if !api.cluster.ownsShard(api.Node().ID, indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName) + return nil, nil, ErrClusterDoesNotOwnShard } // Find the Index. - api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) - index := api.Holder.Index(indexName) + api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard) + index := api.holder.Index(indexName) if index == nil { - api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) - return nil, nil, ErrIndexNotFound + api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) + return nil, nil, newNotFoundError(ErrIndexNotFound) } - // Retrieve frame. - frame := index.Frame(frameName) - if frame == nil { - api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) - return nil, nil, ErrFrameNotFound + // Retrieve field. + field := index.Field(fieldName) + if field == nil { + api.server.logger.Printf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error()) + return nil, nil, ErrFieldNotFound } - return index, frame, nil + return index, field, nil } // SetCoordinator makes a new Node the cluster coordinator. @@ -788,22 +703,22 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validating api method") } - oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.NodeByID(id) + oldNode = api.cluster.nodeByID(api.cluster.Coordinator) + newNode = api.cluster.nodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } // If the new coordinator is this node, do the SetCoordinator directly. - if newNode.ID == api.LocalID() { - return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + if newNode.ID == api.Node().ID { + return oldNode, newNode, api.cluster.setCoordinator(newNode) } // Send the set-coordinator message to new node. - err = api.Broadcaster.SendTo( + err = api.server.SendTo( newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), + &SetCoordinatorMessage{ + New: newNode, }) if err != nil { return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) @@ -818,13 +733,13 @@ func (api *API) RemoveNode(id string) (*Node, error) { return nil, errors.Wrap(err, "validating api method") } - removeNode := api.Cluster.nodeByID(id) + removeNode := api.cluster.unprotectedNodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") } // Start the resize process (similar to NodeJoin) - err := api.Cluster.NodeLeave(removeNode) + err := api.cluster.nodeLeave(removeNode) if err != nil { return removeNode, errors.Wrap(err, "calling node leave") } @@ -837,15 +752,57 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validating api method") } - err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + err := api.cluster.completeCurrentJob(resizeJobStateAborted) return errors.Wrap(err, "complete current job") } +// translateStoreBufferSize is the buffer size used for streaming data. +const translateStoreBufferSize = 65536 + +func (api *API) GetTranslateData(ctx context.Context, w io.WriteCloser, offset int64) error { + rc, err := api.server.primaryTranslateStore.Reader(ctx, offset) + if err != nil { + return errors.Wrap(err, "read from translate store") + } + + // Ensure reader is closed when the client disconnects. + go func() { <-ctx.Done(); rc.Close() }() + + go func() { + defer rc.Close() + defer w.Close() + + buf := make([]byte, translateStoreBufferSize) + + // Copy from reader to client until store or client disconnect. + for { + // Read from store. + n, err := rc.Read(buf) + if err == io.EOF { + return + } else if err != nil { + api.server.logger.Printf("api: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + api.server.logger.Printf("api: translate store response write error: %s", err) + return + } + } + }() + + return nil +} + // State returns the cluster state which is usually "NORMAL", but could be // "STARTING", "RESIZING", or potentially others. See cluster.go for more // details. func (api *API) State() string { - return api.Cluster.State() + return api.cluster.State() } // Version returns the Pilosa version. @@ -854,14 +811,14 @@ func (api *API) Version() string { } // Info returns information about this server instance -func (api *API) Info() ServerInfo { - return ServerInfo{ - SliceWidth: SliceWidth, +func (api *API) Info() serverInfo { + return serverInfo{ + ShardWidth: ShardWidth, } } -type ServerInfo struct { - SliceWidth uint64 `json:"sliceWidth"` +type serverInfo struct { + ShardWidth uint64 `json:"shardWidth"` } type apiMethod int @@ -870,17 +827,15 @@ type apiMethod int const ( apiClusterMessage apiMethod = iota apiCreateField - apiCreateFrame apiCreateIndex apiDeleteField - apiDeleteFrame apiDeleteIndex apiDeleteView apiExportCSV - apiFields apiFragmentBlockData apiFragmentBlocks - apiFrameAttrDiff + apiField + apiFieldAttrDiff //apiHosts // not implemented apiImport apiImportValue @@ -888,26 +843,23 @@ const ( apiIndexAttrDiff //apiLocalID // not implemented //apiLongQueryTime // not implemented - apiMarshalFragment - //apiMaxSlices // not implemented + //apiMaxShards // not implemented apiQuery apiRecalculateCaches apiRemoveNode apiResizeAbort //apiSchema // not implemented apiSetCoordinator - apiSliceNodes + apiShardNodes //apiState // not implemented //apiStatsWithTags // not implemented - apiUnmarshalFragment //apiVersion // not implemented apiViews ) var methodsCommon = map[apiMethod]struct{}{ - apiClusterMessage: struct{}{}, - apiMarshalFragment: struct{}{}, - apiSetCoordinator: struct{}{}, + apiClusterMessage: struct{}{}, + apiSetCoordinator: struct{}{}, } var methodsResizing = map[apiMethod]struct{}{ @@ -916,17 +868,15 @@ var methodsResizing = map[apiMethod]struct{}{ var methodsNormal = map[apiMethod]struct{}{ apiCreateField: struct{}{}, - apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, apiDeleteField: struct{}{}, - apiDeleteFrame: struct{}{}, apiDeleteIndex: struct{}{}, apiDeleteView: struct{}{}, apiExportCSV: struct{}{}, - apiFields: struct{}{}, apiFragmentBlockData: struct{}{}, apiFragmentBlocks: struct{}{}, - apiFrameAttrDiff: struct{}{}, + apiField: struct{}{}, + apiFieldAttrDiff: struct{}{}, apiImport: struct{}{}, apiImportValue: struct{}{}, apiIndex: struct{}{}, @@ -934,7 +884,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiQuery: struct{}{}, apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, - apiSliceNodes: struct{}{}, - apiUnmarshalFragment: struct{}{}, + apiShardNodes: struct{}{}, apiViews: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index 2eb2913ae..01217092f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/attr.go b/attr.go index 03ea4f43f..0f553be95 100644 --- a/attr.go +++ b/attr.go @@ -24,10 +24,10 @@ import ( // Attribute data type enum. const ( - AttrTypeString = 1 - AttrTypeInt = 2 - AttrTypeBool = 3 - AttrTypeFloat = 4 + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 ) // AttrStore represents an interface for handling row/column attributes. @@ -42,57 +42,39 @@ type AttrStore interface { BlockData(i uint64) (map[uint64]map[string]interface{}, error) } -func init() { - NopAttrStore = &nopAttrStore{} -} +// nopStore represents an AttrStore that doesn't do anything. +var nopStore AttrStore = nopAttrStore{} -// NopAttrStore represents an AttrStore that doesn't do anything. -var NopAttrStore AttrStore - -func NewNopAttrStore(string) AttrStore { - return &nopAttrStore{} -} +// newNopAttrStore returns an attr store which does nothing. It returns a global +// object to avoid unecessary allocations. +func newNopAttrStore(string) AttrStore { return nopStore } // nopAttrStore represents a no-op implementation of the AttrStore interface. type nopAttrStore struct{} // Path is a no-op implementation of AttrStore Path method. -func (s *nopAttrStore) Path() string { return "" } +func (s nopAttrStore) Path() string { return "" } // Open is a no-op implementation of AttrStore Open method. -func (s *nopAttrStore) Open() error { - return nil -} +func (s nopAttrStore) Open() error { return nil } // Close is a no-op implementation of AttrStore Close method. -func (s *nopAttrStore) Close() error { - return nil -} +func (s nopAttrStore) Close() error { return nil } // Attrs is a no-op implementation of AttrStore Attrs method. -func (s *nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { - return nil, nil -} +func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil } // SetAttrs is a no-op implementation of AttrStore SetAttrs method. -func (s *nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { - return nil -} +func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil } // SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method. -func (s *nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { - return nil -} +func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil } // Blocks is a no-op implementation of AttrStore Blocks method. -func (s *nopAttrStore) Blocks() ([]AttrBlock, error) { - return nil, nil -} +func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } // BlockData is a no-op implementation of AttrStore BlockData method. -func (s *nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { - return nil, nil -} +func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } // AttrBlock represents a checksummed block of the attribute store. type AttrBlock struct { @@ -100,12 +82,12 @@ type AttrBlock struct { Checksum []byte `json:"checksum"` } -// AttrBlocks represents a list of blocks. -type AttrBlocks []AttrBlock +// attrBlocks represents a list of blocks. +type attrBlocks []AttrBlock // Diff returns a list of block ids that are different or are new in other. // Block lists must be in sorted order. -func (a AttrBlocks) Diff(other []AttrBlock) []uint64 { +func (a attrBlocks) Diff(other []AttrBlock) []uint64 { var ids []uint64 for { // Read next block from each list. @@ -165,19 +147,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr { pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.Type = AttrTypeString + pb.Type = attrTypeString pb.StringValue = value case float64: - pb.Type = AttrTypeFloat + pb.Type = attrTypeFloat pb.FloatValue = value case uint64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = int64(value) case int64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = value case bool: - pb.Type = AttrTypeBool + pb.Type = attrTypeBool pb.BoolValue = value } return pb @@ -186,13 +168,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr { // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { switch attr.Type { - case AttrTypeString: + case attrTypeString: return attr.Key, attr.StringValue - case AttrTypeInt: + case attrTypeInt: return attr.Key, attr.IntValue - case AttrTypeBool: + case attrTypeBool: return attr.Key, attr.BoolValue - case AttrTypeFloat: + case attrTypeFloat: return attr.Key, attr.FloatValue default: return attr.Key, nil @@ -221,3 +203,31 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) { } return decodeAttrs(pb.GetAttrs()), nil } + +func newMemAttrStore() AttrStore { + return &memAttrStore{ + store: make(map[uint64]map[string]interface{}), + } +} + +// memAttrStore represents an in-memory implementation of the AttrStore interface. +type memAttrStore struct { + store map[uint64]map[string]interface{} +} + +func (s *memAttrStore) Path() string { return "" } +func (s *memAttrStore) Open() error { return nil } +func (s *memAttrStore) Close() error { return nil } +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return s.store[id], nil } +func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + s.store[id] = m + return nil +} +func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + for id, v := range m { + s.store[id] = v + } + return nil +} +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil } +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil } diff --git a/attr_test.go b/attr_test.go index 8253c8e1d..0c848e2ff 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,15 +15,20 @@ package pilosa_test import ( + "io/ioutil" + "os" "reflect" + "runtime" + "sync" "testing" - "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) { t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) } } + +// AttrStore represents a test wrapper for pilosa.AttrStore. +type AttrStore struct { + pilosa.AttrStore +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(string) pilosa.AttrStore { + f, err := ioutil.TempFile("", "pilosa-attr-") + if err != nil { + panic(err) + } + f.Close() + os.Remove(f.Name()) + + return &AttrStore{boltdb.NewAttrStore(f.Name())} +} + +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + +// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +// Close closes the database and removes the underlying data. +func (s *AttrStore) Close() error { + defer os.RemoveAll(s.Path()) + return s.AttrStore.Close() +} diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index 3604de5b6..1a9ddcf6e 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -30,17 +30,17 @@ import ( "github.com/pkg/errors" ) -// AttrBlockSize is the size of attribute blocks for anti-entropy. -const AttrBlockSize = 100 +// attrBlockSize is the size of attribute blocks for anti-entropy. +const attrBlockSize = 100 -// AttrCache represents a cache for attributes. -type AttrCache struct { +// attrCache represents a cache for attributes. +type attrCache struct { mu sync.RWMutex attrs map[uint64]map[string]interface{} } // Get returns the cached attributes for a given id. -func (c *AttrCache) Get(id uint64) map[string]interface{} { +func (c *attrCache) Get(id uint64) map[string]interface{} { c.mu.RLock() defer c.mu.RUnlock() attrs := c.attrs[id] @@ -57,40 +57,40 @@ func (c *AttrCache) Get(id uint64) map[string]interface{} { } // Set updates the cached attributes for a given id. -func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) { +func (c *attrCache) Set(id uint64, attrs map[string]interface{}) { c.mu.Lock() defer c.mu.Unlock() c.attrs[id] = attrs } -// AttrStore represents a storage layer for attributes. -type AttrStore struct { +// attrStore represents a storage layer for attributes. +type attrStore struct { mu sync.RWMutex path string db *bolt.DB - attrCache *AttrCache + attrCache *attrCache } -// NewAttrCache returns a new instance of AttrCache. -func NewAttrCache() *AttrCache { - return &AttrCache{ +// newAttrCache returns a new instance of AttrCache. +func newAttrCache() *attrCache { + return &attrCache{ attrs: make(map[uint64]map[string]interface{}), } } // NewAttrStore returns a new instance of AttrStore. func NewAttrStore(path string) pilosa.AttrStore { - return &AttrStore{ + return &attrStore{ path: path, - attrCache: NewAttrCache(), + attrCache: newAttrCache(), } } // Path returns path to the store's data file. -func (s *AttrStore) Path() string { return s.path } +func (s *attrStore) Path() string { return s.path } // Open opens and initializes the store. -func (s *AttrStore) Open() error { +func (s *attrStore) Open() error { // Open storage. db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { @@ -112,7 +112,7 @@ func (s *AttrStore) Open() error { } // Close closes the store. -func (s *AttrStore) Close() error { +func (s *attrStore) Close() error { if s.db != nil { s.db.Close() } @@ -120,7 +120,7 @@ func (s *AttrStore) Close() error { } // Attrs returns a set of attributes by ID. -func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { +func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) { s.mu.RLock() defer s.mu.RUnlock() @@ -147,7 +147,7 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { } // SetAttrs sets attribute values for a given ID. -func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { +func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error { // Ignore empty maps. if len(m) == 0 { return nil @@ -184,7 +184,7 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { } // SetBulkAttrs sets attribute values for a set of ids. -func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { +func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { s.mu.Lock() defer s.mu.Unlock() @@ -220,7 +220,7 @@ func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { } // Blocks returns a list of all blocks in the store. -func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { +func (s *attrStore) Blocks() ([]pilosa.AttrBlock, error) { tx, err := s.db.Begin(false) if err != nil { return nil, errors.Wrap(err, "starting transaction") @@ -228,7 +228,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { defer tx.Rollback() // Wrap cursor to segment by block. - cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize) + cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize) // Iterate over each block. var blocks []pilosa.AttrBlock @@ -251,7 +251,7 @@ func (s *AttrStore) Blocks() ([]pilosa.AttrBlock, error) { } // BlockData returns all data for a single block. -func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { +func (s *attrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { m := make(map[uint64]map[string]interface{}) // Start read-only transaction. @@ -262,8 +262,8 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro defer tx.Rollback() // Move to the start of the block. - min := u64tob(uint64(i) * AttrBlockSize) - max := u64tob(uint64(i+1) * AttrBlockSize) + min := u64tob(uint64(i) * attrBlockSize) + max := u64tob(uint64(i+1) * attrBlockSize) cur := tx.Bucket([]byte("attrs")).Cursor() for k, v := cur.Seek(min); k != nil; k, v = cur.Next() { // Exit if we're past the end of the block. diff --git a/broadcast.go b/broadcast.go index 77b76126d..a3ea01a4f 100644 --- a/broadcast.go +++ b/broadcast.go @@ -16,223 +16,150 @@ package pilosa import ( "fmt" - "reflect" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) -// MemberSet represents an interface for Node membership and inter-node communication. -type MemberSet interface { - // Open starts any network activity implemented by the MemberSet - // Node is the local node, used for membership broadcasts. - Open(n *Node) error +// Serializer is an interface for serializing pilosa types to bytes and back. +type Serializer interface { + Marshal(Message) ([]byte, error) + Unmarshal([]byte, Message) error } -// StaticMemberSet represents a basic MemberSet for testing. -type StaticMemberSet struct { - nodes []*Node +// broadcaster is an interface for broadcasting messages. +type broadcaster interface { + SendSync(Message) error + SendAsync(Message) error + SendTo(*Node, Message) error } -// NewStaticMemberSet creates a statically defined MemberSet. -func NewStaticMemberSet(nodes []*Node) *StaticMemberSet { - return &StaticMemberSet{ - nodes: nodes, - } -} - -// Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. -func (s *StaticMemberSet) Open(n *Node) error { - return nil -} - -// Broadcaster is an interface for broadcasting messages. -type Broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error - SendTo(to *Node, pb proto.Message) error -} +// Message is the interface implemented by all core pilosa types which can be serialized to messages. +// TODO add at least a single "isMessage()" method. +type Message interface{} func init() { NopBroadcaster = &nopBroadcaster{} - NopGossiper = &nopGossiper{} } // NopBroadcaster represents a Broadcaster that doesn't do anything. -var NopBroadcaster Broadcaster +var NopBroadcaster broadcaster type nopBroadcaster struct{} // SendSync A no-op implementation of Broadcaster SendSync method. -func (n *nopBroadcaster) SendSync(pb proto.Message) error { - return nil -} +func (nopBroadcaster) SendSync(Message) error { return nil } // SendAsync A no-op implementation of Broadcaster SendAsync method. -func (n *nopBroadcaster) SendAsync(pb proto.Message) error { - return nil -} +func (nopBroadcaster) SendAsync(Message) error { return nil } // SendTo is a no-op implementation of Broadcaster SendTo method. -func (c *nopBroadcaster) SendTo(to *Node, pb proto.Message) error { - return nil -} - -// BroadcastHandler is the interface for the pilosa object which knows how to -// handle broadcast messages. (Hint: this is implemented by pilosa.Server) -type BroadcastHandler interface { - ReceiveMessage(pb proto.Message) error -} - -// BroadcastReceiver is the interface for the object which will listen for and -// decode broadcast messages before passing them to pilosa to handle. The -// implementation of this could be an http server which listens for messages, -// gets the protobuf payload, and then passes it to -// BroadcastHandler.ReceiveMessage. -type BroadcastReceiver interface { - // Start starts listening for broadcast messages - it should return - // immediately, spawning a goroutine if necessary. - Start(BroadcastHandler) error -} - -type nopBroadcastReceiver struct{} - -func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } - -// NopBroadcastReceiver is a no-op implementation of the BroadcastReceiver. -var NopBroadcastReceiver = &nopBroadcastReceiver{} - -// Gossiper is an interface for sharing messages via gossip. -type Gossiper interface { - SendAsync(pb proto.Message) error -} - -// NopBroadcaster represents a Broadcaster that doesn't do anything. -var NopGossiper Gossiper - -type nopGossiper struct{} - -// SendAsync A no-op implementation of Gossiper SendAsync method. -func (n *nopGossiper) SendAsync(pb proto.Message) error { - return nil -} +func (nopBroadcaster) SendTo(*Node, Message) error { return nil } // Broadcast message types. const ( - MessageTypeCreateSlice = iota - MessageTypeCreateIndex - MessageTypeDeleteIndex - MessageTypeCreateFrame - MessageTypeDeleteFrame - MessageTypeCreateView - MessageTypeDeleteView - MessageTypeCreateField - MessageTypeDeleteField - MessageTypeClusterStatus - MessageTypeResizeInstruction - MessageTypeResizeInstructionComplete - MessageTypeSetCoordinator - MessageTypeUpdateCoordinator - MessageTypeNodeState - MessageTypeRecalculateCaches - MessageTypeNodeEvent + messageTypeCreateShard = iota + messageTypeCreateIndex + messageTypeDeleteIndex + messageTypeCreateField + messageTypeDeleteField + messageTypeCreateView + messageTypeDeleteView + messageTypeClusterStatus + messageTypeResizeInstruction + messageTypeResizeInstructionComplete + messageTypeSetCoordinator + messageTypeUpdateCoordinator + messageTypeNodeState + messageTypeRecalculateCaches + messageTypeNodeEvent + messageTypeNodeStatus ) -// MarshalMessage encodes the protobuf message into a byte slice. -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice - case *internal.CreateIndexMessage: - typ = MessageTypeCreateIndex - case *internal.DeleteIndexMessage: - typ = MessageTypeDeleteIndex - case *internal.CreateFrameMessage: - typ = MessageTypeCreateFrame - case *internal.DeleteFrameMessage: - typ = MessageTypeDeleteFrame - case *internal.CreateViewMessage: - typ = MessageTypeCreateView - case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView - case *internal.CreateFieldMessage: - typ = MessageTypeCreateField - case *internal.DeleteFieldMessage: - typ = MessageTypeDeleteField - case *internal.ClusterStatus: - typ = MessageTypeClusterStatus - case *internal.ResizeInstruction: - typ = MessageTypeResizeInstruction - case *internal.ResizeInstructionComplete: - typ = MessageTypeResizeInstructionComplete - case *internal.SetCoordinatorMessage: - typ = MessageTypeSetCoordinator - case *internal.UpdateCoordinatorMessage: - typ = MessageTypeUpdateCoordinator - case *internal.NodeStateMessage: - typ = MessageTypeNodeState - case *internal.RecalculateCaches: - typ = MessageTypeRecalculateCaches - case *internal.NodeEventMessage: - typ = MessageTypeNodeEvent - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) - } - buf, err := proto.Marshal(m) +// MarshalInternalMessage serializes the pilosa message and adds pilosa internal +// type info which is used by the internal messaging stuff. +func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) { + typ := getMessageType(m) + buf, err := s.Marshal(m) if err != nil { - return nil, errors.Wrap(err, "marshalling") + return nil, errors.Wrap(err, "marshaling") } return append([]byte{typ}, buf...), nil } -// UnmarshalMessage decodes the byte slice into a protobuf message. -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - - var m proto.Message +func getMessage(typ byte) Message { switch typ { - case MessageTypeCreateSlice: - m = &internal.CreateSliceMessage{} - case MessageTypeCreateIndex: - m = &internal.CreateIndexMessage{} - case MessageTypeDeleteIndex: - m = &internal.DeleteIndexMessage{} - case MessageTypeCreateFrame: - m = &internal.CreateFrameMessage{} - case MessageTypeDeleteFrame: - m = &internal.DeleteFrameMessage{} - case MessageTypeCreateView: - m = &internal.CreateViewMessage{} - case MessageTypeDeleteView: - m = &internal.DeleteViewMessage{} - case MessageTypeCreateField: - m = &internal.CreateFieldMessage{} - case MessageTypeDeleteField: - m = &internal.DeleteFieldMessage{} - case MessageTypeClusterStatus: - m = &internal.ClusterStatus{} - case MessageTypeResizeInstruction: - m = &internal.ResizeInstruction{} - case MessageTypeResizeInstructionComplete: - m = &internal.ResizeInstructionComplete{} - case MessageTypeSetCoordinator: - m = &internal.SetCoordinatorMessage{} - case MessageTypeUpdateCoordinator: - m = &internal.UpdateCoordinatorMessage{} - case MessageTypeNodeState: - m = &internal.NodeStateMessage{} - case MessageTypeRecalculateCaches: - m = &internal.RecalculateCaches{} - case MessageTypeNodeEvent: - m = &internal.NodeEventMessage{} + case messageTypeCreateShard: + return &CreateShardMessage{} + case messageTypeCreateIndex: + return &CreateIndexMessage{} + case messageTypeDeleteIndex: + return &DeleteIndexMessage{} + case messageTypeCreateField: + return &CreateFieldMessage{} + case messageTypeDeleteField: + return &DeleteFieldMessage{} + case messageTypeCreateView: + return &CreateViewMessage{} + case messageTypeDeleteView: + return &DeleteViewMessage{} + case messageTypeClusterStatus: + return &ClusterStatus{} + case messageTypeResizeInstruction: + return &ResizeInstruction{} + case messageTypeResizeInstructionComplete: + return &ResizeInstructionComplete{} + case messageTypeSetCoordinator: + return &SetCoordinatorMessage{} + case messageTypeUpdateCoordinator: + return &UpdateCoordinatorMessage{} + case messageTypeNodeState: + return &NodeStateMessage{} + case messageTypeRecalculateCaches: + return &RecalculateCaches{} + case messageTypeNodeEvent: + return &NodeEvent{} + case messageTypeNodeStatus: + return &NodeStatus{} default: - return nil, fmt.Errorf("invalid message type: %d", typ) + panic(fmt.Sprintf("unknown message type %d", typ)) + } +} + +func getMessageType(m Message) byte { + switch m.(type) { + case *CreateShardMessage: + return messageTypeCreateShard + case *CreateIndexMessage: + return messageTypeCreateIndex + case *DeleteIndexMessage: + return messageTypeDeleteIndex + case *CreateFieldMessage: + return messageTypeCreateField + case *DeleteFieldMessage: + return messageTypeDeleteField + case *CreateViewMessage: + return messageTypeCreateView + case *DeleteViewMessage: + return messageTypeDeleteView + case *ClusterStatus: + return messageTypeClusterStatus + case *ResizeInstruction: + return messageTypeResizeInstruction + case *ResizeInstructionComplete: + return messageTypeResizeInstructionComplete + case *SetCoordinatorMessage: + return messageTypeSetCoordinator + case *UpdateCoordinatorMessage: + return messageTypeUpdateCoordinator + case *NodeStateMessage: + return messageTypeNodeState + case *RecalculateCaches: + return messageTypeRecalculateCaches + case *NodeEvent: + return messageTypeNodeEvent + case *NodeStatus: + return messageTypeNodeStatus + default: + panic(fmt.Sprintf("don't have type for message %#v", m)) } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - return m, nil } diff --git a/broadcast_test.go b/broadcast_test.go deleted file mode 100644 index 970a249cb..000000000 --- a/broadcast_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "bytes" - "reflect" - "testing" - - "io/ioutil" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/server" -) - -// Ensure a message can be marshaled and unmarshaled. -func TestMessage_Marshal(t *testing.T) { - - testMessageMarshal(t, &internal.CreateSliceMessage{ - Index: "i", - Slice: 8, - }) - - testMessageMarshal(t, &internal.DeleteIndexMessage{ - Index: "i", - }) -} - -func testMessageMarshal(t *testing.T, m proto.Message) { - marshalled, err := pilosa.MarshalMessage(m) - if err != nil { - t.Fatal(err) - } - unmarshalled, err := pilosa.UnmarshalMessage(marshalled) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(unmarshalled, m) { - t.Fatalf("unexpected message marshalling: %s", unmarshalled) - } -} - -// Ensure that BroadcastReceiver can register a BroadcastHandler. -func TestBroadcast_BroadcastReceiver(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-") - if err != nil { - panic(err) - } - com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) - com.Config.Bind = "localhost:0" - com.Config.DataDir = path - err = com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO - if err != nil { - t.Fatalf("setting up server: %v", err) - } - s := com.Server - - sbr := NewSimpleBroadcastReceiver() - sbh := NewSimpleBroadcastHandler() - - s.BroadcastReceiver = sbr - s.BroadcastReceiver.Start(sbh) - - msg := &internal.DeleteIndexMessage{ - Index: "i", - } - - s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) - - // Make sure the message received is what was sentd - if !reflect.DeepEqual(sbh.receivedMessage, msg) { - t.Fatalf("unexpected message: %s", sbh.receivedMessage) - } -} - -type SimpleBroadcastReceiver struct { - broadcastHandler pilosa.BroadcastHandler -} - -func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver { - return &SimpleBroadcastReceiver{} -} - -func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error { - r.broadcastHandler = h - return nil -} - -func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error { - r.broadcastHandler.ReceiveMessage(pb) - return nil -} - -type SimpleBroadcastHandler struct { - receivedMessage proto.Message -} - -func NewSimpleBroadcastHandler() *SimpleBroadcastHandler { - return &SimpleBroadcastHandler{} -} - -func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error { - h.receivedMessage = pb.(proto.Message) - return nil -} diff --git a/cache.go b/cache.go index 06046220a..044d85f3c 100644 --- a/cache.go +++ b/cache.go @@ -22,17 +22,16 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/lru" ) const ( - // ThresholdFactor is used to calculate the threshold for new items entering the cache - ThresholdFactor = 1.1 + // thresholdFactor is used to calculate the threshold for new items entering the cache + thresholdFactor = 1.1 ) -// Cache represents a cache of counts. -type Cache interface { +// cache represents a cache of counts. +type cache interface { Add(id uint64, n uint64) BulkAdd(id uint64, n uint64) Get(id uint64) uint64 @@ -48,22 +47,22 @@ type Cache interface { Recalculate() // Returns an ordered list of the top ranked bitmaps. - Top() []BitmapPair + Top() []bitmapPair // SetStats defines the stats client used in the cache. SetStats(s StatsClient) } -// LRUCache represents a least recently used Cache implementation. -type LRUCache struct { +// lruCache represents a least recently used Cache implementation. +type lruCache struct { cache *lru.Cache counts map[uint64]uint64 stats StatsClient } -// NewLRUCache returns a new instance of LRUCache. -func NewLRUCache(maxEntries uint32) *LRUCache { - c := &LRUCache{ +// newLRUCache returns a new instance of LRUCache. +func newLRUCache(maxEntries uint32) *lruCache { + c := &lruCache{ cache: lru.New(int(maxEntries)), counts: make(map[uint64]uint64), stats: NopStatsClient, @@ -73,34 +72,34 @@ func NewLRUCache(maxEntries uint32) *LRUCache { } // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion. -func (c *LRUCache) BulkAdd(id, n uint64) { +func (c *lruCache) BulkAdd(id, n uint64) { c.Add(id, n) } // Add adds a count to the cache. -func (c *LRUCache) Add(id, n uint64) { +func (c *lruCache) Add(id, n uint64) { c.cache.Add(id, n) c.counts[id] = n } // Get returns a count for a given id. -func (c *LRUCache) Get(id uint64) uint64 { +func (c *lruCache) Get(id uint64) uint64 { n, _ := c.cache.Get(id) nn, _ := n.(uint64) return nn } // Len returns the number of items in the cache. -func (c *LRUCache) Len() int { return c.cache.Len() } +func (c *lruCache) Len() int { return c.cache.Len() } // Invalidate is a no-op. -func (c *LRUCache) Invalidate() {} +func (c *lruCache) Invalidate() {} // Recalculate is a no-op. -func (c *LRUCache) Recalculate() {} +func (c *lruCache) Recalculate() {} // IDs returns a list of all IDs in the cache. -func (c *LRUCache) IDs() []uint64 { +func (c *lruCache) IDs() []uint64 { a := make([]uint64, 0, len(c.counts)) for id := range c.counts { a = append(a, id) @@ -110,33 +109,33 @@ func (c *LRUCache) IDs() []uint64 { } // Top returns all counts in the cache. -func (c *LRUCache) Top() []BitmapPair { - a := make([]BitmapPair, 0, len(c.counts)) +func (c *lruCache) Top() []bitmapPair { + a := make([]bitmapPair, 0, len(c.counts)) for id, n := range c.counts { - a = append(a, BitmapPair{ + a = append(a, bitmapPair{ ID: id, Count: uint64(n), }) } - sort.Sort(BitmapPairs(a)) + sort.Sort(bitmapPairs(a)) return a } // SetStats defines the stats client used in the cache. -func (c *LRUCache) SetStats(s StatsClient) { +func (c *lruCache) SetStats(s StatsClient) { c.stats = s } -func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } +func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) } // Ensure LRUCache implements Cache. -var _ Cache = &LRUCache{} +var _ cache = &lruCache{} -// RankCache represents a cache with sorted entries. -type RankCache struct { +// rankCache represents a cache with sorted entries. +type rankCache struct { mu sync.Mutex entries map[uint64]uint64 - rankings []BitmapPair // cached, ordered list + rankings []bitmapPair // cached, ordered list updateN int updateTime time.Time @@ -155,17 +154,17 @@ type RankCache struct { } // NewRankCache returns a new instance of RankCache. -func NewRankCache(maxEntries uint32) *RankCache { - return &RankCache{ +func NewRankCache(maxEntries uint32) *rankCache { + return &rankCache{ maxEntries: maxEntries, - thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), stats: NopStatsClient, } } // Add adds a count to the cache. -func (c *RankCache) Add(id uint64, n uint64) { +func (c *rankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() // Ignore if the column count is below the threshold. @@ -179,7 +178,7 @@ func (c *RankCache) Add(id uint64, n uint64) { } // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion. -func (c *RankCache) BulkAdd(id uint64, n uint64) { +func (c *rankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() if n < c.thresholdValue { @@ -190,21 +189,21 @@ func (c *RankCache) BulkAdd(id uint64, n uint64) { } // Get returns a count for a given id. -func (c *RankCache) Get(id uint64) uint64 { +func (c *rankCache) Get(id uint64) uint64 { c.mu.Lock() defer c.mu.Unlock() return c.entries[id] } // Len returns the number of items in the cache. -func (c *RankCache) Len() int { +func (c *rankCache) Len() int { c.mu.Lock() defer c.mu.Unlock() return len(c.entries) } // IDs returns a list of all IDs in the cache. -func (c *RankCache) IDs() []uint64 { +func (c *rankCache) IDs() []uint64 { c.mu.Lock() defer c.mu.Unlock() a := make([]uint64, 0, len(c.entries)) @@ -216,21 +215,21 @@ func (c *RankCache) IDs() []uint64 { } // Invalidate recalculates the entries by rank. -func (c *RankCache) Invalidate() { +func (c *rankCache) Invalidate() { c.mu.Lock() defer c.mu.Unlock() c.invalidate() } // Recalculate rebuilds the cache. -func (c *RankCache) Recalculate() { +func (c *rankCache) Recalculate() { c.mu.Lock() defer c.mu.Unlock() c.stats.Count("cache.recalculate", 1, 1.0) c.recalculate() } -func (c *RankCache) invalidate() { +func (c *rankCache) invalidate() { // Don't invalidate more than once every X seconds. // TODO: consider making this configurable. if time.Since(c.updateTime).Seconds() < 10 { @@ -240,23 +239,23 @@ func (c *RankCache) invalidate() { c.recalculate() } -func (c *RankCache) recalculate() { +func (c *rankCache) recalculate() { // Convert cache to a sorted list. - rankings := make([]BitmapPair, 0, len(c.entries)) + rankings := make([]bitmapPair, 0, len(c.entries)) for id, cnt := range c.entries { - rankings = append(rankings, BitmapPair{ + rankings = append(rankings, bitmapPair{ ID: id, Count: cnt, }) } - sort.Sort(BitmapPairs(rankings)) + sort.Sort(bitmapPairs(rankings)) // Store the count of the item at the threshold index. c.rankings = rankings length := len(c.rankings) c.stats.Gauge("RankCache", float64(length), 1.0) - var removeItems []BitmapPair // cached, ordered list + var removeItems []bitmapPair // cached, ordered list if length > int(c.maxEntries) { c.thresholdValue = rankings[c.maxEntries].Count removeItems = c.rankings[c.maxEntries:] @@ -278,38 +277,38 @@ func (c *RankCache) recalculate() { } // SetStats defines the stats client used in the cache. -func (c *RankCache) SetStats(s StatsClient) { +func (c *rankCache) SetStats(s StatsClient) { c.stats = s } // Top returns an ordered list of pairs. -func (c *RankCache) Top() []BitmapPair { return c.rankings } +func (c *rankCache) Top() []bitmapPair { return c.rankings } // WriteTo writes the cache to w. -func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) { +func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) { panic("FIXME: TODO") } // ReadFrom read from r into the cache. -func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) { +func (c *rankCache) ReadFrom(r io.Reader) (n int64, err error) { panic("FIXME: TODO") } // Ensure RankCache implements Cache. -var _ Cache = &RankCache{} +var _ cache = &rankCache{} -// BitmapPair represents a id/count pair with an associated identifier. -type BitmapPair struct { +// bitmapPair represents a id/count pair with an associated identifier. +type bitmapPair struct { ID uint64 Count uint64 } -// BitmapPairs is a sortable list of BitmapPair objects. -type BitmapPairs []BitmapPair +// bitmapPairs is a sortable list of BitmapPair objects. +type bitmapPairs []bitmapPair -func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitmapPairs) Len() int { return len(p) } -func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } +func (p bitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p bitmapPairs) Len() int { return len(p) } +func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } // Pair holds an id/count pair. type Pair struct { @@ -318,22 +317,6 @@ type Pair struct { Count uint64 `json:"count"` } -func encodePair(p Pair) *internal.Pair { - return &internal.Pair{ - ID: p.ID, - Key: p.Key, - Count: p.Count, - } -} - -func decodePair(pb *internal.Pair) Pair { - return Pair{ - ID: pb.ID, - Key: pb.Key, - Count: pb.Count, - } -} - // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -341,14 +324,14 @@ func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Pairs) Len() int { return len(p) } func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count } -// PairHeap is a heap implementation over a group of Pairs. -type PairHeap struct { +// pairHeap is a heap implementation over a group of Pairs. +type pairHeap struct { Pairs } // Less implemets the Sort interface. // reports whether the element with index i should sort before the element with index j. -func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count } +func (p pairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count } // Push appends the element onto the Pair slice. func (p *Pairs) Push(x interface{}) { @@ -409,22 +392,6 @@ func (p Pairs) String() string { return buf.String() } -func encodePairs(a Pairs) []*internal.Pair { - other := make([]*internal.Pair, len(a)) - for i := range a { - other[i] = encodePair(a[i]) - } - return other -} - -func decodePairs(a []*internal.Pair) []Pair { - other := make([]Pair, len(a)) - for i := range a { - other[i] = decodePair(a[i]) - } - return other -} - // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 @@ -461,60 +428,52 @@ func (p uint64Slice) merge(other []uint64) []uint64 { return ret } -// BitmapCache provides an interface for caching full bitmaps. -type BitmapCache interface { +// bitmapCache provides an interface for caching full bitmaps. +type bitmapCache interface { Fetch(id uint64) (*Row, bool) Add(id uint64, b *Row) } -// SimpleCache implements BitmapCache +// simpleCache implements BitmapCache // it is meant to be a short-lived cache for cases where writes are continuing to access // the same row within a short time frame (i.e. good for write-heavy loads) // A read-heavy use case would cause the cache to get bigger, potentially causing the // node to run out of memory. -type SimpleCache struct { +type simpleCache struct { cache map[uint64]*Row } // Fetch retrieves the bitmap at the id in the cache. -func (s *SimpleCache) Fetch(id uint64) (*Row, bool) { +func (s *simpleCache) Fetch(id uint64) (*Row, bool) { m, ok := s.cache[id] return m, ok } // Add adds the bitmap to the cache, keyed on the id. -func (s *SimpleCache) Add(id uint64, b *Row) { +func (s *simpleCache) Add(id uint64, b *Row) { s.cache[id] = b } -// NopCache represents a no-op Cache implementation. -type NopCache struct { +// nopCache represents a no-op Cache implementation. +type nopCache struct { stats StatsClient } // Ensure NopCache implements Cache. -var _ Cache = &NopCache{} - -// NewNopCache returns a new instance of NopCache. -func NewNopCache() *NopCache { - return &NopCache{ - stats: NopStatsClient, - } +var globalNopCache cache = nopCache{ + stats: NopStatsClient, } -func (c *NopCache) Add(id uint64, n uint64) {} -func (c *NopCache) BulkAdd(id uint64, n uint64) {} -func (c *NopCache) Get(id uint64) uint64 { return 0 } -func (c *NopCache) IDs() []uint64 { return make([]uint64, 0) } +func (c nopCache) Add(uint64, uint64) {} +func (c nopCache) BulkAdd(uint64, uint64) {} +func (c nopCache) Get(uint64) uint64 { return 0 } +func (c nopCache) IDs() []uint64 { return []uint64{} } -func (c *NopCache) Invalidate() {} -func (c *NopCache) Len() int { return 0 } -func (c *NopCache) Recalculate() { -} -func (c *NopCache) SetStats(s StatsClient) { - c.stats = s -} +func (c nopCache) Invalidate() {} +func (c nopCache) Len() int { return 0 } +func (c nopCache) Recalculate() {} +func (c nopCache) SetStats(StatsClient) {} -func (c *NopCache) Top() []BitmapPair { - return []BitmapPair{} +func (c nopCache) Top() []bitmapPair { + return []bitmapPair{} } diff --git a/client.go b/client.go index f3e0e5dce..6c912bd99 100644 --- a/client.go +++ b/client.go @@ -1,879 +1,10 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( - "bytes" "context" - "encoding/json" - "fmt" "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "sort" - "strconv" - - "crypto/tls" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" - "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - -// InternalHTTPClient represents a client to the Pilosa cluster. -type InternalHTTPClient struct { - defaultURI *URI - - // The client to use for HTTP communication. - HTTPClient *http.Client -} - -// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { - if host == "" { - return nil, ErrHostRequired - } - - uri, err := NewURIFromAddress(host) - if err != nil { - return nil, errors.Wrap(err, "getting URI") - } - - client := NewInternalHTTPClientFromURI(uri, remoteClient) - return client, nil -} - -func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { - return &InternalHTTPClient{ - defaultURI: defaultURI, - HTTPClient: remoteClient, - } -} - -// Host returns the host the client was initialized with. -func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } - -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) -} - -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSlicesMaxResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return rsp.Standard, nil -} - -// Schema returns all index and frame schema information. -func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { - // Execute request against the host. - u := c.defaultURI.Path("/schema") - - // Build request. - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSchemaResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - return rsp.Indexes, nil -} - -// CreateIndex creates a new index on the server. -func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - // Encode query request. - buf, err := json.Marshal(&postIndexRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "encoding request") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrIndexExists - default: - return errors.New(string(body)) - } -} - -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var a []*Node - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return a, nil -} - -// Query executes query against the index. -func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - return c.QueryNode(ctx, c.defaultURI, index, queryRequest) -} - -// QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - if index == "" { - return nil, ErrIndexRequired - } else if queryRequest.Query == "" { - return nil, ErrQueryRequired - } - - // Encode request object. - buf, err := proto.Marshal(queryRequest) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Create HTTP request. - u := uri.Path(fmt.Sprintf("/index/%s/query", index)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) - } - - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { - return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) - } - - return qresp, nil -} - -// Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportPayload(index, frame, slice, bits) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportPayloadK(index, frame, columns) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - node := &Node{ - URI: *c.defaultURI, - } - - // Import to node. - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - - return nil -} - -func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - err := c.CreateIndex(ctx, name, options) - if err == nil || err == ErrIndexExists { - return nil - } - return err -} - -func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { - err := c.CreateFrame(ctx, indexName, frameName, options) - if err == nil || err == ErrFrameExists { - return nil - } - return err -} - -// marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowIDs := Bits(bits).RowIDs() - columnIDs := Bits(bits).ColumnIDs() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Frame: frame, - Slice: slice, - RowIDs: rowIDs, - ColumnIDs: columnIDs, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowKeys := Bits(bits).RowKeys() - columnKeys := Bits(bits).ColumnKeys() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Frame: frame, - RowKeys: rowKeys, - ColumnKeys: columnKeys, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportValuePayload(index, frame, field, slice, vals) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - columnIDs := FieldValues(vals).ColumnIDs() - values := FieldValues(vals).Values() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ - Index: index, - Frame: frame, - Slice: slice, - Field: field, - ColumnIDs: columnIDs, - Values: values, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importValueNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import-value") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Attempt nodes in random order. - var e error - for _, i := range rand.Perm(len(nodes)) { - node := nodes[i] - - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) - continue - } else { - return nil - } - } - - return e -} - -// exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { - // Create URL. - u := nodePathToURL(node, "/export") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Generate HTTP request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Validate status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: %d", resp.StatusCode) - } - - // Copy body to writer. - if _, err := io.Copy(w, resp.Body); err != nil { - return errors.Wrap(err, "copying") - } - - return nil -} - -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) { - node := &Node{ - URI: uri, - } - return c.backupSliceNode(ctx, index, frame, slice, node) -} - -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - - // Return error if status is not OK. - if resp.StatusCode == http.StatusNotFound { - resp.Body.Close() - return nil, ErrFragmentNotFound - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - - return resp.Body, nil -} - -// CreateFrame creates a new frame on the server. -func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { - if index == "" { - return ErrIndexRequired - } - - // Encode query request. - buf, err := json.Marshal(&postFrameRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrFrameExists - default: - return errors.New(string(body)) - } -} - -// FragmentBlocks returns a list of block checksums for a fragment on a host. -// Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) { - u := uriPathToURL(c.defaultURI, "/fragment/blocks") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFragmentNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp getFragmentBlocksResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Blocks, nil -} - -// BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) { - buf, err := proto.Marshal(&internal.BlockDataRequest{ - Index: index, - Frame: frame, - Slice: slice, - Block: uint64(block), - }) - if err != nil { - return nil, nil, errors.Wrap(err, "marshaling") - } - - u := uriPathToURL(c.defaultURI, "/fragment/block/data") - req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/protobuf") - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Accept", "application/protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // fallthrough - case http.StatusNotFound: - return nil, nil, nil - default: - return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp internal.BlockDataResponse - if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, errors.Wrap(err, "unmarshalling") - } - return rsp.RowIDs, rsp.ColumnIDs, nil -} - -// ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) - - // Encode request. - buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postIndexAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) - - // Encode request. - buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFrameNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postFrameAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { - msg, err := MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - - u := uriPathToURL(uri, "/cluster/message") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) - if err != nil { - return errors.Wrap(err, "making new request") - } - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - - return nil -} - // Bit represents the intersection of a row and a column. It can be specifed by // integer ids or string keys. type Bit struct { @@ -884,163 +15,13 @@ type Bit struct { Timestamp int64 } -// Bits is a slice of Bit. -type Bits []Bit - -func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Bits) Len() int { return len(p) } - -func (p Bits) Less(i, j int) bool { - if p[i].RowID == p[j].RowID { - if p[i].ColumnID < p[j].ColumnID { - return p[i].Timestamp < p[j].Timestamp - } - return p[i].ColumnID < p[j].ColumnID - } - return p[i].RowID < p[j].RowID -} - -// RowIDs returns a slice of all the row IDs. -func (p Bits) RowIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].RowID - } - return other -} - -// ColumnIDs returns a slice of all the column IDs. -func (p Bits) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// RowKeys returns a slice of all the row keys. -func (p Bits) RowKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].RowKey - } - return other -} - -// ColumnKeys returns a slice of all the column keys. -func (p Bits) ColumnKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].ColumnKey - } - return other -} - -// Timestamps returns a slice of all the timestamps. -func (p Bits) Timestamps() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Timestamp - } - return other -} - -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]Bit { - m := make(map[uint64][]Bit) - for _, bit := range p { - slice := bit.ColumnID / SliceWidth - m[slice] = append(m[slice], bit) - } - - for slice, bits := range m { - sort.Sort(Bits(bits)) - m[slice] = bits - } - - return m -} - -// FieldValues represents the value for a column within a -// range-encoded frame. +// FieldValue represents the value for a column within a +// range-encoded field. type FieldValue struct { ColumnID uint64 Value int64 } -// FieldValues represents a slice of field values. -type FieldValues []FieldValue - -func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p FieldValues) Len() int { return len(p) } - -func (p FieldValues) Less(i, j int) bool { - return p[i].ColumnID < p[j].ColumnID -} - -// ColumnIDs returns a slice of all the column IDs. -func (p FieldValues) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// Values returns a slice of all the values. -func (p FieldValues) Values() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Value - } - return other -} - -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { - m := make(map[uint64][]FieldValue) - for _, val := range p { - slice := val.ColumnID / SliceWidth - m[slice] = append(m[slice], val) - } - - for slice, vals := range m { - sort.Sort(FieldValues(vals)) - m[slice] = vals - } - - return m -} - -// BitsByPos is a slice of bits sorted row then column. -type BitsByPos []Bit - -func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitsByPos) Len() int { return len(p) } -func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) - if p0 == p1 { - return p[i].Timestamp < p[j].Timestamp - } - return p0 < p1 -} - -func uriPathToURL(uri *URI, path string) url.URL { - return url.URL{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - Path: path, - } -} - -func nodePathToURL(node *Node, path string) url.URL { - return url.URL{ - Scheme: node.URI.Scheme(), - Host: node.URI.HostPort(), - Path: path, - } -} - // InternalClient should be implemented by any struct that enables any transport between nodes // TODO: Refactor // Note from Travis: Typically an interface containing more than two or three methods is an indication that @@ -1048,22 +29,105 @@ func nodePathToURL(node *Node, path string) url.URL { // While I understand that putting the entire Client behind an interface might require this many methods, // I don't want to let it go unquestioned. type InternalClient interface { - MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) + MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) - Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error - ImportK(ctx context.Context, index, frame string, bits []Bit) error + FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) + Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) + Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error + ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error - ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error - CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error - FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + EnsureField(ctx context.Context, indexName string, fieldName string) error + ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error + CreateField(ctx context.Context, index, field string) error + FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + SendMessage(ctx context.Context, uri *URI, msg []byte) error + RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) +} + +//=============== + +type InternalQueryClient interface { + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) +} + +type nopInternalQueryClient struct{} + +func (n *nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { + return nil, nil +} + +func newNopInternalQueryClient() *nopInternalQueryClient { + return &nopInternalQueryClient{} +} + +var _ InternalQueryClient = newNopInternalQueryClient() + +//=============== + +type nopInternalClient struct{} + +func newNopInternalClient() nopInternalClient { + return nopInternalClient{} +} + +var _ InternalClient = newNopInternalClient() + +func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) { + return nil, nil +} +func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil } +func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { + return nil +} +func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { + return nil, nil +} +func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) { + return nil, nil +} +func (n nopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) { + return nil, nil +} +func (n nopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { + return nil +} +func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { + return nil +} +func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { + return nil +} +func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { + return nil +} +func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { + return nil +} +func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { + return nil +} +func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) { + return nil, nil +} +func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { + return nil, nil, nil +} +func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { + return nil +} +func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { + return nil, nil } diff --git a/client_test.go b/client_test.go deleted file mode 100644 index 6e5f2dc41..000000000 --- a/client_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "context" - "fmt" - "net/http" - "reflect" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" - "github.com/pilosa/pilosa/test" -) - -func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { - numNodes := len(c.Nodes) - hldr := make([]*test.Holder, numNodes) - server := make([]*test.Server, numNodes) - for i := 0; i < numNodes; i++ { - hldr[i] = test.MustOpenHolder() - server[i] = test.NewServer() - server[i].Handler.API.Cluster = c - server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.API.Holder = hldr[i].Holder - } - return server, hldr -} - -var defaultClient *http.Client - -func init() { - defaultClient = server.GetHTTPClient(nil) - -} - -// Test distributed TopN Row count across 3 nodes. -func TestClient_MultiNode(t *testing.T) { - cluster := test.NewCluster(3) - s, hldr := createCluster(cluster) - - for i := 0; i < len(cluster.Nodes); i++ { - defer hldr[i].Close() - defer s[i].Close() - } - - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) - e.Holder = hldr[0].Holder - e.Node = cluster.Nodes[0] - e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) - } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) - e.Holder = hldr[1].Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) - } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) - e.Holder = hldr[2].Holder - e.Node = cluster.Nodes[2] - e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) - } - - // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - sliceNums := []uint64{1, 2, 6} - for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) - ownsNum := false - for _, ownNum := range owns { - if ownNum == num { - ownsNum = true - break - } - } - if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) - } - } - - baseBit0 := SliceWidth * sliceNums[0] - baseBit1 := SliceWidth * sliceNums[1] - baseBit2 := SliceWidth * sliceNums[2] - - maxSlice := uint64(0) - for _, x := range sliceNums { - if x > maxSlice { - maxSlice = x - } - } - - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) - - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) - - // Rebuild the RankCache. - // We have to do this to avoid the 10-second cache invalidation delay - // built into cache.Invalidate() - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() - - // Connect to each node to compare results. - client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host(), defaultClient) - client[1] = test.MustNewClient(s[1].Host(), defaultClient) - client[2] = test.MustNewClient(s[2].Host(), defaultClient) - - topN := 4 - queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), - Remote: false, - } - result, err := client[0].Query(context.Background(), "i", queryRequest) - if err != nil { - t.Fatal(err) - } - - // Check the results before every node has the correct max slice value. - pairs := result.Results[0].Pairs - for _, pair := range pairs { - if pair.ID == 22 && pair.Count != 3 { - t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) - } - } - - // Set max slice to correct value. - hldr[0].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) - - result, err = client[0].Query(context.Background(), "i", queryRequest) - if err != nil { - t.Fatal(err) - } - - // Test must return exactly N results. - if len(result.Results[0].Pairs) != topN { - t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) - } - p := []*internal.Pair{ - {ID: 100, Count: 12}, - {ID: 22, Count: 11}, - {ID: 98, Count: 8}, - {ID: 99, Count: 7}} - - // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].Pairs, p) { - t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) - } - - result1, err := client[1].Query(context.Background(), "i", queryRequest) - if err != nil { - t.Fatal(err) - } - result2, err := client[2].Query(context.Background(), "i", queryRequest) - if err != nil { - t.Fatal(err) - } - - // Compare TopN results across all nodes in the cluster. - if !reflect.DeepEqual(result, result1) { - t.Fatalf("TopN result should be the same on node0 and node1: %s", spew.Sdump(result1)) - } - - if !reflect.DeepEqual(result, result2) { - t.Fatalf("TopN result should be the same on node0 and node2: %s", spew.Sdump(result2)) - } -} - -// Ensure client can bulk import data. -func TestClient_Import(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Load bitmap into cache to ensure cache gets updated. - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f.Row(0) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) - if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ - {RowID: 0, ColumnID: 1}, - {RowID: 0, ColumnID: 5}, - {RowID: 200, ColumnID: 6}, - }); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { - t.Fatalf("unexpected columns: %+v", a) - } - if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { - t.Fatalf("unexpected columns: %+v", a) - } -} - -// Ensure client can bulk import value data. -func TestClient_ImportValue(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - fld := pilosa.Field{ - Name: "fld", - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 100, - } - - // Load bitmap into cache to ensure cache gets updated. - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) - if err != nil { - t.Fatal(err) - } - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) - if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ - {ColumnID: 1, Value: -10}, - {ColumnID: 2, Value: 20}, - {ColumnID: 3, Value: 40}, - }); err != nil { - t.Fatal(err) - } - - // Verify Sum. - sum, cnt, err := frame.FieldSum(nil, fld.Name) - if err != nil { - t.Fatal(err) - } - if sum != 50 || cnt != 3 { - t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) - } - - // Verify Min. - min, cnt, err := frame.FieldMin(nil, fld.Name) - if err != nil { - t.Fatal(err) - } - if min != -10 || cnt != 1 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) - } - - // Verify Min with Filter. - filter, err := frame.FieldRange(fld.Name, pql.GT, 40) - if err != nil { - t.Fatal(err) - } - min, cnt, err = frame.FieldMin(filter, fld.Name) - if err != nil { - t.Fatal(err) - } - if min != -100 || cnt != 0 { - t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) - } - - // Verify Max. - max, cnt, err := frame.FieldMax(nil, fld.Name) - if err != nil { - t.Fatal(err) - } - if max != 40 || cnt != 1 { - t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) - } -} - -// Ensure client can retrieve a list of all checksums for blocks in a fragment. -func TestClient_FragmentBlocks(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Set two bits on blocks 0 & 3. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) - - // Set a bit on a different slice. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Retrieve blocks. - c := test.MustNewClient(s.Host(), defaultClient) - blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0) - if err != nil { - t.Fatal(err) - } else if len(blocks) != 2 { - t.Fatalf("unexpected blocks: %s", spew.Sdump(blocks)) - } else if blocks[0].ID != 0 { - t.Fatalf("unexpected block id(0): %d", blocks[0].ID) - } else if blocks[1].ID != 3 { - t.Fatalf("unexpected block id(1): %d", blocks[1].ID) - } - - // Verify data matches local blocks. - if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { - t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) - } -} diff --git a/cluster.go b/cluster.go index 63f4f62b8..a36b2c92e 100644 --- a/cluster.go +++ b/cluster.go @@ -21,7 +21,6 @@ import ( "hash/fnv" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "sort" @@ -37,8 +36,8 @@ import ( ) const ( - // DefaultPartitionN is the default number of partitions in a cluster. - DefaultPartitionN = 256 + // defaultPartitionN is the default number of partitions in a cluster. + defaultPartitionN = 256 // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" @@ -46,17 +45,16 @@ const ( ClusterStateResizing = "RESIZING" // NodeState represents the state of a node during startup. - NodeStateLoading = "LOADING" - NodeStateReady = "READY" + nodeStateReady = "READY" - // ResizeJob states. - ResizeJobStateRunning = "RUNNING" + // resizeJob states. + resizeJobStateRunning = "RUNNING" // Final states. - ResizeJobStateDone = "DONE" - ResizeJobStateAborted = "ABORTED" + resizeJobStateDone = "DONE" + resizeJobStateAborted = "ABORTED" - ResizeJobActionAdd = "ADD" - ResizeJobActionRemove = "REMOVE" + resizeJobActionAdd = "ADD" + resizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. @@ -70,52 +68,6 @@ func (n Node) String() string { return fmt.Sprintf("Node: %s", n.ID) } -// EncodeNodes converts a slice of Nodes into its internal representation. -func EncodeNodes(a []*Node) []*internal.Node { - other := make([]*internal.Node, len(a)) - for i := range a { - other[i] = EncodeNode(a[i]) - } - return other -} - -// EncodeNode converts a Node into its internal representation. -func EncodeNode(n *Node) *internal.Node { - return &internal.Node{ - ID: n.ID, - URI: n.URI.Encode(), - IsCoordinator: n.IsCoordinator, - } -} - -// DecodeNodes converts a proto message into a slice of Nodes. -func DecodeNodes(a []*internal.Node) []*Node { - if len(a) == 0 { - return nil - } - other := make([]*Node, len(a)) - for i := range a { - other[i] = DecodeNode(a[i]) - } - return other -} - -// DecodeNode converts a proto message into a Node. -func DecodeNode(node *internal.Node) *Node { - return &Node{ - ID: node.ID, - URI: decodeURI(node.URI), - IsCoordinator: node.IsCoordinator, - } -} - -func DecodeNodeEvent(ne *internal.NodeEventMessage) *NodeEvent { - return &NodeEvent{ - Event: NodeEventType(ne.Event), - Node: DecodeNode(ne.Node), - } -} - // Nodes represents a list of nodes. type Nodes []*Node @@ -211,30 +163,26 @@ type nodeAction struct { action string } -// Cluster represents a collection of nodes. -type Cluster struct { - ID string - Node *Node - Nodes []*Node // TODO phase this out? - MemberSet MemberSet +// cluster represents a collection of nodes. +type cluster struct { + id string + Node *Node + Nodes []*Node // TODO phase this out? // Hashing algorithm used to assign partitions to nodes. Hasher Hasher // The number of partitions in the cluster. - PartitionN int + partitionN int // The number of replicas a partition has. ReplicaN int // Threshold for logging long-running queries - LongQueryTime time.Duration + longQueryTime time.Duration - // Maximum number of SetBit() or ClearBit() commands per request. - MaxWritesPerRequest int - - // EventReceiver receives NodeEvents pertaining to node membership. - EventReceiver EventReceiver + // Maximum number of Set() or Clear() commands per request. + maxWritesPerRequest int // Data directory path. Path string @@ -244,8 +192,8 @@ type Cluster struct { Static bool // Static is primarily used for testing in a non-gossip environment. state string Coordinator string - Holder *Holder - Broadcaster Broadcaster + holder *Holder + broadcaster broadcaster joiningLeavingNodes chan nodeAction @@ -255,57 +203,57 @@ type Cluster struct { joined bool mu sync.RWMutex - jobs map[int64]*ResizeJob - currentJob *ResizeJob + jobs map[int64]*resizeJob + currentJob *resizeJob // Close management wg sync.WaitGroup closing chan struct{} - Logger Logger + logger Logger - // - RemoteClient *http.Client + InternalClient InternalClient } -// NewCluster returns a new instance of Cluster with defaults. -func NewCluster() *Cluster { - return &Cluster{ - Hasher: &jmphasher{}, - PartitionN: DefaultPartitionN, - ReplicaN: 1, - EventReceiver: NopEventReceiver, +// newCluster returns a new instance of Cluster with defaults. +func newCluster() *cluster { + return &cluster{ + Hasher: &jmphasher{}, + partitionN: defaultPartitionN, + ReplicaN: 1, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), + jobs: make(map[int64]*resizeJob), closing: make(chan struct{}), joining: make(chan struct{}), - Logger: NopLogger, + InternalClient: newNopInternalClient(), + + logger: NopLogger, } } -// Coordinator returns the coordinator node. -func (c *Cluster) CoordinatorNode() *Node { - return c.nodeByID(c.Coordinator) +// coordinatorNode returns the coordinator node. +func (c *cluster) coordinatorNode() *Node { + return c.unprotectedNodeByID(c.Coordinator) } -// IsCoordinator is true if this node is the coordinator. -func (c *Cluster) IsCoordinator() bool { +// isCoordinator is true if this node is the coordinator. +func (c *cluster) isCoordinator() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.isCoordinator() + return c.unprotectedIsCoordinator() } -func (c *Cluster) isCoordinator() bool { +func (c *cluster) unprotectedIsCoordinator() bool { return c.Coordinator == c.Node.ID } -// SetCoordinator tells the current node to become the +// setCoordinator tells the current node to become the // Coordinator. In response to this, the current node // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. -func (c *Cluster) SetCoordinator(n *Node) error { +func (c *cluster) setCoordinator(n *Node) error { c.mu.Lock() // Verify that the new Coordinator value matches // this node. @@ -315,32 +263,32 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.updateCoordinator(n) + _ = c.unprotectedUpdateCoordinator(n) c.mu.Unlock() // Send the update coordinator message to all nodes. - err := c.Broadcaster.SendSync( - &internal.UpdateCoordinatorMessage{ - New: EncodeNode(n), + err := c.broadcaster.SendSync( + &UpdateCoordinatorMessage{ + New: n, }) if err != nil { return fmt.Errorf("problem sending UpdateCoordinator message: %v", err) } // Broadcast cluster status. - return c.Broadcaster.SendSync(c.Status()) + return c.broadcaster.SendSync(c.Status()) } -// UpdateCoordinator updates this nodes Coordinator value as well as +// updateCoordinator updates this nodes Coordinator value as well as // changing the corresponding node's IsCoordinator value // to true, and sets all other nodes to false. Returns true if the value // changed. -func (c *Cluster) UpdateCoordinator(n *Node) bool { +func (c *cluster) updateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - return c.updateCoordinator(n) + return c.unprotectedUpdateCoordinator(n) } -func (c *Cluster) updateCoordinator(n *Node) bool { +func (c *cluster) unprotectedUpdateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID @@ -356,10 +304,10 @@ func (c *Cluster) updateCoordinator(n *Node) bool { return changed } -// AddNode adds a node to the Cluster and updates and saves the +// addNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddNode(node *Node) error { - c.Logger.Printf("add node %s to cluster on %s", node, c.Node) +func (c *cluster) addNode(node *Node) error { + c.logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. if node.IsCoordinator { @@ -375,7 +323,7 @@ func (c *Cluster) AddNode(node *Node) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.AddID(node.ID) { + if !c.Topology.addID(node.ID) { return nil } @@ -383,9 +331,9 @@ func (c *Cluster) AddNode(node *Node) error { return c.saveTopology() } -// RemoveNode removes a node from the Cluster and updates and saves the +// removeNode removes a node from the Cluster and updates and saves the // new topology. -func (c *Cluster) RemoveNode(node *Node) error { +func (c *cluster) removeNode(node *Node) error { // remove from cluster if !c.removeNodeBasicSorted(node) { return nil @@ -395,7 +343,7 @@ func (c *Cluster) RemoveNode(node *Node) error { if c.Topology == nil { return fmt.Errorf("Cluster.Topology is nil") } - if !c.Topology.RemoveID(node.ID) { + if !c.Topology.removeID(node.ID) { return nil } @@ -403,41 +351,41 @@ func (c *Cluster) RemoveNode(node *Node) error { return c.saveTopology() } -// NodeIDs returns the list of IDs in the cluster. -func (c *Cluster) NodeIDs() []string { +// nodeIDs returns the list of IDs in the cluster. +func (c *cluster) nodeIDs() []string { return Nodes(c.Nodes).IDs() } -func (c *Cluster) setID(id string) { +func (c *cluster) setID(id string) { // Don't overwrite ClusterID. - if c.ID != "" { + if c.id != "" { return } - c.ID = id + c.id = id // Make sure the Topology is updated. - c.Topology.ClusterID = c.ID + c.Topology.clusterID = c.id } -func (c *Cluster) State() string { +func (c *cluster) State() string { c.mu.RLock() defer c.mu.RUnlock() return c.state } -func (c *Cluster) SetState(state string) { +func (c *cluster) SetState(state string) { c.mu.Lock() c.setState(state) c.mu.Unlock() } -func (c *Cluster) setState(state string) { +func (c *cluster) setState(state string) { // Ignore cases where the state hasn't changed. if state == c.state { return } - c.Logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) + c.logger.Printf("change cluster state from %s to %s on %s", c.state, state, c.Node.ID) var doCleanup bool @@ -455,43 +403,43 @@ func (c *Cluster) setState(state string) { // been removed. // It's safe to do a cleanup after state changes back to normal. if doCleanup { - var cleaner HolderCleaner + var cleaner holderCleaner cleaner.Node = c.Node - cleaner.Holder = c.Holder + cleaner.Holder = c.holder cleaner.Cluster = c cleaner.Closing = c.closing // Clean holder. if err := cleaner.CleanHolder(); err != nil { - c.Logger.Printf("holder clean error: err=%s", err) + c.logger.Printf("holder clean error: err=%s", err) } } } -func (c *Cluster) SetNodeState(state string) error { - if c.IsCoordinator() { - return c.ReceiveNodeState(c.Node.ID, state) +func (c *cluster) setNodeState(state string) error { + if c.isCoordinator() { + return c.receiveNodeState(c.Node.ID, state) } // Send node state to coordinator. - ns := &internal.NodeStateMessage{ + ns := &NodeStateMessage{ NodeID: c.Node.ID, State: state, } - c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) - if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { + c.logger.Printf("Sending State %s (%s)", state, c.Coordinator) + if err := c.sendTo(c.coordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } return nil } -// ReceiveNodeState sets node state in Topology in order for the +// receiveNodeState sets node state in Topology in order for the // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. -func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { - if !c.IsCoordinator() { +func (c *cluster) receiveNodeState(nodeID string, state string) error { + if !c.isCoordinator() { return nil } @@ -501,7 +449,7 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { } c.Topology.nodeStates[nodeID] = state - c.Logger.Printf("received state %s (%s)", state, nodeID) + c.logger.Printf("received state %s (%s)", state, nodeID) // Set cluster state to NORMAL. if c.haveTopologyAgreement() && c.allNodesReady() { @@ -511,28 +459,23 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { return nil } -// localNode is not being used. -//func (c *Cluster) localNode() *Node { -// return c.NodeByURI(c.URI) -//} - -// Status returns the internal ClusterStatus representation. -func (c *Cluster) Status() *internal.ClusterStatus { - return &internal.ClusterStatus{ - ClusterID: c.ID, +// Status returns the the cluster's status including what nodes it contains, its ID, and current state. +func (c *cluster) Status() *ClusterStatus { + return &ClusterStatus{ + ClusterID: c.id, State: c.state, - Nodes: EncodeNodes(c.Nodes), + Nodes: c.Nodes, } } -func (c *Cluster) NodeByID(id string) *Node { +func (c *cluster) nodeByID(id string) *Node { c.mu.RLock() defer c.mu.RUnlock() - return c.nodeByID(id) + return c.unprotectedNodeByID(id) } -// nodeByID returns a node reference by ID. -func (c *Cluster) nodeByID(id string) *Node { +// unprotectedNodeByID returns a node reference by ID. +func (c *cluster) unprotectedNodeByID(id string) *Node { for _, n := range c.Nodes { if n.ID == id { return n @@ -542,7 +485,7 @@ func (c *Cluster) nodeByID(id string) *Node { } // nodePositionByID returns the position of the node in slice c.Nodes. -func (c *Cluster) nodePositionByID(nodeID string) int { +func (c *cluster) nodePositionByID(nodeID string) int { for i, n := range c.Nodes { if n.ID == nodeID { return i @@ -553,8 +496,8 @@ func (c *Cluster) nodePositionByID(nodeID string) int { // addNodeBasicSorted adds a node to the cluster, sorted by id. // Returns a pointer to the node and true if the node was added. -func (c *Cluster) addNodeBasicSorted(node *Node) bool { - n := c.nodeByID(node.ID) +func (c *cluster) addNodeBasicSorted(node *Node) bool { + n := c.unprotectedNodeByID(node.ID) if n != nil { return false } @@ -569,7 +512,7 @@ func (c *Cluster) addNodeBasicSorted(node *Node) bool { // removeNodeBasicSorted removes a node from the cluster, maintaining // the sort order. Returns true if the node was removed. -func (c *Cluster) removeNodeBasicSorted(node *Node) bool { +func (c *cluster) removeNodeBasicSorted(node *Node) bool { i := c.nodePositionByID(node.ID) if i < 0 { return false @@ -584,9 +527,9 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool { // frag is a struct of basic fragment information. type frag struct { - frame string + field string view string - slice uint64 + shard uint64 } func fragsDiff(a, b []frag) []frag { @@ -617,36 +560,36 @@ func (a fragsByHost) add(b fragsByHost) fragsByHost { return a } -type viewsByFrame map[string][]string +type viewsByField map[string][]string -func (a viewsByFrame) addView(frame, view string) { - a[frame] = append(a[frame], view) +func (a viewsByField) addView(field, view string) { + a[field] = append(a[field], view) } -func (c *Cluster) fragsByHost(idx *Index) fragsByHost { - // frameViews is a map of frame to slice of views. - frameViews := make(viewsByFrame) +func (c *cluster) fragsByHost(idx *Index) fragsByHost { + // fieldViews is a map of field to slice of views. + fieldViews := make(viewsByField) - for _, frame := range idx.Frames() { - for _, view := range frame.Views() { - frameViews.addView(frame.Name(), view.Name()) + for _, field := range idx.Fields() { + for _, view := range field.views() { + fieldViews.addView(field.Name(), view.name) } } - return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) + return c.fragCombos(idx.Name(), idx.maxShard(), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index -// by creating every combination of frame/view specified in `frameViews` up to maxSlice. -func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { +// by creating every combination of field/view specified in `fieldViews` up to maxShard. +func (c *cluster) fragCombos(idx string, maxShard uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) - for i := uint64(0); i <= maxSlice; i++ { - nodes := c.SliceNodes(idx, i) + for i := uint64(0); i <= maxShard; i++ { + nodes := c.shardNodes(idx, i) for _, n := range nodes { - // for each frame/view combination: - for frame, views := range frameViews { + // for each field/view combination: + for field, views := range fieldViews { for _, view := range views { - t[n.ID] = append(t[n.ID], frag{frame, view, i}) + t[n.ID] = append(t[n.ID], frag{field, view, i}) } } } @@ -657,7 +600,7 @@ func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFram // diff compares c with another cluster and determines if a node is being // added or removed. An error is returned for any case other than where // exactly one node is added or removed. -func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) { +func (c *cluster) diff(other *cluster) (action string, nodeID string, err error) { lenFrom := len(c.Nodes) lenTo := len(other.Nodes) // Determine if a node is being added or removed. @@ -669,10 +612,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenTo-lenFrom > 1 { return "", "", errors.New("adding more than one node at a time is not supported") } - action = ResizeJobActionAdd + action = resizeJobActionAdd // Determine the node ID that is being added. for _, n := range other.Nodes { - if c.nodeByID(n.ID) == nil { + if c.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -682,10 +625,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenFrom-lenTo > 1 { return "", "", errors.New("removing more than one node at a time is not supported") } - action = ResizeJobActionRemove + action = resizeJobActionRemove // Determine the node ID that is being removed. for _, n := range c.Nodes { - if other.nodeByID(n.ID) == nil { + if other.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -696,8 +639,8 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) // fragSources returns a list of ResizeSources - for each node in the `to` cluster - // required to move from cluster `c` to cluster `to`. -func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.ResizeSource, error) { - m := make(map[string][]*internal.ResizeSource) +func (c *cluster) fragSources(to *cluster, idx *Index) (map[string][]*ResizeSource, error) { + m := make(map[string][]*ResizeSource) // Determine if a node is being added or removed. action, diffNodeID, err := c.diff(to) @@ -717,11 +660,11 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // If a node is being removed, however, then it will most likely // require that a replica fragment be the source data. srcCluster := c - if action == ResizeJobActionAdd && c.ReplicaN > 1 { - srcCluster = NewCluster() + if action == resizeJobActionAdd && c.ReplicaN > 1 { + srcCluster = newCluster() srcCluster.Nodes = Nodes(c.Nodes).Clone() srcCluster.Hasher = c.Hasher - srcCluster.PartitionN = c.PartitionN + srcCluster.partitionN = c.partitionN srcCluster.ReplicaN = 1 } @@ -736,7 +679,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R srcNodesByFrag := make(map[frag]string) for nodeID, frags := range srcFrags { // If a node is being removed, don't consider it as a source. - if action == ResizeJobActionRemove && nodeID == diffNodeID { + if action == resizeJobActionRemove && nodeID == diffNodeID { continue } for _, frag := range frags { @@ -756,7 +699,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // Get the ResizeSource for each diff. for nodeID, diff := range diffs { - m[nodeID] = []*internal.ResizeSource{} + m[nodeID] = []*ResizeSource{} for _, frag := range diff { // If there is no valid source node ID for a fragment, // it likely means that the replica factor was not @@ -767,12 +710,12 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return nil, errors.New("not enough data to perform resize (replica factor may need to be increased)") } - src := &internal.ResizeSource{ - Node: EncodeNode(c.nodeByID(srcNodeID)), + src := &ResizeSource{ + Node: c.unprotectedNodeByID(srcNodeID), Index: idx.Name(), - Frame: frag.frame, + Field: frag.field, View: frag.view, - Slice: frag.slice, + Shard: frag.shard, } m[nodeID] = append(m[nodeID], src) @@ -782,30 +725,30 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// Partition returns the partition that a slice belongs to. -func (c *Cluster) Partition(index string, slice uint64) int { +// partition returns the partition that a shard belongs to. +func (c *cluster) partition(index string, shard uint64) int { var buf [8]byte - binary.BigEndian.PutUint64(buf[:], slice) + binary.BigEndian.PutUint64(buf[:], shard) // Hash the bytes and mod by partition count. h := fnv.New64a() h.Write([]byte(index)) h.Write(buf[:]) - return int(h.Sum64() % uint64(c.PartitionN)) + return int(h.Sum64() % uint64(c.partitionN)) } -// SliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { - return c.PartitionNodes(c.Partition(index, slice)) +// shardNodes returns a list of nodes that own a fragment. +func (c *cluster) shardNodes(index string, shard uint64) []*Node { + return c.partitionNodes(c.partition(index, shard)) } -// OwnsSlice returns true if a host owns a fragment. -func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) +// ownsShard returns true if a host owns a fragment. +func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool { + return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) } -// PartitionNodes returns a list of nodes that own a partition. -func (c *Cluster) PartitionNodes(partitionID int) []*Node { +// partitionNodes returns a list of nodes that own a partition. +func (c *cluster) partitionNodes(partitionID int) []*Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. replicaN := c.ReplicaN @@ -827,34 +770,20 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices finds the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) - // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].URI == uri { - slices = append(slices, i) - } - } - return slices -} - -// ContainsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) +// containsShards is like OwnsShards, but it includes replicas. +func (c *cluster) containsShards(index string, maxShard uint64, node *Node) []uint64 { + var shards []uint64 + for i := uint64(0); i <= maxShard; i++ { + p := c.partition(index, i) // Determine the nodes for partition. - nodes := c.PartitionNodes(p) + nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { - slices = append(slices, i) + shards = append(shards, i) } } } - return slices + return shards } // Hasher represents an interface to hash integers into buckets. @@ -863,8 +792,8 @@ type Hasher interface { Hash(key uint64, n int) int } -// NewHasher returns a new instance of the default hasher. -func NewHasher() Hasher { return &jmphasher{} } +// newHasher returns a new instance of the default hasher. +func newHasher() Hasher { return &jmphasher{} } // jmphasher represents an implementation of jmphash. Implements Hasher. type jmphasher struct{} @@ -880,7 +809,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { return int(b) } -func (c *Cluster) Open() error { +func (c *cluster) setup() error { // Cluster always comes up in state STARTING until cluster membership is determined. c.state = ClusterStateStarting @@ -889,34 +818,35 @@ func (c *Cluster) Open() error { return errors.Wrap(err, "loading topology") } - c.ID = c.Topology.ClusterID + c.id = c.Topology.clusterID // Only the coordinator needs to consider the .topology file. - if c.IsCoordinator() { + if c.isCoordinator() { err := c.considerTopology() if err != nil { - return fmt.Errorf("considerTopology: %v", err) + return errors.Wrap(err, "considerTopology") } } // Add the local node to the cluster. - err := c.AddNode(c.Node) + err := c.addNode(c.Node) if err != nil { return errors.Wrap(err, "adding local node") } + return nil +} - // Start the EventReceiver. - if err := c.EventReceiver.Start(c); err != nil { - return fmt.Errorf("starting EventReceiver: %v", err) - } - - // Open MemberSet communication. - if err := c.MemberSet.Open(c.Node); err != nil { - return fmt.Errorf("opening MemberSet: %v", err) +func (c *cluster) open() error { + err := c.setup() + if err != nil { + return errors.Wrap(err, "setting up cluster") } + return c.waitForStarted() +} +func (c *cluster) waitForStarted() error { // If not coordinator then wait for ClusterStatus from coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { // In the case where a node has been restarted and memberlist has // not had enough time to determine the node went down/up, then // the coorninator needs to be alerted that this node is back up @@ -925,23 +855,23 @@ func (c *Cluster) Open() error { // TODO: Because the normal code path already sends a NodeJoin event (via // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. - msg := &internal.NodeEventMessage{ - Event: uint32(NodeJoin), - Node: EncodeNode(c.Node), + msg := &NodeEvent{ + Event: NodeJoin, + Node: c.Node, } - if err := c.Broadcaster.SendAsync(msg); err != nil { + if err := c.broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) } - c.Logger.Printf("wait for joining to complete") + c.logger.Printf("%v wait for joining to complete", c.Node.ID) <-c.joining - c.Logger.Printf("joining has completed") + c.logger.Printf("joining has completed") } return nil } -func (c *Cluster) Close() error { +func (c *cluster) close() error { // Notify goroutines of closing and wait for completion. close(c.closing) c.wg.Wait() @@ -949,43 +879,43 @@ func (c *Cluster) Close() error { return nil } -func (c *Cluster) markAsJoined() { - c.Logger.Printf("mark node as joined (received coordinator update)") +func (c *cluster) markAsJoined() { + c.logger.Printf("mark node as joined (received coordinator update)") if !c.joined { c.joined = true close(c.joining) } } -func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) +func (c *cluster) needTopologyAgreement() bool { + return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } -func (c *Cluster) haveTopologyAgreement() bool { +func (c *cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } -func (c *Cluster) allNodesReady() bool { +func (c *cluster) allNodesReady() bool { if c.Static { return true } - for _, uri := range c.Topology.NodeIDs { - if c.Topology.nodeStates[uri] != NodeStateReady { + for _, uri := range c.Topology.nodeIDs { + if c.Topology.nodeStates[uri] != nodeStateReady { return false } } return true } -func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { +func (c *cluster) handleNodeAction(nodeAction nodeAction) error { j, err := c.generateResizeJob(nodeAction) if err != nil { - c.Logger.Printf("generateResizeJob error: err=%s", err) + c.logger.Printf("generateResizeJob error: err=%s", err) if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + c.logger.Printf("setStateAndBroadcast error: err=%s", err) } return errors.Wrap(err, "setting state") } @@ -995,11 +925,11 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { // channel, which is not consumed until the code below. var eg errgroup.Group eg.Go(func() error { - return j.Run() + return j.run() }) - // Wait for the ResizeJob to finish or be aborted. - c.Logger.Printf("wait for jobResult") + // Wait for the resizeJob to finish or be aborted. + c.logger.Printf("wait for jobResult") jobResult := <-j.result // Make sure j.Run() didn't return an error. @@ -1007,99 +937,101 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { return errors.Wrap(err, "running job") } - c.Logger.Printf("received jobResult: %s", jobResult) + c.logger.Printf("received jobResult: %s", jobResult) switch jobResult { - case ResizeJobStateDone: - if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + case resizeJobStateDone: + if err := c.completeCurrentJob(resizeJobStateDone); err != nil { return errors.Wrap(err, "completing finished job") } // Add/remove uri to/from the cluster. - if j.action == ResizeJobActionRemove { - return c.RemoveNode(nodeAction.node) - } else if j.action == ResizeJobActionAdd { - return c.AddNode(nodeAction.node) + if j.action == resizeJobActionRemove { + return c.removeNode(nodeAction.node) + } else if j.action == resizeJobActionAdd { + return c.addNode(nodeAction.node) } - case ResizeJobStateAborted: - if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + case resizeJobStateAborted: + if err := c.completeCurrentJob(resizeJobStateAborted); err != nil { return errors.Wrap(err, "completing aborted job") } } return nil } -func (c *Cluster) setStateAndBroadcast(state string) error { +func (c *cluster) setStateAndBroadcast(state string) error { c.SetState(state) + if c.Static { + return nil + } // Broadcast cluster status changes to the cluster. - c.Logger.Printf("broadcasting ClusterStatus: %s", state) - return c.Broadcaster.SendSync(c.Status()) + c.logger.Printf("broadcasting ClusterStatus: %s", state) + return c.broadcaster.SendSync(c.Status()) } -func (c *Cluster) sendTo(node *Node, msg proto.Message) error { - if err := c.Broadcaster.SendTo(node, msg); err != nil { +func (c *cluster) sendTo(node *Node, m Message) error { + if err := c.broadcaster.SendTo(node, m); err != nil { return errors.Wrap(err, "sending") } return nil } -// ListenForJoins handles cluster-resize events. -func (c *Cluster) ListenForJoins() { +// listenForJoins handles cluster-resize events. +func (c *cluster) listenForJoins() { c.wg.Add(1) - go func() { defer c.wg.Done(); c.listenForJoins() }() -} + go func() { + defer c.wg.Done() -func (c *Cluster) listenForJoins() { - // When a cluster starts, the state is STARTING. - // We first want to wait for at least one node to join. - // Then we want to clear out the joiningLeavingNodes queue (buffered channel). - // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. - // We use a bool `setNormal` to indicate when at least one node has joined. + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + var setNormal bool - var setNormal bool + for { - for { + // Handle all pending joins before changing state back to NORMAL. + select { + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + default: + } - // Handle all pending joins before changing state back to NORMAL. - select { - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) + // Only change state to NORMAL if we have successfully added at least one host. + if setNormal { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.logger.Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true continue } - setNormal = true - continue - default: } - - // Only change state to NORMAL if we have successfully added at least one host. - if setNormal { - // Put the cluster back to state NORMAL and broadcast. - if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.Logger.Printf("setStateAndBroadcast error: err=%s", err) - } - } - - // Wait for a joining host or a close. - select { - case <-c.closing: - return - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) - continue - } - setNormal = true - continue - } - } + }() } -// generateResizeJob creates a new ResizeJob based on the new node being -// added/removed. It also saves a reference to the ResizeJob in the `jobs` map +// generateResizeJob creates a new resizeJob based on the new node being +// added/removed. It also saves a reference to the resizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { - c.Logger.Printf("generateResizeJob: %v", nodeAction) +func (c *cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) { + c.logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1107,7 +1039,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, errors.Wrap(err, "generating job") } - c.Logger.Printf("generated ResizeJob: %d", j.ID) + c.logger.Printf("generated resizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1121,35 +1053,35 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { return j, nil } -// generateResizeJobByAction returns a ResizeJob with instructions based on +// generateResizeJobByAction returns a resizeJob with instructions based on // the difference between Cluster and a new Cluster with/without uri. -// Broadcaster is associated to the ResizeJob here for use in broadcasting +// Broadcaster is associated to the resizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) - j.Broadcaster = c.Broadcaster +func (c *cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) { + j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action) + j.Broadcaster = c.broadcaster // toCluster is a clone of Cluster with the new node added/removed for comparison. - toCluster := NewCluster() + toCluster := newCluster() toCluster.Nodes = Nodes(c.Nodes).Clone() toCluster.Hasher = c.Hasher - toCluster.PartitionN = c.PartitionN + toCluster.partitionN = c.partitionN toCluster.ReplicaN = c.ReplicaN - if nodeAction.action == ResizeJobActionRemove { + if nodeAction.action == resizeJobActionRemove { toCluster.removeNodeBasicSorted(nodeAction.node) - } else if nodeAction.action == ResizeJobActionAdd { + } else if nodeAction.action == resizeJobActionAdd { toCluster.addNodeBasicSorted(nodeAction.node) } // multiIndex is a map of sources initialized with all the nodes in toCluster. - multiIndex := make(map[string][]*internal.ResizeSource) + multiIndex := make(map[string][]*ResizeSource) for _, n := range toCluster.Nodes { multiIndex[n.ID] = nil } // Add to multiIndex the instructions for each index. - for _, idx := range c.Holder.Indexes() { + for _, idx := range c.holder.Indexes() { fragSources, err := c.fragSources(toCluster, idx) if err != nil { return nil, errors.Wrap(err, "getting sources") @@ -1166,12 +1098,12 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, j.IDs[id] = true continue } - instr := &internal.ResizeInstruction{ + instr := &ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.nodeByID(id)), - Coordinator: EncodeNode(c.CoordinatorNode()), + Node: toCluster.unprotectedNodeByID(id), + Coordinator: c.coordinatorNode(), Sources: sources, - Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. + Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), } j.Instructions = append(j.Instructions, instr) @@ -1180,42 +1112,42 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, return j, nil } -// CompleteCurrentJob sets the state of the current ResizeJob +// completeCurrentJob sets the state of the current resizeJob // then removes the pointer to currentJob. -func (c *Cluster) CompleteCurrentJob(state string) error { +func (c *cluster) completeCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() - if !c.isCoordinator() { + if !c.unprotectedIsCoordinator() { return ErrNodeNotCoordinator } if c.currentJob == nil { return ErrResizeNotRunning } - c.currentJob.SetState(state) + c.currentJob.setState(state) c.currentJob = nil return nil } -// FollowResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - c.Logger.Printf("follow resize instruction on %s", c.Node.ID) +// followResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { + c.logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. - if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil { return errors.Wrap(err, "merging cluster status") } - c.Logger.Printf("MergeClusterStatus done, start goroutine") + c.logger.Printf("MergeClusterStatus done, start goroutine") // The actual resizing runs in a goroutine because we don't want to block // the distribution of other ResizeInstructions to the rest of the cluster. go func() { // Make sure the holder has opened. - <-c.Holder.opened + <-c.holder.opened // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -1225,63 +1157,60 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err if err := func() error { // Sync the schema received in the resize instruction. - c.Logger.Printf("Holder ApplySchema") - if err := c.Holder.ApplySchema(instr.Schema); err != nil { + c.logger.Printf("Holder ApplySchema") + if err := c.holder.applySchema(instr.Schema); err != nil { return errors.Wrap(err, "applying schema") } - // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions - // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - srcURI := decodeURI(src.Node.URI) + srcURI := src.Node.URI - // Retrieve frame. - f := c.Holder.Frame(src.Index, src.Frame) + // Retrieve field. + f := c.holder.Field(src.Index, src.Field) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Create view. - v, err := f.CreateViewIfNotExists(src.View) + v, err := f.createViewIfNotExists(src.View) if err != nil { return errors.Wrap(err, "creating view") } // Create the local fragment. - frag, err := v.CreateFragmentIfNotExists(src.Slice) + frag, err := v.CreateFragmentIfNotExists(src.Shard) if err != nil { return errors.Wrap(err, "creating fragment") } - // Stream slice from remote node. - c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI) + // Stream shard from remote node. + c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) + rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found - // on the remote node. This occurs when a slice has been skipped and + // on the remote node. This occurs when a shard has been skipped and // therefore doesn't contain data. The coordinator correctly determined - // the resize instruction to retrieve the slice, but it doesn't have data. + // the resize instruction to retrieve the shard, but it doesn't have data. // TODO: figure out a way to distinguish from "fragment not found" errors // which are true errors and which simply mean the fragment doesn't have data. if err == ErrFragmentNotFound { return nil } - return errors.Wrap(err, "retrieving slice") + return errors.Wrap(err, "retrieving shard") } else if rd == nil { - return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) + return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI) } - // Write to local frame and always close reader. + // Write to local field and always close reader. if err := func() error { defer rd.Close() _, err := frag.ReadFrom(rd) return err }(); err != nil { - return errors.Wrap(err, "copying remote slice") + return errors.Wrap(err, "copying remote shard") } } return nil @@ -1289,20 +1218,20 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err complete.Error = err.Error() } - if err := c.sendTo(DecodeNode(instr.Coordinator), complete); err != nil { - c.Logger.Printf("sending resizeInstructionComplete error: err=%s", err) + if err := c.sendTo(instr.Coordinator, complete); err != nil { + c.logger.Printf("sending resizeInstructionComplete error: err=%s", err) } }() return nil } -func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error { - j := c.Job(complete.JobID) + j := c.job(complete.JobID) // Abort the job if an error exists in the complete object. if complete.Error != "" { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.New(complete.Error) } @@ -1310,33 +1239,31 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc defer j.mu.Unlock() if j.isComplete() { - return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + return fmt.Errorf("resize job %d is no longer running", j.ID) } // Mark host complete. j.IDs[complete.Node.ID] = true if !j.nodesArePending() { - j.result <- ResizeJobStateDone + j.result <- resizeJobStateDone } return nil } -// Job returns a ResizeJob by id. -func (c *Cluster) Job(id int64) *ResizeJob { +// job returns a resizeJob by id. +func (c *cluster) job(id int64) *resizeJob { c.mu.RLock() defer c.mu.RUnlock() - return c.job(id) + return c.jobs[id] } -func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } - -type ResizeJob struct { +type resizeJob struct { ID int64 IDs map[string]bool - Instructions []*internal.ResizeInstruction - Broadcaster Broadcaster + Instructions []*ResizeInstruction + Broadcaster broadcaster action string result chan string @@ -1347,15 +1274,15 @@ type ResizeJob struct { Logger Logger } -// NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { +// newResizeJob returns a new instance of resizeJob. +func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. ids := make(map[string]bool) - if action == ResizeJobActionRemove { + if action == resizeJobActionRemove { for _, n := range existingNodes { // Exclude the removed node from the map. if n.ID == node.ID { @@ -1363,7 +1290,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } ids[n.ID] = false } - } else if action == ResizeJobActionAdd { + } else if action == resizeJobActionAdd { for _, n := range existingNodes { ids[n.ID] = false } @@ -1371,7 +1298,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { ids[node.ID] = false } - return &ResizeJob{ + return &resizeJob{ ID: rand.Int63(), IDs: ids, action: action, @@ -1380,50 +1307,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } } -func (j *ResizeJob) State() string { - j.mu.RLock() - defer j.mu.RUnlock() - return j.state -} - -func (j *ResizeJob) SetState(state string) { +func (j *resizeJob) setState(state string) { j.mu.Lock() - j.setState(state) + if j.state == "" || j.state == resizeJobStateRunning { + j.state = state + } j.mu.Unlock() } -func (j *ResizeJob) setState(state string) { - if j.state == "" || j.state == ResizeJobStateRunning { - j.state = state - } -} - -// Run distributes ResizeInstructions. -func (j *ResizeJob) Run() error { - j.Logger.Printf("run ResizeJob") +// run distributes ResizeInstructions. +func (j *resizeJob) run() error { + j.Logger.Printf("run resizeJob") // Set job state to RUNNING. - j.SetState(ResizeJobStateRunning) + j.setState(resizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") - j.result <- ResizeJobStateDone + j.Logger.Printf("resizeJob contains no pending tasks; mark as done") + j.result <- resizeJobStateDone return nil } - j.Logger.Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for resizeJob") err := j.distributeResizeInstructions() if err != nil { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.Wrap(err, "distributing instructions") } return nil } // isComplete return true if the job is any one of several completion states. -func (j *ResizeJob) isComplete() bool { +func (j *resizeJob) isComplete() bool { switch j.state { - case ResizeJobStateDone, ResizeJobStateAborted: + case resizeJobStateDone, resizeJobStateAborted: return true default: return false @@ -1431,7 +1348,7 @@ func (j *ResizeJob) isComplete() bool { } // nodesArePending returns true if any node is still working on the resize. -func (j *ResizeJob) nodesArePending() bool { +func (j *resizeJob) nodesArePending() bool { for _, complete := range j.IDs { if !complete { return true @@ -1440,15 +1357,15 @@ func (j *ResizeJob) nodesArePending() bool { return false } -func (j *ResizeJob) distributeResizeInstructions() error { +func (j *resizeJob) distributeResizeInstructions() error { j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) - // Loop through the ResizeInstructions in ResizeJob and send to each host. + // Loop through the ResizeInstructions in resizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. node := &Node{ ID: instr.Node.ID, - URI: decodeURI(instr.Node.URI), + URI: instr.Node.URI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1458,14 +1375,14 @@ func (j *ResizeJob) distributeResizeInstructions() error { return nil } -type NodeIDs []string +type nodeIDs []string -func (n NodeIDs) Len() int { return len(n) } -func (n NodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } -func (n NodeIDs) Less(i, j int) bool { return n[i] < n[j] } +func (n nodeIDs) Len() int { return len(n) } +func (n nodeIDs) Swap(i, j int) { n[i], n[j] = n[j], n[i] } +func (n nodeIDs) Less(i, j int) bool { return n[i] < n[j] } // ContainsID returns true if idi matches one of the nodesets's IDs. -func (n NodeIDs) ContainsID(id string) bool { +func (n nodeIDs) ContainsID(id string) bool { for _, nid := range n { if nid == id { return true @@ -1477,16 +1394,16 @@ func (n NodeIDs) ContainsID(id string) bool { // Topology represents the list of hosts in the cluster. type Topology struct { mu sync.RWMutex - NodeIDs []string + nodeIDs []string - ClusterID string + clusterID string // nodeStates holds the state of each node according to // the coordinator. Used during startup and data load. nodeStates map[string]string } -func NewTopology() *Topology { +func newTopology() *Topology { return &Topology{ nodeStates: make(map[string]string), } @@ -1500,11 +1417,11 @@ func (t *Topology) ContainsID(id string) bool { } func (t *Topology) containsID(id string) bool { - return NodeIDs(t.NodeIDs).ContainsID(id) + return nodeIDs(t.nodeIDs).ContainsID(id) } func (t *Topology) positionByID(nodeID string) int { - for i, tid := range t.NodeIDs { + for i, tid := range t.nodeIDs { if tid == nodeID { return i } @@ -1512,25 +1429,25 @@ func (t *Topology) positionByID(nodeID string) int { return -1 } -// AddID adds the node ID to the topology and returns true if added. -func (t *Topology) AddID(nodeID string) bool { +// addID adds the node ID to the topology and returns true if added. +func (t *Topology) addID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() if t.containsID(nodeID) { return false } - t.NodeIDs = append(t.NodeIDs, nodeID) + t.nodeIDs = append(t.nodeIDs, nodeID) - sort.Slice(t.NodeIDs, + sort.Slice(t.nodeIDs, func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] + return t.nodeIDs[i] < t.nodeIDs[j] }) return true } -// RemoveID removes the node ID from the topology and returns true if removed. -func (t *Topology) RemoveID(nodeID string) bool { +// removeID removes the node ID from the topology and returns true if removed. +func (t *Topology) removeID(nodeID string) bool { t.mu.Lock() defer t.mu.Unlock() @@ -1539,23 +1456,23 @@ func (t *Topology) RemoveID(nodeID string) bool { return false } - copy(t.NodeIDs[i:], t.NodeIDs[i+1:]) - t.NodeIDs[len(t.NodeIDs)-1] = "" - t.NodeIDs = t.NodeIDs[:len(t.NodeIDs)-1] + copy(t.nodeIDs[i:], t.nodeIDs[i+1:]) + t.nodeIDs[len(t.nodeIDs)-1] = "" + t.nodeIDs = t.nodeIDs[:len(t.nodeIDs)-1] return true } -// Encode converts t into its internal representation. -func (t *Topology) Encode() *internal.Topology { +// encode converts t into its internal representation. +func (t *Topology) encode() *internal.Topology { return encodeTopology(t) } // loadTopology reads the topology for the node. -func (c *Cluster) loadTopology() error { +func (c *cluster) loadTopology() error { buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) if os.IsNotExist(err) { - c.Topology = NewTopology() + c.Topology = newTopology() return nil } else if err != nil { return errors.Wrap(err, "reading file") @@ -1575,7 +1492,7 @@ func (c *Cluster) loadTopology() error { } // saveTopology writes the current topology to disk. -func (c *Cluster) saveTopology() error { +func (c *cluster) saveTopology() error { if err := os.MkdirAll(c.Path, 0777); err != nil { return errors.Wrap(err, "creating directory") @@ -1589,38 +1506,12 @@ func (c *Cluster) saveTopology() error { return nil } -func encodeTopology(topology *Topology) *internal.Topology { - if topology == nil { - return nil - } - return &internal.Topology{ - ClusterID: topology.ClusterID, - NodeIDs: topology.NodeIDs, - } -} - -func decodeTopology(topology *internal.Topology) (*Topology, error) { - if topology == nil { - return nil, nil - } - - t := NewTopology() - t.ClusterID = topology.ClusterID - t.NodeIDs = topology.NodeIDs - sort.Slice(t.NodeIDs, - func(i, j int) bool { - return t.NodeIDs[i] < t.NodeIDs[j] - }) - - return t, nil -} - -func (c *Cluster) considerTopology() error { +func (c *cluster) considerTopology() error { // Create ClusterID if one does not already exist. - if c.ID == "" { + if c.id == "" { u := uuid.NewV4() - c.ID = u.String() - c.Topology.ClusterID = c.ID + c.id = u.String() + c.Topology.clusterID = c.id } if c.Static { @@ -1628,13 +1519,13 @@ func (c *Cluster) considerTopology() error { } // If there is no .topology file, it's safe to proceed. - if len(c.Topology.NodeIDs) == 0 { + if len(c.Topology.nodeIDs) == 0 { return nil } // The local node (coordinator) must be in the .topology. if !c.Topology.ContainsID(c.Node.ID) { - return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.NodeIDs) + return fmt.Errorf("coordinator %s is not in topology: %v", c.Node.ID, c.Topology.nodeIDs) } // If local node is the only thing in .topology, continue. @@ -1648,7 +1539,7 @@ func (c *Cluster) considerTopology() error { } // ReceiveEvent represents an implementation of EventHandler. -func (c *Cluster) ReceiveEvent(e *NodeEvent) error { +func (c *cluster) ReceiveEvent(e *NodeEvent) error { // Ignore events sent from this node. if e.Node.ID == c.Node.ID { return nil @@ -1656,9 +1547,9 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { switch e.Event { case NodeJoin: - c.Logger.Printf("received NodeJoin event: %v", e) + c.logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { return nil } return c.nodeJoin(e.Node) @@ -1671,23 +1562,23 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { return nil } -func (c *Cluster) nodeJoin(node *Node) error { +func (c *cluster) nodeJoin(node *Node) error { if c.needTopologyAgreement() { // A host that is not part of the topology can't be added to the STARTING cluster. if !c.Topology.ContainsID(node.ID) { err := fmt.Sprintf("host is not in topology: %s", node.ID) - c.Logger.Printf("%v", err) + c.logger.Printf("%v", err) return errors.New(err) } - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node for agreement") } // Only change to normal if there is no existing data. Otherwise, // the coordinator needs to wait to receive READY messages (nodeStates) // from remote nodes before setting the cluster to state NORMAL. - if ok, err := c.Holder.HasData(); !ok && err == nil { + if ok, err := c.holder.HasData(); !ok && err == nil { // If the result of the previous AddNode completed the joining of nodes // in the topology, then change the state to NORMAL. if c.haveTopologyAgreement() { @@ -1710,13 +1601,13 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the cluster already contains the node, just send it the cluster status. // This is useful in the case where a node is restarted or temporarily leaves // the cluster. - if node := c.nodeByID(node.ID); node != nil { + if node := c.unprotectedNodeByID(node.ID); node != nil { return c.sendTo(node, c.Status()) } // If the holder does not yet contain data, go ahead and add the node. - if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.AddNode(node); err != nil { + if ok, err := c.holder.HasData(); !ok && err == nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1729,16 +1620,16 @@ func (c *Cluster) nodeJoin(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} + c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd} return nil } -// NodeLeave initiates the removal of a node from the cluster. -func (c *Cluster) NodeLeave(node *Node) error { +// nodeLeave initiates the removal of a node from the cluster. +func (c *cluster) nodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. - if !c.IsCoordinator() { - return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) + if !c.isCoordinator() { + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1746,7 +1637,7 @@ func (c *Cluster) NodeLeave(node *Node) error { } // Ensure that node is in the cluster. - if c.nodeByID(node.ID) == nil { + if c.unprotectedNodeByID(node.ID) == nil { return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) } @@ -1756,18 +1647,12 @@ func (c *Cluster) NodeLeave(node *Node) error { } // See if resize job can be generated - _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) - - if err != nil { + if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil { return errors.Wrap(err, "generating job") } - return c.nodeLeave(node) -} - -func (c *Cluster) nodeLeave(node *Node) error { // Get the actual node in the local cluster. - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) // Don't do anything else if the cluster doesn't contain the node. if n == nil { @@ -1775,8 +1660,8 @@ func (c *Cluster) nodeLeave(node *Node) error { } // If the holder does not yet contain data, go ahead and remove the node. - if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.RemoveNode(n); err != nil { + if ok, err := c.holder.HasData(); !ok && err == nil { + if err := c.removeNode(n); err != nil { return errors.Wrap(err, "removing node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1789,28 +1674,28 @@ func (c *Cluster) nodeLeave(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove} return nil } -func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() - c.Logger.Printf("merge cluster status: %v", cs) + c.logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.isCoordinator() { + if c.unprotectedIsCoordinator() { return nil } // Set ClusterID. c.setID(cs.ClusterID) - officialNodes := DecodeNodes(cs.Nodes) + officialNodes := cs.Nodes // Add all nodes from the coordinator. for _, node := range officialNodes { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } } @@ -1831,7 +1716,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } for _, nodeID := range nodeIDsToRemove { - if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { + if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil { return errors.Wrap(err, "removing node") } } @@ -1842,3 +1727,133 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { return nil } + +func (c *cluster) setStatic(hosts []string) error { + c.Static = true + c.Coordinator = c.Node.ID + for _, address := range hosts { + uri, err := NewURIFromAddress(address) + if err != nil { + return errors.Wrap(err, "getting URI") + } + c.Nodes = append(c.Nodes, &Node{URI: *uri}) + } + return nil +} + +type ClusterStatus struct { + ClusterID string + State string + Nodes []*Node +} + +type ResizeInstruction struct { + JobID int64 + Node *Node + Coordinator *Node + Sources []*ResizeSource + Schema *Schema + ClusterStatus *ClusterStatus +} + +type ResizeSource struct { + Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` + Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` +} + +// Schema contains information about indexes and their configuration. +type Schema struct { + Indexes []*IndexInfo +} + +func encodeTopology(topology *Topology) *internal.Topology { + if topology == nil { + return nil + } + return &internal.Topology{ + ClusterID: topology.clusterID, + NodeIDs: topology.nodeIDs, + } +} + +func decodeTopology(topology *internal.Topology) (*Topology, error) { + if topology == nil { + return nil, nil + } + + t := newTopology() + t.clusterID = topology.ClusterID + t.nodeIDs = topology.NodeIDs + sort.Slice(t.nodeIDs, + func(i, j int) bool { + return t.nodeIDs[i] < t.nodeIDs[j] + }) + + return t, nil +} + +type CreateShardMessage struct { + Index string + Shard uint64 +} + +type CreateIndexMessage struct { + Index string + Meta *IndexOptions +} + +type DeleteIndexMessage struct { + Index string +} + +type CreateFieldMessage struct { + Index string + Field string + Meta *FieldOptions +} + +type DeleteFieldMessage struct { + Index string + Field string +} + +type CreateViewMessage struct { + Index string + Field string + View string +} +type DeleteViewMessage struct { + Index string + Field string + View string +} + +type ResizeInstructionComplete struct { + JobID int64 + Node *Node + Error string +} + +type SetCoordinatorMessage struct { + New *Node +} + +type UpdateCoordinatorMessage struct { + New *Node +} + +type NodeStateMessage struct { + NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` +} + +type NodeStatus struct { + Node *Node + MaxShards map[string]uint64 + Schema *Schema +} + +type RecalculateCaches struct{} diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 6b1f6e72b..5b514e517 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,12 +15,16 @@ package pilosa import ( + "bytes" "io/ioutil" + "math/rand" "reflect" "strings" "testing" + "testing/quick" - "github.com/pilosa/pilosa/internal" + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" ) // Ensure that fragCombos creates the correct fragment mapping. @@ -38,20 +42,20 @@ func TestFragCombos(t *testing.T) { node0 := &Node{ID: "node0", URI: *uri0} node1 := &Node{ID: "node1", URI: *uri1} - c := NewCluster() + c := newCluster() c.addNodeBasicSorted(node0) c.addNodeBasicSorted(node1) tests := []struct { idx string - maxSlice uint64 - frameViews viewsByFrame + maxShard uint64 + fieldViews viewsByField expected fragsByHost }{ { idx: "i", - maxSlice: uint64(2), - frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, + maxShard: uint64(2), + fieldViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, @@ -59,8 +63,8 @@ func TestFragCombos(t *testing.T) { }, { idx: "foo", - maxSlice: uint64(3), - frameViews: viewsByFrame{"f": []string{"v0"}}, + maxShard: uint64(3), + fieldViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, @@ -69,7 +73,7 @@ func TestFragCombos(t *testing.T) { } for _, test := range tests { - actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews) + actual := c.fragCombos(test.idx, test.maxShard, test.fieldViews) if !reflect.DeepEqual(actual, test.expected) { t.Errorf("expected: %v, but got: %v", test.expected, actual) } @@ -115,29 +119,29 @@ func TestFragSources(t *testing.T) { node2 := &Node{ID: "node2", URI: *uri2} node3 := &Node{ID: "node3", URI: *uri3} - c1 := NewCluster() + c1 := newCluster() c1.ReplicaN = 1 c1.addNodeBasicSorted(node0) c1.addNodeBasicSorted(node1) - c2 := NewCluster() + c2 := newCluster() c2.ReplicaN = 1 c2.addNodeBasicSorted(node0) c2.addNodeBasicSorted(node1) c2.addNodeBasicSorted(node2) - c3 := NewCluster() + c3 := newCluster() c3.ReplicaN = 2 c3.addNodeBasicSorted(node0) c3.addNodeBasicSorted(node1) - c4 := NewCluster() + c4 := newCluster() c4.ReplicaN = 2 c4.addNodeBasicSorted(node0) c4.addNodeBasicSorted(node1) c4.addNodeBasicSorted(node2) - c5 := NewCluster() + c5 := newCluster() c5.ReplicaN = 2 c5.addNodeBasicSorted(node0) c5.addNodeBasicSorted(node1) @@ -145,44 +149,44 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) + field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 101, nil) + _, err = field.SetBit(1, 101, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 1300000, nil) + _, err = field.SetBit(1, 1300000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 2600000, nil) + _, err = field.SetBit(1, 2600000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 3900000, nil) + _, err = field.SetBit(1, 3900000, nil) if err != nil { t.Fatal(err) } tests := []struct { - from *Cluster - to *Cluster + from *cluster + to *cluster idx *Index - expected map[string][]*internal.ResizeSource + expected map[string][]*ResizeSource err string }{ { from: c1, to: c2, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{}, - "node1": []*internal.ResizeSource{}, - "node2": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{}, + "node1": []*ResizeSource{}, + "node2": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -191,13 +195,13 @@ func TestFragSources(t *testing.T) { from: c4, to: c3, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node1", &internal.URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)}, }, }, err: "", @@ -206,15 +210,15 @@ func TestFragSources(t *testing.T) { from: c5, to: c4, idx: idx, - expected: map[string][]*internal.ResizeSource{ - "node0": []*internal.ResizeSource{ - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, - {&internal.Node{"node2", &internal.URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, + expected: map[string][]*ResizeSource{ + "node0": []*ResizeSource{ + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)}, + {&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)}, }, - "node1": []*internal.ResizeSource{ - {&internal.Node{"node0", &internal.URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, + "node1": []*ResizeSource{ + {&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)}, }, - "node2": []*internal.ResizeSource{}, + "node2": []*ResizeSource{}, }, err: "", }, @@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) { { existingNodes: []*Node{node0, node1}, node: node2, - action: ResizeJobActionAdd, + action: resizeJobActionAdd, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { existingNodes: []*Node{node0, node1, node2}, node: node2, - action: ResizeJobActionRemove, + action: resizeJobActionRemove, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, }, } for _, test := range tests { - actual := NewResizeJob(test.existingNodes, test.node, test.action) + actual := newResizeJob(test.existingNodes, test.node, test.action) if err != nil { t.Fatal(err) } @@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) { } } } + +// Ensure the cluster can fairly distribute partitions across the nodes. +func TestCluster_Owners(t *testing.T) { + c := cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, + }, + Hasher: NewTestModHasher(), + ReplicaN: 2, + } + + // Verify nodes are distributed. + if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } + + // Verify nodes go around the ring. + if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } +} + +// Ensure the partitioner can assign a fragment to a partition. +func TestCluster_Partition(t *testing.T) { + if err := quick.Check(func(index string, shard uint64, partitionN int) bool { + c := newCluster() + c.partitionN = partitionN + + partitionID := c.partition(index, shard) + if partitionID < 0 || partitionID >= partitionN { + t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0], _ = quick.Value(reflect.TypeOf(""), rand) + values[1] = reflect.ValueOf(uint64(rand.Uint32())) + values[2] = reflect.ValueOf(rand.Intn(1000) + 1) + }, + }); err != nil { + t.Fatal(err) + } +} + +// Ensure the hasher can hash correctly. +func TestHasher(t *testing.T) { + for _, tt := range []struct { + key uint64 + bucket []int + }{ + // Generated from the reference C++ code + {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, + {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, + {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, + } { + for i, v := range tt.bucket { + if got := newHasher().Hash(tt.key, i+1); got != v { + t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) + } + } + } +} + +// Ensure ContainsShards can find the actual shard list for node and index. +func TestCluster_ContainsShards(t *testing.T) { + c := NewTestCluster(5) + c.ReplicaN = 3 + shards := c.containsShards("test", 10, c.Nodes[2]) + + if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected shars for node's index: %v", shards) + } +} + +func TestCluster_Nodes(t *testing.T) { + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} + + nodes := []*Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterURI", func(t *testing.T) { + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +// NEXT: move this test to internal and unexport IsCoordinator +func TestCluster_Coordinator(t *testing.T) { + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + + c1 := *newCluster() + c1.Node = node1 + c1.Coordinator = node1.ID + c2 := *newCluster() + c2.Node = node2 + c2.Coordinator = node1.ID + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.isCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Node) + } else if c2.isCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Node) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + + t.Run("AddNode", func(t *testing.T) { + err := c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + err = c1.addNode(node2) + if err != nil { + t.Fatal(err) + } + + actual := c1.nodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) + } + }) +} + +// Ensure that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + t.Run("Single node, no data", func(t *testing.T) { + tc := NewClusterCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + expectedTop := &Topology{ + nodeIDs: []string{node.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.addNode() + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + nodeIDs: []string{node.Node.ID}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.addNode() + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + nodeIDs: []string{"some-other-host"}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "coordinator node0 is not in topology: [some-other-host]" + err := tc.Open() + if err == nil || errors.Cause(err).Error() != expected { + t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.addNode() + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.addNode() + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + nodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs) + } else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.addNode() + node0 := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + nodeIDs: []string{"node0", "node2"}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: node1" + err := tc.addNode() + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.addNode() + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.addNode() + node0 := tc.Clusters[0] + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Bit Data to node0. + if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + tc.SetBit("i", "f", 1, 101, nil) + tc.SetBit("i", "f", 1, 1300000, nil) + + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Field := node0.holder.Field("i", "f") + node0View := node0Field.view("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() + + // addNode needs to block until the resize process has completed. + tc.addNode() + node1 := tc.Clusters[1] + + // Ensure that nodes come up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + nodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs) + } else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs) + } + + // Bits + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node1Field := node1.holder.Field("i", "f") + node1View := node1Field.view("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} + +// Ensures that coordinator can be changed. +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { + c := NewTestCluster(2) + + oldNode := c.Nodes[0] + newNode := c.Nodes[1] + + // Update coordinator to the same value. + if c.updateCoordinator(oldNode) { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) + } + + // Update coordinator to a new value. + if !c.updateCoordinator(newNode) { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) + } + }) +} diff --git a/cluster_test.go b/cluster_test.go deleted file mode 100644 index 913cd5db3..000000000 --- a/cluster_test.go +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "math/rand" - "reflect" - "testing" - "testing/quick" - - "github.com/davecgh/go-spew/spew" -) - -// Ensure the cluster can fairly distribute partitions across the nodes. -func TestCluster_Owners(t *testing.T) { - c := Cluster{ - Nodes: []*Node{ - {URI: NewTestURIFromHostPort("serverA", 1000)}, - {URI: NewTestURIFromHostPort("serverB", 1000)}, - {URI: NewTestURIFromHostPort("serverC", 1000)}, - }, - Hasher: NewTestModHasher(), - ReplicaN: 2, - } - - // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } - - // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } -} - -// Ensure the partitioner can assign a fragment to a partition. -func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := NewCluster() - c.PartitionN = partitionN - - partitionID := c.Partition(index, slice) - if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) - } - - return true - }, &quick.Config{ - Values: func(values []reflect.Value, rand *rand.Rand) { - values[0], _ = quick.Value(reflect.TypeOf(""), rand) - values[1] = reflect.ValueOf(uint64(rand.Uint32())) - values[2] = reflect.ValueOf(rand.Intn(1000) + 1) - }, - }); err != nil { - t.Fatal(err) - } -} - -// Ensure the hasher can hash correctly. -func TestHasher(t *testing.T) { - for _, tt := range []struct { - key uint64 - bucket []int - }{ - // Generated from the reference C++ code - {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, - {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, - {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, - } { - for i, v := range tt.bucket { - if got := NewHasher().Hash(tt.key, i+1); got != v { - t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) - } - } - } -} - -// Ensure OwnsSlices can find the actual slice list for node and index. -func TestCluster_OwnsSlices(t *testing.T) { - c := NewTestCluster(5) - slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) - - if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { - c := NewTestCluster(5) - c.ReplicaN = 3 - slices := c.ContainsSlices("test", 10, c.Nodes[2]) - - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -func TestCluster_Nodes(t *testing.T) { - uri0 := NewTestURIFromHostPort("node0", 0) - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - uri3 := NewTestURIFromHostPort("node3", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - node3 := &Node{ID: "node3", URI: uri3} - - nodes := []*Node{node0, node1, node2} - - t.Run("NodeIDs", func(t *testing.T) { - actual := Nodes(nodes).IDs() - expected := []string{node0.ID, node1.ID, node2.ID} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Filter", func(t *testing.T) { - actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("FilterURI", func(t *testing.T) { - actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Contains", func(t *testing.T) { - actualTrue := Nodes(nodes).Contains(node1) - actualFalse := Nodes(nodes).Contains(node3) - if !reflect.DeepEqual(actualTrue, true) { - t.Errorf("expected: %v, but got: %v", true, actualTrue) - } - if !reflect.DeepEqual(actualFalse, false) { - t.Errorf("expected: %v, but got: %v", false, actualTrue) - } - }) - - t.Run("Clone", func(t *testing.T) { - clone := Nodes(nodes).Clone() - actual := Nodes(clone).URIs() - expected := []URI{uri0, uri1, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) -} - -func TestCluster_Coordinator(t *testing.T) { - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - - c1 := *NewCluster() - c1.Node = node1 - c1.Coordinator = node1.ID - c2 := *NewCluster() - c2.Node = node2 - c2.Coordinator = node1.ID - - t.Run("IsCoordinator", func(t *testing.T) { - if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Node) - } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Node) - } - }) -} - -func TestCluster_Topology(t *testing.T) { - c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - - uri0 := NewTestURIFromHostPort("host0", 0) - uri1 := NewTestURIFromHostPort("host1", 0) - uri2 := NewTestURIFromHostPort("host2", 0) - invalid := NewTestURIFromHostPort("invalid", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} - - t.Run("AddNode", func(t *testing.T) { - err := c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - // add the same host. - err = c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - err = c1.AddNode(node2) - if err != nil { - t.Fatal(err) - } - - actual := c1.NodeIDs() - expected := []string{node0.ID, node1.ID, node2.ID} - - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("ContainsID", func(t *testing.T) { - if !c1.Topology.ContainsID(node1.ID) { - t.Errorf("!ContainsHost error: %v", node1.ID) - } else if c1.Topology.ContainsID(nodeinvalid.ID) { - t.Errorf("ContainsHost error: %v", nodeinvalid.ID) - } - }) -} - -// Ensure that general cluster functionality works as expected. -func TestCluster_ResizeStates(t *testing.T) { - - t.Run("Single node, no data", func(t *testing.T) { - tc := NewClusterCluster(1) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - node := tc.Clusters[0] - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"some-other-host"}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" - err := tc.Open() - if err == nil || err.Error() != expected { - t.Errorf("did not receive expected error: %s", expected) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - tc.AddNode(false) - - node0 := tc.Clusters[0] - node1 := tc.Clusters[1] - - // Ensure that nodes comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"node0", "node2"}, - } - tc.WriteTopology(node0.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node is in state STARTING before the other node joins. - if node0.State() != ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) - } - - // Expect an error by adding a node not in the topology. - expectedError := "host is not in topology: node1" - err := tc.AddNode(false) - if err == nil || err.Error() != expectedError { - t.Errorf("did not receive expected error: %s", expectedError) - } - - tc.AddNode(false) - node2 := tc.Clusters[2] - - // Ensure that node comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { - t.Fatal(err) - } - tc.SetBit("i", "f", "standard", 1, 101, nil) - tc.SetBit("i", "f", "standard", 1, 1300000, nil) - - // Add Field Data to node0. - if err := tc.CreateFrame("i", "fields", FrameOptions{ - Fields: []*Field{ - { - Name: "fld0", - Type: FieldTypeInt, - Min: -100, - Max: 100, - }, - }, - }); err != nil { - t.Fatal(err) - } - tc.SetFieldValue("i", "fields", 1, "fld0", -10) - tc.SetFieldValue("i", "fields", 1, "fld0", 10) - tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) - tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) - - // Before starting the resize, get the CheckSum to use for - // comparison later. - node0Frame := node0.Holder.Frame("i", "f") - node0View := node0Frame.View("standard") - node0Fragment := node0View.Fragment(1) - node0Checksum := node0Fragment.Checksum() - - node0Frame = node0.Holder.Frame("i", "fields") - node0View = node0Frame.View("field_fld0") - node0Fragment = node0View.Fragment(1) - node0ChecksumFld := node0Fragment.Checksum() - - // AddNode needs to block until the resize process has completed. - tc.AddNode(false) - node1 := tc.Clusters[1] - - // Ensure that nodes come up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Bits - // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Frame := node1.Holder.Frame("i", "f") - node1View := node1Frame.View("standard") - node1Fragment := node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { - t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) - } - - // Values - // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. - node1Frame = node1.Holder.Frame("i", "fields") - node1View = node1Frame.View("field_fld0") - node1Fragment = node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) { - t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) -} - -// Ensures that coordinator can be changed. -func TestCluster_UpdateCoordinator(t *testing.T) { - t.Run("UpdateCoordinator", func(t *testing.T) { - c := NewTestCluster(2) - - oldNode := c.Nodes[0] - newNode := c.Nodes[1] - - // Update coordinator to the same value. - if c.UpdateCoordinator(oldNode) { - t.Errorf("did not expect coordinator to change") - } else if c.Coordinator != oldNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) - } - - // Update coordinator to a new value. - if !c.UpdateCoordinator(newNode) { - t.Errorf("expected coordinator to change") - } else if c.Coordinator != newNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) - } - }) -} diff --git a/cmd/bench.go b/cmd/bench.go deleted file mode 100644 index d4b2b8580..000000000 --- a/cmd/bench.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/pilosa/pilosa/ctl" -) - -var Bencher *ctl.BenchCommand - -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr) - benchCmd := &cobra.Command{ - Use: "bench", - Short: "Benchmark operations.", - Long: ` -Executes a benchmark for a given operation against the index. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Bencher.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := benchCmd.Flags() - flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") - flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") - flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") - flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") - ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify) - - return benchCmd -} - -func init() { - subcommandFns["bench"] = NewBenchCommand -} diff --git a/cmd/bench_test.go b/cmd/bench_test.go deleted file mode 100644 index 4b94d9392..000000000 --- a/cmd/bench_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestBenchHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "bench", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa bench") || err != nil { - t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestBenchConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"bench", "--operation", "set-bit"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Bencher.Host, "localhost:12345") - v.Check(cmd.Bencher.Index, "myindex") - v.Check(cmd.Bencher.Frame, "f1") - v.Check(cmd.Bencher.Op, "set-bit") - v.Check(cmd.Bencher.N, 0) - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/cmd/check.go b/cmd/check.go index 56ae13265..8785a78e1 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var Checker *ctl.CheckCommand +var checker *ctl.CheckCommand -func NewCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) +func newCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) checkCmd := &cobra.Command{ Use: "check [path2]...", Short: "Do a consistency check on a pilosa data file.", @@ -39,8 +39,8 @@ Performs a consistency check on data files. if len(args) == 0 { return fmt.Errorf("path required") } - Checker.Paths = args - if err := Checker.Run(context.Background()); err != nil { + checker.Paths = args + if err := checker.Run(context.Background()); err != nil { return err } return nil @@ -50,5 +50,5 @@ Performs a consistency check on data files. } func init() { - subcommandFns["check"] = NewCheckCommand + subcommandFns["check"] = newCheckCommand } diff --git a/cmd/config.go b/cmd/config.go index 9452cbc3c..3d65fa131 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/server" ) -var Conf *ctl.ConfigCommand +var conf *ctl.ConfigCommand -func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) +func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "config", @@ -36,8 +36,8 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Long: `config prints the current configuration to stdout`, RunE: func(cmd *cobra.Command, args []string) error { - Conf.Config = Server.Config - if err := Conf.Run(context.Background()); err != nil { + conf.Config = Server.Config + if err := conf.Run(context.Background()); err != nil { return err } return nil @@ -51,5 +51,5 @@ func NewConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command } func init() { - subcommandFns["config"] = NewConfigCommand + subcommandFns["config"] = newConfigCommand } diff --git a/cmd/export.go b/cmd/export.go index 951c4a0a9..d0f63edbf 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -26,7 +26,7 @@ import ( var Exporter *ctl.ExportCommand -func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) exportCmd := &cobra.Command{ Use: "export", @@ -52,7 +52,7 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify) @@ -60,5 +60,5 @@ The file does not contain any headers. } func init() { - subcommandFns["export"] = NewExportCommand + subcommandFns["export"] = newExportCommand } diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..6f7a49b8f 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") - v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.Field, "f1") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, diff --git a/cmd/generate_config.go b/cmd/generate_config.go index b0622b64d..0b5b81462 100644 --- a/cmd/generate_config.go +++ b/cmd/generate_config.go @@ -24,17 +24,17 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var GenerateConf *ctl.GenerateConfigCommand +var generateConf *ctl.GenerateConfigCommand -func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - GenerateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) +func newGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) confCmd := &cobra.Command{ Use: "generate-config", Short: "Print the default configuration.", Long: `generate-config prints the default configuration to stdout `, RunE: func(cmd *cobra.Command, args []string) error { - if err := GenerateConf.Run(context.Background()); err != nil { + if err := generateConf.Run(context.Background()); err != nil { return err } return nil @@ -45,5 +45,5 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra. } func init() { - subcommandFns["generate-config"] = NewGenerateConfigCommand + subcommandFns["generate-config"] = newGenerateConfigCommand } diff --git a/cmd/import.go b/cmd/import.go index 8dd31181c..7b4d00efd 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -20,20 +20,19 @@ import ( "github.com/spf13/cobra" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/ctl" ) var Importer *ctl.ImportCommand -// NewImportCommand runs the Pilosa import subcommand for ingesting bulk data. -func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +// newImportCommand runs the Pilosa import subcommand for ingesting bulk data. +func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Importer = ctl.NewImportCommand(stdin, stdout, stderr) importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The data -of the CSV file are grouped by slice for the most efficient import. + Long: `Bulk imports one or more CSV files to a host's index and field. The data +of the CSV file are grouped by shard for the most efficient import. The format of the CSV file is: @@ -54,20 +53,19 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags := importCmd.Flags() flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") - flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") - flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.") + flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") - flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") - flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") + //flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field") + //flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked") + //flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) return importCmd } func init() { - subcommandFns["import"] = NewImportCommand + subcommandFns["import"] = newImportCommand } diff --git a/cmd/import_test.go b/cmd/import_test.go index 8117d46fa..e8e5d61f4 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Importer.Host, "localhost:12345") v.Check(cmd.Importer.Index, "myindex") - v.Check(cmd.Importer.Frame, "f1") + v.Check(cmd.Importer.Field, "f1") return v.Error() }, }, diff --git a/cmd/inspect.go b/cmd/inspect.go index e8526a414..096787337 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -25,10 +25,10 @@ import ( "github.com/pilosa/pilosa/ctl" ) -var Inspector *ctl.InspectCommand +var inspector *ctl.InspectCommand -func NewInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) +func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) inspectCmd := &cobra.Command{ Use: "inspect", @@ -42,8 +42,8 @@ Inspects a data file and provides stats. } else if len(args) > 1 { return fmt.Errorf("only one path allowed") } - Inspector.Path = args[0] - if err := Inspector.Run(context.Background()); err != nil { + inspector.Path = args[0] + if err := inspector.Run(context.Background()); err != nil { return err } return nil @@ -53,5 +53,5 @@ Inspects a data file and provides stats. } func init() { - subcommandFns["inspect"] = NewInspectCommand + subcommandFns["inspect"] = newInspectCommand } diff --git a/cmd/server.go b/cmd/server.go index d906cf189..83d18504c 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -27,8 +27,8 @@ import ( // Server is global so that tests can control and verify it. var Server *server.Command -// NewServeCmd creates a pilosa server and runs it with command line flags. -func NewServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { +// newServeCmd creates a pilosa server and runs it with command line flags. +func newServeCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Server = server.NewCommand(stdin, stdout, stderr) serveCmd := &cobra.Command{ Use: "server", @@ -52,5 +52,5 @@ on the configured port.`, } func init() { - subcommandFns["server"] = NewServeCmd + subcommandFns["server"] = newServeCmd } diff --git a/cmd/server_test.go b/cmd/server_test.go index abbe8d7a4..e58f8af4d 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -15,7 +15,6 @@ package cmd_test import ( - "errors" "io/ioutil" "strings" "testing" @@ -24,6 +23,7 @@ import ( "github.com/pilosa/pilosa/cmd" _ "github.com/pilosa/pilosa/test" "github.com/pilosa/pilosa/toml" + "github.com/pkg/errors" ) func TestServerHelp(t *testing.T) { diff --git a/ctl/bench.go b/ctl/bench.go deleted file mode 100644 index 4a71169d9..000000000 --- a/ctl/bench.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "fmt" - "io" - "math/rand" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// BenchCommand represents a command for benchmarking index operations. -type BenchCommand struct { - // Destination host and port. - Host string - - // Name of the index & frame to execute against. - Index string - Frame string - - // Type of operation and number to execute. - Op string - N int - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewBenchCommand returns a new instance of BenchCommand. -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { - return &BenchCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the bench command. -func (cmd *BenchCommand) Run(ctx context.Context) error { - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - switch cmd.Op { - case "set-bit": - return cmd.runSetBit(ctx, client) - case "": - return errors.New("op required") - default: - return fmt.Errorf("unknown bench op: %q", cmd.Op) - } -} - -// runSetBit executes a benchmark of random SetBit() operations. -func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error { - if cmd.N == 0 { - return errors.New("operation count required") - } else if cmd.Index == "" { - return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired - } - - const maxRowID = 1000 - const maxColumnID = 100000 - - startTime := time.Now() - - // Execute operation continuously. - for i := 0; i < cmd.N; i++ { - rowID := rand.Intn(maxRowID) - columnID := rand.Intn(maxColumnID) - - queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID), - Remote: false, - } - if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil { - return err - } - } - - // Print results. - elapsed := time.Since(startTime) - fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds()) - - return nil -} - -func (cmd *BenchCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go deleted file mode 100644 index 766ec8a1a..000000000 --- a/ctl/bench_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bufio" - "bytes" - "context" - "fmt" - "io" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pkg/errors" -) - -func TestBenchCommand_InvalidOption(t *testing.T) { - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - cm := NewBenchCommand(stdin, stdout, stderr) - err := cm.Run(context.Background()) - if errors.Cause(err) != pilosa.ErrHostRequired { - t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err) - } - - cm.Host = "localhost:10101" - err = cm.Run(context.Background()) - if err.Error() != "op required" { - t.Fatalf("Expect err: %s, actual err: %s", "op required", err) - } - - cm.Op = "test" - err = cm.Run(context.Background()) - if err.Error() != "unknown bench op: \"test\"" { - t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err) - } - -} - -func TestBenchCommand_Run(t *testing.T) { - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - - cm := NewBenchCommand(stdin, w, w) - cm.Op = "set-bit" - cm.Host = "localhost:10101" - - err := cm.Run(context.Background()) - if err.Error() != "operation count required" { - t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err) - } - - cm.N = 1 - err = cm.Run(context.Background()) - if err != pilosa.ErrIndexRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err) - } - - cm.Index = "i" - err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) - } - - cm.Frame = "f" - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) - fmt.Println(buf.String()) - if err != nil { - fmt.Println(buf.String()) - } -} - -// declare stdin, stdout, stderr -func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { - rder := []byte{} - stdin := bytes.NewReader(rder) - stdout := bufio.NewWriter(&buf) - stderr := bufio.NewWriter(&buf) - return stdin, stdout, stderr -} diff --git a/ctl/common.go b/ctl/common.go index 11042a704..f7f429c58 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -17,7 +17,7 @@ package ctl import ( "crypto/tls" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -36,8 +36,8 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP flags.BoolVarP(skipVerify, "tls.skip-verify", "", false, "Skip TLS certificate verification (not secure)") } -// CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { +// commandClient returns a pilosa.InternalHTTPClient for the command +func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { @@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) + client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/ctl/export.go b/ctl/export.go index 5ee8d2120..4a20b0cc2 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -30,9 +30,9 @@ type ExportCommand struct { // Remote host and port. Host string - // Name of the index & frame to export from. + // Name of the index & field to export from. Index string - Frame string + Field string // Filename to export to. Path string @@ -57,8 +57,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + } else if cmd.Field == "" { + return pilosa.ErrFieldRequired } // Use output file, if specified. @@ -75,21 +75,21 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Create a client to the server. - client, err := CommandClient(cmd) + client, err := commandClient(cmd) if err != nil { return errors.Wrap(err, "creating client") } - // Determine slice count. - maxSlices, err := client.MaxSliceByIndex(ctx) + // Determine shard count. + maxShards, err := client.MaxShardByIndex(ctx) if err != nil { - return errors.Wrap(err, "getting slice count") + return errors.Wrap(err, "getting shard count") } - // Export each slice. - for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { - logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + // Export each shard. + for shard := uint64(0); shard <= maxShards[cmd.Index]; shard++ { + logger.Printf("exporting shard: %d", shard) + if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, shard, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/export_test.go b/ctl/export_test.go index 469ca7d75..e3189efe3 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -38,31 +38,25 @@ func TestExportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } } func TestExportCommand_Run(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewExportCommand(stdin, stdout, stderr) + hostport := cmd.API.Node().URI.HostPort() + cm.Host = hostport - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() - - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader(""))) cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } diff --git a/ctl/generate_config.go b/ctl/generate_config.go index a64678475..a9429ec80 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -18,9 +18,11 @@ import ( "context" "fmt" "io" - "strings" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // GenerateConfigCommand represents a command for printing a default config. @@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera // Run prints out the default config. func (cmd *GenerateConfigCommand) Run(ctx context.Context) error { - fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` -data-dir = "~/.pilosa" -bind = "localhost:10101" -max-writes-per-request = 5000 - -[cluster] - replicas = 1 - hosts = [ - "localhost:10101", - ] - -[anti-entropy] - interval = "10m0s" - -[profile] - cpu = "" - cpu-time = "30s" - -[metric] - service = "statsd" - host = "127.0.0.1:8125" - poll-interval = "0m15s" -`)+"\n") + conf := server.NewConfig() + ret, err := toml.Marshal(*conf) + if err != nil { + return errors.Wrap(err, "unmarshaling default config") + } + fmt.Fprintf(cmd.Stdout, "%s\n", ret) return nil } diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 56f392bba..26b531e8f 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), "localhost:10101") { + } else if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import.go b/ctl/import.go index a9d70fd7c..76b794a93 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" ) @@ -35,20 +36,16 @@ type ImportCommand struct { // Destination host and port. Host string `json:"host"` - // Name of the index & frame to import into. + // Name of the index & field to import into. Index string `json:"index"` - Frame string `json:"frame"` + Field string `json:"field"` - // Options for index & frame to be created if they don't exist - IndexOptions pilosa.IndexOptions - FrameOptions pilosa.FrameOptions + // Options for index & field to be created if they don't exist + indexOptions pilosa.IndexOptions // CreateSchema ensures the schema exists before import CreateSchema bool - // For Range-Encoded fields, name of the Field to import into. - Field string `json:"field"` - // Indicates that the payload should be treated as string keys. StringKeys bool `json:"StringKeys"` @@ -62,7 +59,7 @@ type ImportCommand struct { Sort bool `json:"sort"` // Reusable client. - Client pilosa.InternalClient `json:"-"` + client pilosa.InternalClient `json:"-"` // Standard input/output *pilosa.CmdIO @@ -83,20 +80,20 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - // Index and frame are validated early before the files are parsed. + // Index and field are validated early before the files are parsed. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + } else if cmd.Field == "" { + return pilosa.ErrFieldRequired } else if len(cmd.Paths) == 0 { return errors.New("path required") } // Create a client to the server. - client, err := CommandClient(cmd) + client, err := commandClient(cmd) if err != nil { return errors.Wrap(err, "creating client") } - cmd.Client = client + cmd.client = client if cmd.CreateSchema { err := cmd.ensureSchema(ctx) @@ -105,10 +102,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } - // Import each path and import by slice. + // Determine the field type in order to correctly handle the input data. + fieldType := pilosa.DefaultFieldType + schema, err := cmd.client.Schema(ctx) + if err != nil { + return errors.Wrap(err, "getting schema") + } + for _, index := range schema { + if index.Name == cmd.Index { + for _, field := range index.Fields { + if field.Name == cmd.Field { + fieldType = field.Options.Type + } + } + } + } + + // Import each path and import by shard. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) - if err := cmd.importPath(ctx, path); err != nil { + if err := cmd.importPath(ctx, fieldType, path); err != nil { return err } } @@ -117,22 +130,22 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { - err := cmd.Client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) + err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions) if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) + err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) if err != nil { - return fmt.Errorf("Error Creating Frame: %s", err) + return fmt.Errorf("Error Creating Field: %s", err) } return nil } // importPath parses a path into bits and imports it to the server. -func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { - // If a field is provided, treat the import data as values to be range-encoded. - if cmd.Field != "" { - return cmd.bufferFieldValues(ctx, path) +func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error { + // If fieldType is `int`, treat the import data as values to be range-encoded. + if fieldType == pilosa.FieldTypeInt { + return cmd.bufferValues(ctx, path) } else { if cmd.StringKeys { return cmd.bufferBitsK(ctx, path) @@ -230,18 +243,18 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group bits by slice. + // Group bits by shard. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := pilosa.Bits(bits).GroupBySlice() + bitsByShard := http.Bits(bits).GroupByShard() // Parse path into bits. - for slice, chunk := range bitsBySlice { + for shard, chunk := range bitsByShard { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(chunk)) + sort.Sort(http.BitsByPos(chunk)) } - logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(chunk)) + if err := cmd.client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -338,15 +351,15 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er // TODO: does it help to sort the rowKeys? logger.Printf("importing keys: n=%d", len(bits)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { + if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil { return errors.Wrap(err, "importing keys") } return nil } -// bufferFieldValues buffers slices of fieldValues to be imported as a batch. -func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error { +// bufferValues buffers slices of FieldValues to be imported as a batch. +func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { a := make([]pilosa.FieldValue, 0, cmd.BufferSize) var r *csv.Reader @@ -394,7 +407,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } val.ColumnID = columnID - // Parse field value. + // Parse FieldValue. value, err := strconv.ParseInt(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid value on row %d: %q", rnum, record[1]) @@ -403,9 +416,9 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er a = append(a, val) - // If we've reached the buffer size then import field values. + // If we've reached the buffer size then import FieldValues. if len(a) == cmd.BufferSize { - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } a = a[:0] @@ -413,29 +426,29 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } // If there are still values in the buffer then flush them. - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } return nil } -// importFieldValues sends batches of fieldValues to the server. -func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error { +// importValues sends batches of FieldValues to the server. +func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group vals by slice. + // Group vals by shard. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := pilosa.FieldValues(vals).GroupBySlice() + valsByShard := http.FieldValues(vals).GroupByShard() - // Parse path into field values. - for slice, vals := range valsBySlice { + // Parse path into FieldValues. + for shard, vals := range valsByShard { if cmd.Sort { - sort.Sort(pilosa.FieldValues(vals)) + sort.Sort(http.FieldValues(vals)) } - logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(vals)) + if err := cmd.client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/ctl/import_test.go b/ctl/import_test.go index 08eea8323..56970e9be 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -15,6 +15,7 @@ package ctl import ( + "bufio" "bytes" "context" "io" @@ -38,11 +39,11 @@ func TestImportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } - cm.Frame = "f" + cm.Field = "f" err = cm.Run(context.Background()) if err.Error() != "path required" { t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err) @@ -50,7 +51,6 @@ func TestImportCommand_Validation(t *testing.T) { } func TestImportCommand_Run(t *testing.T) { - buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -61,18 +61,11 @@ func TestImportCommand_Run(t *testing.T) { t.Fatal(err) } - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() + cmd := test.MustRunCluster(t, 1)[0] + cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" cm.CreateSchema = true cm.Paths = []string{file.Name()} err = cm.Run(ctx) @@ -81,10 +74,8 @@ func TestImportCommand_Run(t *testing.T) { } } -// Ensure that the ImportValue path runs (note: we have specified a value -// for cm.Field.) +// Ensure that the ImportValue path runs. func TestImportCommand_RunValue(t *testing.T) { - buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) @@ -95,22 +86,14 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - defer s.Close() + cmd := test.MustRunCluster(t, 1)[0] + cm.Host = cmd.API.Node().URI.HostPort() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm.Host = s.Host() - - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" - cm.Frame = "f" - cm.Field = "foo" + cm.Field = "f" cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -119,13 +102,14 @@ func TestImportCommand_RunValue(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = "anyhost" + cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { @@ -175,8 +159,63 @@ func TestImportCommand_InvalidFile(t *testing.T) { // MustNewHTTPRequest creates a new HTTP request. Panic on error. func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { req, err := http.NewRequest(method, urlStr, body) + req.Header.Add("Accept", "application/json") if err != nil { panic(err) } return req } + +// declare stdin, stdout, stderr +func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { + rder := []byte{} + stdin := bytes.NewReader(rder) + stdout := bufio.NewWriter(&buf) + stderr := bufio.NewWriter(&buf) + return stdin, stdout, stderr +} + +func TestImportCommand_BugOverwriteValue(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + file, err := ioutil.TempFile("", "import-value.csv") + file.Write([]byte("0,17\n")) + ctx := context.Background() + if err != nil { + t.Fatal(err) + } + + cm.Host = cmd.API.Node().URI.HostPort() + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`))) + + cm.Index = "i" + cm.Field = "f" + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } + + file.Close() + file, err = ioutil.TempFile("", "import-value2.csv") + file.Write([]byte("0,16\n")) + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } + + file.Close() + file, err = ioutil.TempFile("", "import-value3.csv") + file.Write([]byte("0,19\n")) + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with values doesn't work: %s", err) + } +} diff --git a/ctl/server.go b/ctl/server.go index a4c272a46..ad1ac5df5 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + // Translation + flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.") + // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") diff --git a/diagnostics.go b/diagnostics.go index 9219c01ea..9649a6505 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -37,8 +37,8 @@ type versionResponse struct { Message string `json:"message"` } -// DiagnosticsCollector represents a collector/sender of diagnostics data. -type DiagnosticsCollector struct { +// diagnosticsCollector represents a collector/sender of diagnostics data. +type diagnosticsCollector struct { mu sync.Mutex host string VersionURL string @@ -56,9 +56,9 @@ type DiagnosticsCollector struct { server *Server } -// NewDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". -func NewDiagnosticsCollector(host string) *DiagnosticsCollector { - return &DiagnosticsCollector{ +// newDiagnosticsCollector returns a new DiagnosticsCollector given an addr in the format "hostname:port". +func newDiagnosticsCollector(host string) *diagnosticsCollector { + return &diagnosticsCollector{ host: host, VersionURL: defaultVersionCheckURL, startTime: time.Now().Unix(), @@ -70,13 +70,13 @@ func NewDiagnosticsCollector(host string) *DiagnosticsCollector { } // SetVersion of locally running Pilosa Cluster to check against master. -func (d *DiagnosticsCollector) SetVersion(v string) { +func (d *diagnosticsCollector) SetVersion(v string) { d.version = v d.Set("Version", v) } // Flush sends the current metrics. -func (d *DiagnosticsCollector) Flush() error { +func (d *diagnosticsCollector) Flush() error { d.mu.Lock() defer d.mu.Unlock() d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) @@ -99,7 +99,7 @@ func (d *DiagnosticsCollector) Flush() error { } // CheckVersion of the local build against Pilosa master. -func (d *DiagnosticsCollector) CheckVersion() error { +func (d *diagnosticsCollector) CheckVersion() error { var rsp versionResponse req, err := http.NewRequest("GET", d.VersionURL, nil) if err != nil { @@ -131,7 +131,7 @@ func (d *DiagnosticsCollector) CheckVersion() error { } // compareVersion check version strings. -func (d *DiagnosticsCollector) compareVersion(value string) error { +func (d *diagnosticsCollector) compareVersion(value string) error { currentVersion := versionSegments(value) localVersion := versionSegments(d.version) @@ -147,12 +147,12 @@ func (d *DiagnosticsCollector) compareVersion(value string) error { } // Encode metrics maps into the json message format. -func (d *DiagnosticsCollector) encode() ([]byte, error) { +func (d *diagnosticsCollector) encode() ([]byte, error) { return json.Marshal(d.metrics) } // Set adds a key value metric. -func (d *DiagnosticsCollector) Set(name string, value interface{}) { +func (d *diagnosticsCollector) Set(name string, value interface{}) { switch v := value.(type) { case string: if v == "" { @@ -166,7 +166,7 @@ func (d *DiagnosticsCollector) Set(name string, value interface{}) { } // logErr logs the error and returns true if an error exists -func (d *DiagnosticsCollector) logErr(err error) bool { +func (d *diagnosticsCollector) logErr(err error) bool { if err != nil { d.Logger.Printf("%v", err) return true @@ -175,7 +175,7 @@ func (d *DiagnosticsCollector) logErr(err error) bool { } // EnrichWithOSInfo adds OS information to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithOSInfo() { +func (d *diagnosticsCollector) EnrichWithOSInfo() { uptime, err := d.server.systemInfo.Uptime() if !d.logErr(err) { d.Set("HostUptime", uptime) @@ -199,7 +199,7 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() { } // EnrichWithMemoryInfo adds memory information to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { +func (d *diagnosticsCollector) EnrichWithMemoryInfo() { memFree, err := d.server.systemInfo.MemFree() if !d.logErr(err) { d.Set("MemFree", memFree) @@ -215,30 +215,30 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { } // EnrichWithSchemaProperties adds schema info to the diagnostics payload. -func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { - var numSlices uint64 - numFrames := 0 +func (d *diagnosticsCollector) EnrichWithSchemaProperties() { + var numShards uint64 + numFields := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false - for _, index := range d.server.Holder.Indexes() { - numSlices += index.MaxSlice() + 1 + for _, index := range d.server.holder.Indexes() { + numShards += index.maxShard() + 1 numIndexes += 1 - for _, frame := range index.Frames() { - numFrames += 1 - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) + for _, field := range index.Fields() { + numFields += 1 + if field.Type() == FieldTypeInt { + bsiFieldCount += 1 } - if frame.TimeQuantum() != "" { + if field.TimeQuantum() != "" { timeQuantumEnabled = true } } } d.Set("NumIndexes", numIndexes) - d.Set("NumFrames", numFrames) - d.Set("NumSlices", numSlices) + d.Set("NumFields", numFields) + d.Set("NumShards", numShards) d.Set("BSIFieldCount", bsiFieldCount) d.Set("TimeQuantumEnabled", timeQuantumEnabled) } @@ -267,51 +267,51 @@ type SystemInfo interface { MemUsed() (uint64, error) } -// NewNopSystemInfo creates a no-op implementation of SystemInfo. -func NewNopSystemInfo() *NopSystemInfo { - return &NopSystemInfo{} +// newNopSystemInfo creates a no-op implementation of SystemInfo. +func newNopSystemInfo() *nopSystemInfo { + return &nopSystemInfo{} } -// NopSystemInfo is a no-op implementation of SystemInfo. -type NopSystemInfo struct { +// nopSystemInfo is a no-op implementation of SystemInfo. +type nopSystemInfo struct { } // Uptime is a no-op implementation of SystemInfo.Uptime. -func (n *NopSystemInfo) Uptime() (uint64, error) { +func (n *nopSystemInfo) Uptime() (uint64, error) { return 0, nil } // Platform is a no-op implementation of SystemInfo.Platform. -func (n *NopSystemInfo) Platform() (string, error) { +func (n *nopSystemInfo) Platform() (string, error) { return "", nil } // Family is a no-op implementation of SystemInfo.Family. -func (n *NopSystemInfo) Family() (string, error) { +func (n *nopSystemInfo) Family() (string, error) { return "", nil } // OSVersion is a no-op implementation of SystemInfo.OSVersion. -func (n *NopSystemInfo) OSVersion() (string, error) { +func (n *nopSystemInfo) OSVersion() (string, error) { return "", nil } // KernelVersion is a no-op implementation of SystemInfo.KernelVersion. -func (n *NopSystemInfo) KernelVersion() (string, error) { +func (n *nopSystemInfo) KernelVersion() (string, error) { return "", nil } // MemFree is a no-op implementation of SystemInfo.MemFree. -func (n *NopSystemInfo) MemFree() (uint64, error) { +func (n *nopSystemInfo) MemFree() (uint64, error) { return 0, nil } // MemTotal is a no-op implementation of SystemInfo.MemTotal. -func (n *NopSystemInfo) MemTotal() (uint64, error) { +func (n *nopSystemInfo) MemTotal() (uint64, error) { return 0, nil } // MemUsed is a no-op implementation of SystemInfo.MemUsed. -func (n *NopSystemInfo) MemUsed() (uint64, error) { +func (n *nopSystemInfo) MemUsed() (uint64, error) { return 0, nil } diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index 517dbed3d..f1536e1d1 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -29,7 +29,7 @@ func TestDiagnosticsClient(t *testing.T) { server := httptest.NewServer(nil) // Create a new client. - d := NewDiagnosticsCollector(server.URL) + d := newDiagnosticsCollector(server.URL) d.Set("gg", 10) d.Set("ss", "ss") @@ -76,7 +76,7 @@ func TestDiagnosticsVersion_Parse(t *testing.T) { } func TestDiagnosticsVersion_Compare(t *testing.T) { - d := NewDiagnosticsCollector("localhost:10101") + d := newDiagnosticsCollector("localhost:10101") version := "v0.1.1" d.SetVersion(version) @@ -118,7 +118,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) { })) // Create a new client. - d := NewDiagnosticsCollector("localhost:10101") + d := newDiagnosticsCollector("localhost:10101") version := "0.1.1" d.SetVersion(version) @@ -143,7 +143,7 @@ func BenchmarkDiagnostics(b *testing.B) { server := httptest.NewServer(nil) // Create a new client. - d := NewDiagnosticsCollector(server.URL) + d := newDiagnosticsCollector(server.URL) prev := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(prev) diff --git a/docs/administration.md b/docs/administration.md index dc2e923f2..71ab54672 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -24,19 +24,19 @@ Pilosa holds all row/column bitmap data in main memory. While this data is compr #### CPUs -Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [slice](../data-model/#slice), so a single query will only use a number of cores up to the number of slices stored on that host. Multiple queries can still take advantage of multiple cores as well though, so tuning in this area is dependent on the expected workload. +Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload. #### Disk -Even though the main dataset is in memory Pilosa does back up to disk frequently. We recommend SSDs—especially if you have a write heavy application. +Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application. #### Network -Pilosa is designed to be a distributed application, with data replication shared across the cluster. As such every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all node exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions it not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there already should be a system of record, or ability to rebuild a cluster quickly from backups. +Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups. #### Overview -While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. +While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. ### Open File Limits @@ -56,23 +56,23 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -##### Importing Field Values +##### Importing Integer Values -If you are using [BSI Range-Encoding](../data-model/#bsi-range-encoding) field values, you can import field values for a single frame and single field using `--field`. The CSV file should be in the format `Column,Value`. +If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`. ``` -pilosa import -i project -f stargazer --field star_count project-stargazer-counts.csv +pilosa import -i project -f stargazer-counts project-stargazer-counts.csv ```
-

Note that you must first create a frame and a field. View Create Frame for more details.

+

Note that you must first create a field. View Create Field for more details. The `-e` flag can create the necessary schema when using a field of type "set".

#### Exporting -Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the frame. The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format `Row,Column` and sorted by column. +Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column. ```request -curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0" \ +curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \ --header "Accept: text/csv" ``` ```response @@ -122,7 +122,7 @@ Pilosa v0.9 introduces a few compatibility changes that need to be addressed. Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called [`topology-generator`](https://github.com/pilosa/upgrade-utils/tree/master/v0.9/topology-generator): -1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. +1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. 2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`. 3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`. 4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position. @@ -132,8 +132,8 @@ Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topo **Application changes**: 1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported. -2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed. -3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release. +2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-field, as index-level time-quantums have been removed. +3. Inverse fields have been deprecated, removed from docs, and will be unsupported in the next release. ### Resizing the Cluster @@ -211,7 +211,7 @@ curl localhost:10101/cluster/resize/set-coordinator \ ### Backup/restore -Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Frame->Views->Fragment->numbered slice files. These data files can be routinely backed up to restore nodes in a cluster. +Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster. Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node. @@ -230,12 +230,12 @@ Note: This will only work when the replication factor is >= 2 - To accomplish this you will first need: - List of all indexes on your cluster - - List of all frames in your indexes - - Max slice per index, listed in the `/slices/max` endpoint -- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice -- Using the list of slices owned by this node you will then need to manually: - - setup a directory structure similar to the other nodes with a path for each Index/Frame - - copy each owned slice for an existing node to this new node + - List of all fields in your indexes + - Max shard per index, listed in the `/internal/shards/max` endpoint +- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard +- Using the list of shards owned by this node you will then need to manually: + - setup a directory structure similar to the other nodes with a path for each Index/Field + - copy each owned shard for an existing node to this new node - Modify the cluster config file to replace the previous node address with the new node address. - Restart the cluster - Wait for the first sync (10 minutes) to validate Index connections @@ -249,11 +249,11 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **Cluster:** List of nodes in the cluster. - **NumNodes:** Number of nodes in the cluster. - **NumCPU:** Number of cores per node -- **BSIEnabled:** Bit Slice Index Frames in use. -- **TimeQuantumEnabled:** Time Quantum Frames in use. +- **BSIEnabled:** Bit Sliced Index Fields in use. +- **TimeQuantumEnabled:** Time Quantum Fields in use. - **NumIndexes:** Number of indexes in the Cluster. -- **NumFrames:** Number of frames in the Cluster. -- **NumSlices:** Number of slices in the Cluster. +- **NumFields:** Number of fields in the Cluster. +- **NumShards:** Number of shards in the Cluster. - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. - **GoRoutines:** Go routine count. @@ -274,16 +274,16 @@ StatsD Tags adhere to the DataDog format (key:value), and we tag the following: - NodeID - Index -- Frame +- Field - View -- Slice +- Shard #### Events We currently track the following events - **Index:** The creation of a new index. -- **Frame:** The creation of a new frame. -- **MaxSlice:** The creation of a new Slice. +- **Field:** The creation of a new field. +- **MaxShard:** The creation of a new Shard. - **SetBit:** Count of set bits. - **ClearBit:** Count of cleared bits. - **ImportBit:** During a bulk data import this represents the count of bits created. diff --git a/docs/api-reference.md b/docs/api-reference.md index f94cc5009..c3a0144df 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -17,7 +17,7 @@ Returns the schema of all indexes in JSON. curl -XGET localhost:10101/index ``` ``` response -{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} +{"indexes":[{"name":"user","fields":[{"name":"collab"}]}]} ``` ### List index schema @@ -30,7 +30,7 @@ Returns the schema of the specified index in JSON. curl -XGET localhost:10101/index/user ``` ``` response -{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} +{"name":"user", "fields":[{"name":"collab"}]} ``` ### Create index @@ -43,7 +43,7 @@ Creates an index with the given name. curl -XPOST localhost:10101/index/user ``` ``` response -{} +{"success":true} ``` ### Remove index @@ -56,7 +56,7 @@ Removes the given index. curl -XDELETE localhost:10101/index/user ``` ``` response -{} +{"success":true} ``` ### Query index @@ -68,102 +68,81 @@ Sends a [query](../query-language/) to the Pilosa server with the given index. T ``` request curl localhost:10101/index/user/query \ -X POST \ - -d 'Bitmap(frame="language", row=5)' + -d 'Row(language=5)' ``` ``` response -{"results":[{"attrs":{},"bits":[100]}]} +{"results":[{"attrs":{},"columns":[100]}]} ``` In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`. The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`. -The query is executed for all [slices](../data-model/#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. +The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices. ``` request -curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ +curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \ -X POST \ - -d 'Bitmap(frame="language", row=5)' + -d 'Row(language=5)' ``` ``` response { - "results":[{"attrs":{},"bits":[100]}], + "results":[{"attrs":{},"columns":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] } ``` -By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. +By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. -### Create frame +### Create field -`POST /index//frame/` +`POST /index//field/` -Creates a frame in the given index with the given name. +Creates a field in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: -* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. +* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field. +* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: * `name` (string): Field name. -* `type` (string): Field type, currently only "int" is supported. +* `type` (string): Field type, "set", "int" or "time". * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. ``` request -curl localhost:10101/index/user/frame/language -X POST +curl localhost:10101/index/user/field/language -X POST ``` ``` response -{} +{"success":true} ``` ``` request -curl localhost:10101/index/repository/frame/stats \ +curl localhost:10101/index/repository/field/stats \ -X POST \ -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` ``` response -{} +{"success":true} ``` -### Remove frame +### Remove field -`DELETE /index//frame/` +`DELETE /index//field/` -Removes the given frame. +Removes the given field. ``` request -curl -XDELETE localhost:10101/index/user/frame/language +curl -XDELETE localhost:10101/index/user/field/language ``` ``` response -{} -``` - -### Create Field - -`POST /index//frame//field/` - -Creates a new field to store integer values in the given frame. - -The request payload is JSON, and it must contain the fields `type`, `min`, `max`. - -* `type` (string): Field type, currently only "int" is supported. -* `min` (int): Minimum value allowed for this field. -* `max` (int): Maximum value allowed for this field. - -``` request -curl localhost:10101/index/repository/frame/stats/field/pullrequests \ - -X POST \ - -d '{"type": "int", "min": 0, "max": 1000000}' -``` -``` response -{} +{"success":true} ``` ### Get version @@ -191,7 +170,7 @@ in a multi-node cluster, the cache is only recalculated on the node that receives the request. ``` request -curl -XGET localhost:10101/recalculate-caches +curl -XPOST localhost:10101/recalculate-caches ``` Response: `204 No Content` diff --git a/docs/client-libraries.md b/docs/client-libraries.md index f445139b0..a277141ec 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -46,16 +46,16 @@ func main() { panic(err) } - // We need to refer to indexes and frames before we can use them in a query. + // We need to refer to indexes and fields before we can use them in a query. repository, _ := schema.Index("repository") - stargazer, _ := repository.Frame("stargazer") - language, _ := repository.Frame("language") + stargazer, _ := repository.Field("stargazer") + language, _ := repository.Field("language") var response *pilosa.QueryResponse // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Bitmap(14)) - fmt.Println("User 14 starred: ", response.Result().Bitmap().Bits) + response, _ = client.Query(stargazer.Row(14)) + fmt.Println("User 14 starred: ", response.Result().Row().Columns) // What are the top 5 languages in the sample data? response, err = client.Query(language.TopN(5)) @@ -68,29 +68,29 @@ func main() { // Which repositories were starred by both user 14 and 19: response, _ = client.Query( repository.Intersect( - stargazer.Bitmap(14), - stargazer.Bitmap(19))) - fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap().Bits) + stargazer.Row(14), + stargazer.Row(19))) + fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns) // Which repositories were starred by user 14 or 19: response, _ = client.Query( repository.Union( - stargazer.Bitmap(14), - stargazer.Bitmap(19))) - fmt.Println("User 14 or 19 starred:", response.Result().Bitmap().Bits) + stargazer.Row(14), + stargazer.Row(19))) + fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns) // Which repositories were starred by user 14 or 19 and were written in language 1: response, _ = client.Query( repository.Intersect( repository.Union( - stargazer.Bitmap(14), - stargazer.Bitmap(19), + stargazer.Row(14), + stargazer.Row(19), ), - language.Bitmap(1))) - fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap().Bits) + language.Row(1))) + fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.SetBit(99999, 77777)) + client.Query(stargazer.Set(99999, 77777)) } ``` @@ -112,6 +112,7 @@ We are going to use the index you have created in the [Getting Started](../getti Error handling has been omitted in the example below for brevity. ```python +from __future__ import print_function from pilosa import Index, Client, PilosaError, TimeQuantum # We will just use the default client which assumes the server is at http://localhost:10101 @@ -122,8 +123,8 @@ client = Client() # and the stargazer data should be imported. # See the Getting Started repository: https://github.com/pilosa/getting-started/ -# Let's create Index and Frame objects, which will contain the settings -# for the corresponding indexes and frames. +# Let's create Index and Field objects, which will contain the settings +# for the corresponding indexes and fields. try: schema = client.schema() except PilosaError as e: @@ -132,13 +133,13 @@ except PilosaError as e: # We will just terminate the program in this case. raise SystemExit(e) -# We need to refer to indexes and frames before we can use them in a query. +# We need to refer to indexes and fields before we can use them in a query. repository = schema.index("repository") -stargazer = repository.frame("stargazer") -language = repository.frame("language") +stargazer = repository.field("stargazer") +language = repository.field("language") # Which repositories did user 8 star: -repository_ids = client.query(stargazer.bitmap(14)).result.bitmap.bits +repository_ids = client.query(stargazer.row(14)).result.row.columns print("User 8 starred: ", repository_ids) # What are the top 5 languages in the sample data: @@ -147,33 +148,33 @@ print("Top 5 languages: ", [item.id for item in top_languages]) # Which repositories were starred by both user 14 and 19: query = repository.intersect( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ) -mutually_starred = client.query(query).result.bitmap.bits +mutually_starred = client.query(query).result.row.columns print("Both user 14 and 19 starred:", mutually_starred) # Which repositories were starred by user 14 or 19: query = repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ) -either_starred = client.query(query).result.bitmap.bits +either_starred = client.query(query).result.row.columns print("User 14 or 19 starred:", either_starred) # Which repositories were starred by user 14 or 19 and were written in language 1: query = repository.intersect( repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ), - language.bitmap(1) + language.row(1) ) -mutually_starred = client.query(query).result.bitmap.bits +mutually_starred = client.query(query).result.row.columns print("User 14 or 19 starred, written in language 1:", mutually_starred) # Set user 99999 as a stargazer for repository 77777 -client.query(stargazer.setbit(99999, 77777)) +client.query(stargazer.set(99999, 77777)) ``` Running the above program should produce output like this: @@ -218,10 +219,10 @@ public class StarTrace { throw new RuntimeException(ex); } - // We need to refer to indexes and frames before we can use them in a query. + // We need to refer to indexes and fields before we can use them in a query. Index repository = schema.index("repository"); - Frame stargazer = repository.frame("stargazer"); - Frame language = repository.frame("language"); + Field stargazer = repository.field("stargazer"); + Field language = repository.field("language"); QueryResponse response; QueryResult result; @@ -229,8 +230,8 @@ public class StarTrace { List repositoryIDs; // Which repositories did user 14 star: - response = client.query(stargazer.bitmap(14)); - repositoryIDs = response.getResult().getBitmap().getBits(); + response = client.query(stargazer.row(14)); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 starred: " + repositoryIDs); // What are the top 5 languages in the sample data: @@ -245,36 +246,36 @@ public class StarTrace { // Which repositories were starred by both user 14 and 19: query = repository.intersect( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("Both user 14 and 19 starred: " + repositoryIDs); // Which repositories were starred by user 14 or 19: query = repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 or 19 starred: " + repositoryIDs); // Which repositories were starred by user 14 or 19 and were written in language 1: query = repository.intersect( repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ), - language.bitmap(1) + language.row(1) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); // Set user 99999 as a stargazer for repository 77777: - client.query(stargazer.setBit(99999, 77777)); + client.query(stargazer.set(99999, 77777)); } } ``` diff --git a/docs/configuration.md b/docs/configuration.md index 200c91449..22a67ddcf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,7 +106,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Max Writes Per Request -* Description: Maximum number of mutating commands allowed per request. This includes SetBit, ClearBit, SetRowAttrs, SetColumnAttrs, and SetFieldValue. +* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs. * Flag: `--max-writes-per-request=5000` * Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000` * Config: diff --git a/docs/data-model.md b/docs/data-model.md index 9569b1f90..e453d9245 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -6,10 +6,10 @@ nav = [ "Index", "Column", "Row", - "Frame", + "Field", "Time Quantum", "Attribute", - "Slice", + "Shard", "View", ] +++ @@ -22,7 +22,7 @@ The central component of Pilosa's data model is a boolean matrix. Each cell in t Rows and columns can represent anything (they could even represent the same set of things - a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix. -Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *frames* and quickly retrieves the top rows in a frame sorted by the number of bits set in each row. +Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation on multiple rows such as Intersect or Union are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row. Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 263 on a single-node cluster, for example, will not work well due to memory limitations. @@ -35,20 +35,22 @@ The purpose of the Index is to represent a data namespace. You cannot perform cr ### Column -Column ids are sequential increasing integers and are common to all Frames within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable. +Column ids are sequential increasing integers and are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable. ### Row -Row ids are sequential increasing integers namespaced to each Frame within an Index. +Row ids are sequential increasing integers namespaced to each Field within an Index. -### Frame +### Field -Frames are used to segment rows within an index, for example to define different functional groups. A frame might correspond to a single field in a relational table, where each row in a standard frame represents a single possible value of the field. Similarly, a frame with BSI values could represent all possible integer values of a field . +Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field. #### Relational Analogy The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here. +TODO diagram showing a few rows of a relational table and corresponding pilosa index + Entities: Relational | Pilosa @@ -56,19 +58,19 @@ Entities: Database | N/A *(internal: Holder)* Table | Index Row | Column - Column | Frame + Column | Field Value | Row Value (int) | Field.Value (see [BSI](#bsi-range-encoding)) Simple queries: - Relational | Pilosa ----------------------------------------------|------------------------------------ - `select ID from People where Name = 'Bob'` | `Bitmap(frame=Name, row=[Bob])` - `select ID from People where Age > 30` | `Range(frame=Default, Age > 30)` - `select ID from People where Member = true` | `Bitmap(frame=Member, row=[true])` + Relational | Pilosa +-----------------------------------------------|------------------------------------ + `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` + `select ID from People where Age > 30` | `Range(Age > 30)` + `select ID from People where Member = true` | `Row(Member=0)` -In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple frames. For example, this SQL join: +Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: ```sql select AVG(p.Age) from People p @@ -80,44 +82,44 @@ where c.Make = 'Ford' can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed): ```pql -Sum(Bitmap(frame="Car-Make", row=[Ford]), frame=Default, field=Age) +Sum(Row(Car-Make="Ford"), field=Age) ``` This is one major component of Pilosa's ability to combine relationships from multiple data stores. #### Ranked -Ranked Frames maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Frame creation. +Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation. -![ranked frame diagram](/img/docs/frame-ranked.svg) -*Ranked frame diagram* +![ranked field diagram](/img/docs/field-ranked.svg) +*Ranked field diagram* #### LRU The LRU cache maintains the most recently accessed Rows. -![lru frame diagram](/img/docs/frame-lru.svg) -*LRU frame diagram* +![lru field diagram](/img/docs/field-lru.svg) +*LRU field diagram* ### Time Quantum -Setting a time quantum on a frame creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example - if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. ### Attribute Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure. -Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all frames in an index. Row attributes apply to all bits in the corresponding row. +Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row. -### Slice +### Shard -Indexes are sharded into groups of columns called Slices. Each Slice contains a fixed number of columns, which is the SliceWidth. SliceWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 220. +Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 220. Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm. ### View -Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. +Views represent the various data layouts within a Field. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. #### Standard @@ -125,34 +127,34 @@ The standard View contains the same Row/Column format as the input data. #### Time Quantums -If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the diagram below: +If a Field has a time quantum, then Views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below: ``` -SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") -SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") +Set(3, A=8, 2017-05-18T00:00) +Set(3, A=8, 2017-05-19T00:00) ``` -![time quantum frame diagram](/img/docs/frame-time-quantum.svg) -*Time quantum frame diagram* +![time quantum field diagram](/img/docs/field-time-quantum.svg) +*Time quantum fueld diagram* #### BSI Range-Encoding -Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. +Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. -For example, the following `SetFieldValue()` queries will result in the data described in the diagram below: +For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below: ``` -SetFieldValue(col=1, frame="A", field0=1) -SetFieldValue(col=2, frame="A", field0=2) -SetFieldValue(col=3, frame="A", field0=3) -SetFieldValue(col=4, frame="A", field0=7) -SetFieldValue(col=2, frame="A", field1=1) -SetFieldValue(col=3, frame="A", field1=6) +Set(1, A=1) +Set(2, A=2) +Set(3, A=3) +Set(4, A=7) +Set(2, B=1) +Set(3, B=6) ``` -![BSI frame diagram](/img/docs/frame-bsi.svg) -*BSI frame diagram* +![BSI field diagram](/img/docs/field-bsi.svg) +*BSI field diagram* Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa. diff --git a/docs/examples.md b/docs/examples.md index 6c4f70cc1..692cd7e39 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -3,7 +3,6 @@ title = "Examples" weight = 4 nav = [ "Transportation", - "Chemical similarity search", ] +++ @@ -33,9 +32,9 @@ The NYC taxi data is comprised of a number of csv files listed here: http://www. * Dropoff time: timestamp * Pickup time: timestamp -We import these fields, creating one or more Pilosa frames from each of them: +We import these fields, creating one or more Pilosa fields from each of them: -frame |mapping +field |mapping ------------|--------------------- cab_type |direct map of enum int → row ID dist_miles |round(dist) → row ID @@ -52,24 +51,24 @@ pickup_month |month(timestamp) → row ID pickup_day |day(timestamp) → row ID pickup_time |time of day mapped to one of 48 half-hour buckets → row ID -We also created two extra frames that represent the duration and average speed of each ride: +We also created two extra fields that represent the duration and average speed of each ride: -frame |mapping +field |mapping --------------------|------------- duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID #### Mapping -Each column that we want to use must be mapped to a combination of frames and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. +Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities. -##### 0 columns → 1 frame +##### 0 columns → 1 field -**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this frame. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this frame are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type frame is constant. +**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant. -##### 1 column → 1 frame +##### 1 column → 1 field -The following three frames are mapped in a simple direct way from single columns of the original data. +The following three fields are mapped in a simple direct way from single columns of the original data. **dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times. @@ -84,7 +83,7 @@ lfm := pdk.LinearFloatMapper{ `Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. -This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the frame to use (`Frame`). +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`). TODO update so this makes sense ```go pdk.BitMapper{ Frame: "dist_miles", @@ -129,27 +128,27 @@ Here, we define a list of Mappers, each including a name, which we use to refer **passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. -##### 1 column → multiple frames +##### 1 column → multiple fields When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. -We do this by storing time data in four separate frames for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of frame "year", row 6 of frame "month", and row 24 of frame "day". +We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day". -We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of frame "time_of_day". +We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day". -We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total frames for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. +We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time. -##### Multiple columns → 1 frame +##### Multiple columns → 1 field The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID. -We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two frames for two locations: pickup_grid_id, drop_grid_id. +We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id. Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient. ##### Complex mappings -We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the frame `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the frame `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: +We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work: ```go durm := pdk.CustomMapper{ Func: func(fields ...interface{}) interface{} { @@ -172,7 +171,7 @@ Now we can run some example queries. Count per cab type can be retrieved, sorted, with a single PQL call. ```request -TopN(frame=cab_type) +TopN(cab_type) ``` ```response {"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]} @@ -181,7 +180,7 @@ TopN(frame=cab_type) High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs. ```request -TopN(frame=pickup_grid_id) +TopN(pickup_grid_id) ``` ```response {"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]} @@ -193,7 +192,7 @@ Average of `total_amount` per `passenger_count` can be computed with some postpr queries = '' pcounts = range(10) for i in pcounts: - queries += "TopN(Bitmap(id=%d, frame='passenger_count'), frame=total_amount_dollars)" % i + queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i resp = requests.post(qurl, data=queries) average_amounts = [] @@ -209,6 +208,8 @@ Note that the BSI-powered @@ -326,3 +327,6 @@ python benchmarks.py -id 6223 As Matt Swain’s blog post also did a great job using mongoDB for chemical similarity search, we compared benchmark on 500000 molecules between mongoDB aggregation framework with Pilosa. Both using the same molecule, Morgan fingerprint folded to fixed lengths of 4096 bits and were run on a MacBook Pro with a 2.8 GHz 2-core Intel Core i7 processor, memory of 16 GB 1600 MHz DDR3, single host cluster + + +--> diff --git a/docs/getting-started.md b/docs/getting-started.md index c5383e81c..6f935f735 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -34,19 +34,20 @@ Let's make sure Pilosa is running: curl localhost:10101/status ``` ``` response -{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]} +{"state":"NORMAL","nodes":[{"id":"91715a50-7d50-4c54-9a03-873801da1cd1","uri":{"scheme":"http","host":"localhost","port +":10101},"isCoordinator":true}],"localID":"91715a50-7d50-4c54-9a03-873801da1cd1"} ``` ### Sample Project In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about 1,000 popular Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. -Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation. +Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Fields. So the "repository" index might have a "languages" field as well as a "tags" field. You can learn more about indexes and fields in the [Data Model](../data-model/) section of the documentation. #### Create the Schema Note: -The queries in this section which are used to set up the indexes in Pilosa just return the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: +If at any time you want to verify the data structure, you can request the schema as follows: ``` request curl localhost:10101/schema @@ -55,35 +56,37 @@ curl localhost:10101/schema {"indexes":null} ``` -Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: +Before we can import data or run queries, we need to create our indexes and the fields within them. Let's create the repository index first: ``` request curl localhost:10101/index/repository -X POST ``` ``` response -{} +{"success":true} ``` -Let's create the `stargazer` frame which has user IDs of stargazers as its rows: +Let's create the `stargazer` field which has user IDs of stargazers as its rows: ``` request -curl localhost:10101/index/repository/frame/stargazer \ +curl localhost:10101/index/repository/field/stargazer \ -X POST \ - -d '{"options": {"timeQuantum": "YMD"}}' + -d '{"options": {"type": "time", "timeQuantum": "YMD"}}' ``` ``` response -{} +{"success":true} ``` -Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. +Since our data contains time stamps whcih represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. -Next up is the `language` frame, which will contain IDs for programming languages: +Next up is the `language` field, which will contain IDs for programming languages: ``` request -curl localhost:10101/index/repository/frame/language \ +curl localhost:10101/index/repository/field/language \ -X POST ``` ``` response -{} +{"success":true} ``` +The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options. + #### Import Data From CSV Files Download the `stargazer.csv` and `language.csv` files here: @@ -112,22 +115,18 @@ Note that both the user IDs and the repository IDs were remapped to sequential i #### Make Some Queries -
-

Note the Pilosa server comes with a WebUI for constructing queries in a browser. In local development, it is available at localhost:10101.

-
- Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(frame="stargazer", row=14)' + -d 'Row(stargazer=14)' ``` ``` response { "results":[ { "attrs":{}, - "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + "columns":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] } ] } @@ -137,7 +136,7 @@ What are the top 5 languages in the sample data: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'TopN(frame="language", n=5)' + -d 'TopN(language, n=5)' ``` ``` response { @@ -158,8 +157,8 @@ Which repositories were starred by user 14 and 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19) + Row(stargazer=14), + Row(stargazer=19) )' ``` ``` response @@ -167,7 +166,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[2,3,362,396,416,461,464,466,470,486] + "columns":[2,3,362,396,416,461,464,466,470,486] } ] } @@ -178,8 +177,8 @@ Which repositories were starred by user 14 or 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Union( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19) + Row(stargazer=14), + Row(stargazer=19) )' ``` ``` response @@ -187,7 +186,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + "columns":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] } ] } @@ -198,9 +197,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19), - Bitmap(frame="language", row=1) + Row(stargazer=14), + Row(stargazer=19), + Row(language=1) )' ``` ``` response @@ -208,7 +207,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[2,362,416,461] + "columns":[2,362,416,461] } ] } @@ -218,7 +217,7 @@ Set user 99999 as a stargazer for repository 77777: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", col=77777, row=99999)' + -d 'Set(77777, stargazer=99999)' ``` ``` response {"results":[true]} diff --git a/docs/glossary.md b/docs/glossary.md index fb1a46690..81323674d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -6,25 +6,25 @@ nav = [] ## Glossary -[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [slice](#slice) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. +[Anti-entropy](../configuration/#anti-entropy-interval): A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies. [Attribute](../data-model/#attribute): Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store. -[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [frame](#frame), at the intersection of a [row](#row) and [column](#column). +[Bit](../data-model/#overview): Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column). -[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. +[Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). [BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. -Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. +Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. -[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [frames](#frame) within an [index](#index). +[Column](../data-model/#column): Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index). -[Field](../data-model/#bsi-range-encoding): A group of rows used to store integer values with [BSI](#bsi), for use in [Range](#range-bsi) and [Sum](#sum) queries. +Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of three types: set, [int](#bsi), and time. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). -[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. +[Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. [Gossip](https://en.wikipedia.org/wiki/Gossip_protocol): A protocol used by Pilosa for internal communication. @@ -32,11 +32,11 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. -[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field). -MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +MaxShard: The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5. -[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field). Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). @@ -54,20 +54,20 @@ nav = [] [Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. -[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [frame](#frame) within an [index](#index). Represented as a [Bitmap](#bitmap). +[Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap). -[Slice](../data-model/#slice): [Columns](#column) are sharded on a preset [width](#slicewidth). Each shard is referred to as a slice in Pilosa. Slices are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). +[Slice](../data-model/#shard): Prior to Pilosa 1.0, shards were known as slices. -SliceWidth: This is the number of [columns](#column) in a [slice](#slice). `SliceWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. +[Shard](../data-model/#shard): [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). -[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in [BSI](#bsi) [fields](#field). +ShardWidth: This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 220 or about one million. It can be modified, but only at compile time, and before ingesting any data. -[Tanimoto](../examples/#chemical-similarity-search): Used for similarity queries on Pilosa data. The [Tanimoto Coefficient](https://en.wikipedia.org/wiki/Jaccard_index#Tanimoto_similarity_and_distance) between two [Bitmaps](#bitmap) A and B is the ratio of the size of their intersection to the size of their union (|A∩B|/|A∪B|). +[Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field). -[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for time [Range](#range) queries. +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [Range](#range) queries on time [fields](#field). [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). -[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field). -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/installation.md b/docs/installation.md index 039ea370d..639293338 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -144,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` @@ -173,7 +170,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -262,7 +258,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -298,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` @@ -327,7 +321,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. diff --git a/docs/pdk.md b/docs/pdk.md index 2ca4b6ee8..dd6ddaf61 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -18,7 +18,7 @@ Running `pdk -h` will give the most up to date list of all the tools and example `pdk kafka` reads either JSON or Avro encoded records from Kafka (using the Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each record from Kafka is assigned a Pilosa column, and each value in a record is -assigned a row or field. Frame and field names are built from the "path" through +assigned a row or field. Pilosa field names are built from the "path" through the record to arrive at that field. For example: ```json @@ -38,30 +38,30 @@ the record to arrive at that field. For example: This JSON object would result in the following Pilosa schema: -| Name | Field | Type | Min | Max | Size | -|----------------|-----------|--------|-----|------------|--------| -| name | | ranked | | | 100000 | -| favorite_foods | | ranked | | | 100000 | -| default | | ranked | | | 100000 | -| | age | int | 0 | 2147483647 | | -| location | | ranked | | | 1000 | -| | latitude | int | 0 | 2147483647 | | -| | longitude | int | 0 | 2147483647 | | -| location-city | | ranked | | | 100000 | -| location-state | | ranked | | | 100000 | +| Field | Type | Min | Max | Size | +|----------------|--------|-----|------------|--------| +| name | ranked | | | 100000 | +| favorite_foods | ranked | | | 100000 | +| default | ranked | | | 100000 | +| age | int | 0 | 2147483647 | | +| location | ranked | | | 1000 | +| latitude | int | 0 | 2147483647 | | +| longitude | int | 0 | 2147483647 | | +| location-city | ranked | | | 100000 | +| location-state | ranked | | | 100000 | -All frames are created as ranked frames by default, with the cache size listed above. Fields are created with -a minimum size of zero and a fixed maximum of 2147483647. Fields at the top level -are created in the default frame. Frames are a dash-separated concatenation of -all key values in the path - you can see this with frames like location-city. +All set fields are created as ranked fields by default, with the cache size +listed above. Integer fields are created with a minimum size of zero and a +fixed maximum of 2147483647. Field names are a dash-separated concatenation of +all key values in the path - you can see this with fields like location-city. Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts, kafka topics, kafka group, etc.), but there are a few options that give some control over the way data is indexed, and ingestion performance. -* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. -* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state". +* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. +* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state". * `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it. * `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used. * `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb. diff --git a/docs/query-language.md b/docs/query-language.md index a598bf351..2ea85a06e 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -1,4 +1,4 @@ -+++ +v+++ title = "Query Language" weight = 6 nav = [ @@ -29,13 +29,13 @@ There will be one item in the `results` array for each PQL query in the request. ##### Examples -Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. +Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index and fields, and to populate them with some data. -The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would: +The examples just show the PQL quer(ies) needed - to run the query `Set(10, stargazer=1)` against a server using curl, you would: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", col=10, row=1)' + -d 'Set(10, stargazer=1)' ``` ``` response {"results":[true]} @@ -43,28 +43,27 @@ curl localhost:10101/index/repository/query \ #### Arguments and Types -* `frame` The frame specifies on which Pilosa [frame](../glossary/#frame) the query will operate. Valid frame names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. -* `TIMESTAMP` This is a timestamp in quotes with the following format `"YYYY-MM-DDTHH:MM"` (e.g. "2006-01-02T15:04") +* `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. +* `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04) * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations -#### SetBit +#### Set **Spec:** ``` -SetBit(, , , - [timestamp=TIMESTAMP]) +Set(, =, [TIMESTAMP]) ``` **Description:** -`SetBit` assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given frame with the given column. +`Set` assigns a value of 1 to a bit in the binary matrix, thus associating the given row (the `` value) in the given field with the given column. **Result Type:** boolean @@ -77,17 +76,17 @@ A return value of `false` indicates that the bit was already set to 1 and nothin Set the bit at row 1, column 10: ```request -SetBit(frame="stargazer", col=10, row=1) +Set(10, stargazer=1) ``` ```response {"results":[true]} ``` -This sets a bit in the stargazer frame, representing that the user with id=1 has starred the repository with id=10. +This sets a bit in the stargazer field, representing that the user with id=1 has starred the repository with id=10. -SetBit also supports providing a timestamp. To write the date that a user starred a repository: +Set also supports providing a timestamp. To write the date that a user starred a repository: ```request -SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") +Set(10, stargazer=1, 2016-01-01T00:00) ``` ```response {"results":[true]} @@ -95,24 +94,32 @@ SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") Set multiple bits in a single request: ```request -SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2) +Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30) ``` ```response {"results":[false,true,true,true]} ``` +Set the field "pullrequests" to integer value 2 at column 10: +```request +Set(10, pullrequests=2) +``` +```response +{"results":[true]} +``` + #### SetRowAttrs **Spec:** ``` -SetRowAttrs(, , +SetRowAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` **Description:** -`SetRowAttrs` associates arbitrary key/value pairs with a row in a frame. Setting a value of `null`, without quotes, deletes an attribute. +`SetRowAttrs` associates arbitrary key/value pairs with a row in a field. Setting a value of `null`, without quotes, deletes an attribute. **Result Type:** null @@ -122,17 +129,17 @@ SetRowAttrs queries always return `null` upon success. Set attributes `username` and `active` on row 10: ```request -SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true) +SetRowAttrs(stargazer, 10, username="mrpi", active=true) ``` ```response {"results":[null]} ``` -Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", row=10)`. +Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Row](../query-language/#row) query like so `Row(stargazer=10)`. Delete attribute `username` on row 10: ```request -SetRowAttrs(frame="stargazer", row=10, username=null) +SetRowAttrs(stargazer, 10, username=null) ``` ```response {"results":[null]} @@ -143,7 +150,7 @@ SetRowAttrs(frame="stargazer", row=10, username=null) **Spec:** ``` -SetColumnAttrs(, , +SetColumnAttrs(, , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -154,13 +161,13 @@ SetColumnAttrs(, , **Result Type:** null -SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. To avoid confusion, `frame` cannot be used as an attribute name. +SetColumnAttrs queries always return `null` upon success. Setting a value of `null`, without quotes, deletes an attribute. **Examples:** Set attributes `stars`, `url`, and `active` on column 10: ```request -SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true) +SetColumnAttrs(10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` ```response {"results":[null]} @@ -170,13 +177,13 @@ Set url value and active status for project 10. These are arbitrary key/value pa ColumnAttrs can be requested by adding the URL parameter `columnAttrs=true` to a query. For example: ```request -curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Bitmap(frame="stargazer", row=1)Bitmap(frame="stargazer", row=2)' +curl localhost:10101/index/repository/query?columnAttrs=true -XPOST -d 'Row(stargazer=1) Row(stargazer=2)' ``` ```response { "results":[ - {"attrs":{},"bits":[10,20]}, - {"attrs":{},"bits":[10,30]} + {"attrs":{},"cols":[10,20]}, + {"attrs":{},"cols":[10,30]} ], "columnAttrs":[ {"id":10,"attrs":{"active":true,"stars":123,"url":"http://projects.pilosa.com/10"}}, @@ -189,25 +196,25 @@ In this example, ColumnAttrs have been set on columns 10 and 20, but not column Delete the `url` attribute on column 10: ```request -SetColumnAttrs(col=10, url=null) +SetColumnAttrs(10, url=null) ``` ```response {"results":[null]} ``` -#### ClearBit +#### Clear **Spec:** ``` -ClearBit(, , ) +Clear(, =) ``` **Description:** -`ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column. +`Clear` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. -Note that clearing bits from time views is not supported. +Note that clearing a column on a time field will remove all data for that column. **Result Type:** boolean @@ -217,9 +224,9 @@ A return value of `false` indicates that the bit was already set to 0 and nothin **Examples:** -Clear the bit at row 1 and column 10 in the stargazer frame: +Clear the bit at row 1 and column 10 in the stargazer field: ```request -ClearBit(frame="stargazer", col=10, row=1) +Clear(10, stargazer=1) ``` ```response {"results":[true]} @@ -227,79 +234,48 @@ ClearBit(frame="stargazer", col=10, row=1) This represents removing the relationship between the user with id=1 and the repository with id=10. -#### SetFieldValue - -**Spec:** - -``` -SetFieldValue(, , ) -``` - -**Description:** - -`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`. - -**Result Type:** null - -SetFieldValue returns `null` upon success. - -**Examples:** - -Set the field value `pullrequest` to the value 2, on column 10 in frame `stats`: -```request -SetFieldValue(col=10, frame="stats", pullrequests=2) -``` -```response -{"results":[null]} -``` - -This represents setting the number of pull requests of repository 10 to 2. - -This example assumes the existence of the frame `stats` and the field `pullrequests`. See [frame creation](../api-reference/#create-frame) and [field creation](../api-reference/#create-field) for more information. - - ### Read Operations -#### Bitmap +#### Row **Spec:** ``` -Bitmap(, ( | =UINT)) +Row(=) ``` **Description:** -`Bitmap` retrieves the indices of all the set bits in a row or column based on whether the row or column argument is provided in the query. It also retrieves any attributes set on that row or column. +`Row` retrieves the indices of all the columns in a row. It also retrieves any attributes set on that row. -**Result Type:** object with attrs and bits. +**Result Type:** object with attrs and columns. -e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` +e.g. `{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]}` **Examples:** -Query all columns with a bit set in row 1 of the frame `stargazer` (repositories that are starred by user 1): +Query all columns with a bit set in row 1 of the field `stargazer` (repositories that are starred by user 1): ```request -Bitmap(frame="stargazer", row=1) +Row(stargazer=1) ``` ```response -{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]} +{"attrs":{"username":"mrpi","active":true},"columns":[10, 20]} ``` * attrs are the attributes for user 1 -* bits are the repositories which user 1 has starred. +* columns are the repositories which user 1 has starred. #### Union **Spec:** ``` -Union([BITMAP_CALL ...]) +Union([ROW_CALL ...]) ``` **Description:** -Union performs a logical OR on the results of all `BITMAP_CALL` queries passed to it. +Union performs a logical OR on the results of all `ROW_CALL` queries passed to it. **Result Type:** object with attrs and bits @@ -309,28 +285,27 @@ attrs will always be empty Query columns with a bit set in either of two rows (repositories that are starred by either of two users): ```request -Union(Bitmap(frame="stargazer", stargazer_id=1), Bitmap(frame="stargazer", stargazer_id=2)) +Union(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"attrs":{},"bits":[10, 20, 30]} +{"attrs":{},"columns":[10, 20, 30]} ``` -* bits are repositories that were starred by user 1 OR user 2 +* columns are repositories that were starred by user 1 OR user 2 #### Intersect - **Spec:** ``` -Intersect(, [BITMAP_CALL ...]) +Intersect(, [ROW_CALL ...]) ``` **Description:** -Intersect performs a logical AND on the results of all `BITMAP_CALL` queries passed to it. +Intersect performs a logical AND on the results of all `ROW_CALL` queries passed to it. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -339,27 +314,27 @@ attrs will always be empty Query columns with a bit set in both of two rows (repositories that are starred by both of two users): ```request -Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) +Intersect(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"attrs":{},"bits":[10]} +{"attrs":{},"columns":[10]} ``` -* bits are repositories that were starred by user 1 AND user 2 +* columns are repositories that were starred by user 1 AND user 2 #### Difference **Spec:** ``` -Difference(, [BITMAP_CALL ...]) +Difference(, [ROW_CALL ...]) ``` **Description:** -Difference returns all of the bits from the first `BITMAP_CALL` argument passed to it, without the bits from each subsequent `BITMAP_CALL`. +Difference returns all of the bits from the first `ROW_CALL` argument passed to it, without the bits from each subsequent `ROW_CALL`. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -367,37 +342,37 @@ attrs will always be empty Query columns with a bit set in one row and not another (repositories that are starred by one user and not another): ```request -Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2)) +Difference(Row(stargazer=1), Row(stargazer=2)) ``` ```response -{"results":[{"attrs":{},"bits":[20]}]} +{"results":[{"attrs":{},"columns":[20]}]} ``` -* bits are repositories that were starred by user 1 BUT NOT user 2 +* columns are repositories that were starred by user 1 BUT NOT user 2 Query for the opposite difference: ```request -Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1)) +Difference(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"attrs":{},"bits":[30]} +{"attrs":{},"columns":[30]} ``` -* Bits are repositories that were starred by user 2 BUT NOT user 1 +* columnss are repositories that were starred by user 2 BUT NOT user 1 #### Xor **Spec:** ``` -Xor(, [BITMAP_CALL ...]) +Xor(, [ROW_CALL ...]) ``` **Description:** -Xor performs a logical XOR on the results of each `BITMAP_CALL` query passed to it. +Xor performs a logical XOR on the results of each `ROW_CALL` query passed to it. -**Result Type:** object with attrs and bits +**Result Type:** object with attrs and columns attrs will always be empty @@ -406,24 +381,24 @@ attrs will always be empty Query columns with a bit set in exactly one of two rows (repositories that are starred by only one of two users): ```request -Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) +Xor(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"results":[{"attrs":{},"bits":[10,20,30]}]} +{"results":[{"attrs":{},"columns":[10,20,30]}]} ``` -* bits are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) +* columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) #### Count **Spec:** ``` -Count() +Count() ``` **Description:** -Returns the number of set bits in the `BITMAP_CALL` passed in. +Returns the number of set bits in the `ROW_CALL` passed in. **Result Type:** int @@ -431,7 +406,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in. Query the number of bits set in a row (the number of repositories a user has starred): ```request -Count(Bitmap(frame="stargazer", row=1)) +Count(Row(stargazer=1)) ``` ```response {"results":[1]} @@ -444,34 +419,34 @@ Count(Bitmap(frame="stargazer", row=1)) **Spec:** ``` -TopN([BITMAP_CALL], , [n=UINT], - [, ]) +TopN([ROW_CALL], , [n=UINT], + [attrName=, attrValues=<[]ATTR_VALUE>]) ``` **Description:** -Return the id and count of the top `n` bitmaps (by count of bits) in the frame. -The `field` and `filters` arguments work together to only return Bitmaps which -have the attribute specified by `field` with one of the values specified in -`filters`. +Return the id and count of the top `n` rows (by count of bits) in the field. +The `attrName` and `attrValues` arguments work together to only return rows which +have the attribute specified by `attrName` with one of the values specified in +`attrValues`. **Result Type:** array of key/count objects **Caveats:** -* Performing a TopN() query on a frame with cache type ranked will return the top bitmaps sorted by count in descending order. -* Frames with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of frame will return bitmaps sorted in order of most recently set bit. -* The frame's cache size determines the number of sorted bitmaps to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. -* Once full, the cache will truncate the set of bitmaps according to the frame option CacheSize. Bitmaps that straddle the limit and have the same count will be truncated in no particular order. -* The TopN query's attribute filter is applied to the existing sorted cache of bitmaps. Bitmaps that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored. +* Performing a TopN() query on a field with cache type ranked will return the top rows sorted by count in descending order. +* Fields with cache type lru will maintain an LRU (Least Recently Used replacement policy) cache, thus a TopN query on this type of field will return rows sorted in order of most recently set bit. +* The field's cache size determines the number of sorted rows to maintain in the cache for purposes of TopN queries. There is a tradeoff between performance and accuracy; increasing the cache size will improve accuracy of results at the cost of performance. +* Once full, the cache will truncate the set of rows according to the field option CacheSize. Rows that straddle the limit and have the same count will be truncated in no particular order. +* The TopN query's attribute filter is applied to the existing sorted cache of rows. Rows that fall outside of the sorted cache range, even if they would normally pass the filter, are ignored. -See [frame creation](../api-reference/#create-frame) for more information about the cache. +See [field creation](../api-reference/#create-field) for more information about the cache. **Examples:** Basic TopN query: ```request -TopN(frame="stargazer") +TopN(stargazer) ``` ```response {"results":[[{"id":1240,"count":102},{"id":4734,"count":100},{"id":12709,"count":93},...]]} @@ -479,11 +454,11 @@ TopN(frame="stargazer") * `id` is a row ID (user ID) * `count` is a count of columns (repositories) -* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer frame. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository. +* Results are the number of bits set in the corresponding row (repositories that each user starred) in descending order for all rows (users) in the stargazer field. For example user 1240 starred 102 repositories, user 4734 starred 100 repositories, user 12709 starred 93 repository. Limit the number of results: ```request -TopN(frame="stargazer", n=2) +TopN(stargazer, n=2) ``` ```response {"results":[[{"id":1240,"count":102},{"id":4734,"count":100}]]} @@ -491,19 +466,19 @@ TopN(frame="stargazer", n=2) * Results are the top two rows (users) sorted by number of bits set (repositories they've starred) in descending order. -Filter based on an existing Bitmap: +Filter based on an existing row: ```request -TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2) +TopN(Row(language=1), stargazer, n=2) ``` ```response {"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]} ``` -* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language frame (repositories that they've starred which are written in language 1). +* Results are the top two users (rows) sorted by the number of bits set in the intersection with row 1 of the language field (repositories that they've starred which are written in language 1). Filter based on attributes: ```request -TopN(frame="stargazer", n=2, field=active, filters=[true]) +TopN(stargazer, n=2, attrName=active, attrValues=[true]) ``` ```response {"results":[[{"id":10,"count":1},{"id":13,"count":1}]]} @@ -516,31 +491,30 @@ TopN(frame="stargazer", n=2, field=active, filters=[true]) **Spec:** ``` -Range(, , - , ) +Range(=, , ) ``` **Description:** -Similar to `Bitmap`, but only returns bits which were set with timestamps -between the given `start` and `end` timestamps. +Similar to `Row`, but only returns bits which were set with timestamps +between the given `start` (first) and `end` (second) timestamps. **Result Type:** object with attrs and bits **Examples:** -Query all columns with a bit set in row 1 of a frame (repositories that a user has starred), within a date range: +Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: ```request -Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) ``` ```response -{{"attrs":{},"bits":[10]} +{{"attrs":{},"columns":[10]} ``` This example assumes timestamps have been set on some bits. -* bits are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. +* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. #### Range (BSI) @@ -548,16 +522,15 @@ This example assumes timestamps have been set on some bits. **Spec:** ``` -Range(, ) +Range([ ] ) ``` **Description:** -The `Range` query is overloaded to work on `field` values as well as `timestamp` values. +The `Range` query is overloaded to work on `integer` values as well as `timestamp` values. Returns bits that are true for the comparison operator. -**Result Type:** object with attrs and bits - +**Result Type:** object with attrs and columns **Examples:** @@ -565,13 +538,13 @@ In our source data, commitactivity was counted over the last year. The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): ```request -Range(frame="stats", commitactivity > 100) +Range(commitactivity > 100) ``` ```response -{{"attrs":{},"bits":[10]} +{{"attrs":{},"columns":[10]} ``` -* bits are repositories which had at least 100 commits in the last year. +* columns are repositories which had at least 100 commits in the last year. BSI range queries support the following operators: @@ -583,35 +556,37 @@ BSI range queries support the following operators: `>=` | greater-than-or-equal-to, GTE | integer `==` | equal-to, EQ | integer `!=` | not-equal-to, NEQ | integer or `null` - `><` | between, BETWEEN | [integer, integer] -The `BETWEEN` form specifies an interval with both bounds, using the `><` operator, and a two-element list containing the lower and upper bounds of the interval: +`<`, and `<=` can be chained together to represent a bounded interval. For example: -```pql -Range(frame="stats", commitactivity >< [100, 200]) +```request +Range(50 < commitactivity < 150) +``` +```response +{{"attrs":{},"columns":[10]} ``` -This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported. #### Min **Spec:** ``` -Min([BITMAP_CALL], , ) +Min([ROW_CALL], field=) ``` **Description:** -Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. +Returns the minimum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered. **Result Type:** object with the min and count of columns containing the min value. **Examples:** -Query the minimum value of all fields in a frame (minimum size of all repositories): +Query the minimum value of a field (minimum size of all repositories): ```request -Min(frame="stats", field="diskusage") +Min(field="diskusage") ``` ```response {"value":4,"count":2} @@ -624,20 +599,20 @@ Min(frame="stats", field="diskusage") **Spec:** ``` -Max([BITMAP_CALL], , ) +Max([ROW_CALL], field=) ``` **Description:** -Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. +Returns the maximum value of all BSI integer values in this `field`. If the optional `Row` call is supplied, only columns with set bits are considered, otherwise all columns are considered. **Result Type:** object with the max and count of columns containing the max value. **Examples:** -Query the maximum value of all fields in a frame (maximum size of all repositories): +Query the maximum value of a field (maximum size of all repositories): ```request -Max(frame="stats", field="diskusage") +Max(field="diskusage") ``` ```response {"value":88,"count":13} @@ -650,20 +625,20 @@ Max(frame="stats", field="diskusage") **Spec:** ``` -Sum([BITMAP_CALL], , ) +Sum([ROW_CALL], field=) ``` **Description:** -Returns the count and computed sum of all BSI integer values in the `field` and `frame`. If the optional `Bitmap` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. +Returns the count and computed sum of all BSI integer values in the `field`. If the optional `Row` call is supplied, columns with set bits are summed, otherwise the sum is across all columns. -**Result Type:** object with the computed sum and count of the bitmap field. +**Result Type:** object with the computed sum and count of the values in the integer field. **Examples:** Query the size of all repositories. ```request -Sum(frame="stats", field="diskusage") +Sum(field="diskusage") ``` ```response {"value":10,"count":3} diff --git a/docs/tutorials.md b/docs/tutorials.md index 06cdab286..e55a2094a 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -407,37 +407,65 @@ curl localhost:10101/index/patients \ -X POST ``` ``` response -{} +{"success":true} ``` -In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. +In addition to storing rows of bits, a frame can also contain fields that store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. ``` request -curl localhost:10101/index/patients/frame/measurements \ +curl localhost:10101/index/patients/field/age \ -X POST \ - -d '{"options":{ - "fields": [ - {"name": "age", "type": "int", "min": 0, "max": 120}, - {"name": "weight", "type": "int", "min": 0, "max": 500}, - {"name": "tcells", "type": "int", "min": 0, "max": 2000} - ] - }}' + -d '{"options":{"type": "int", "min": 0, "max": 120}}' ``` ``` response -{} +{"success":true} ``` -If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). +``` request +curl localhost:10101/index/patients/field/weight \ + -X POST \ + -d '{"options":{"type": "int", "min": 0, "max": 500}}' +``` +``` response +{"success":true} +``` + +``` request +curl localhost:10101/index/patients/field/tcells \ + -X POST \ + -d '{"options":{"type": "int", "min": 0, "max": 2000}}' +``` +``` response +{"success":true} +``` Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. -This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: +The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)' + -d 'Set(1, age=34)' ``` ``` response -{"results":[null]} +{"results":[true]} +``` + +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Set(1, weight=128)' +``` +``` response +{"results":[true]} +``` + +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Set(1, tcells=1145)' +``` +``` response +{"results":[true]} ``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -454,7 +482,7 @@ Assuming we have a file called `ages.csv` that is structured like this: 8,33 9,63 ``` -where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command: +where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command: ``` pilosa import -i patients -f measurements --field age ages.csv ``` @@ -465,10 +493,10 @@ In order to find all patients over the age of 40, then simply run a `Range` quer ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Range(frame="measurements", age > 40)' + -d 'Range(age > 40)' ``` ``` response -{"results":[{"attrs":{},"bits":[2,6,9]}]} +{"results":[{"attrs":{},"columns":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. @@ -477,21 +505,21 @@ To find the average age of all patients, run a `Sum` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(frame="measurements", field="age")' + -d 'Sum(field="age")' ``` ``` response -{"results":[{"sum":377,"count":9}]} +{"results":[{"value":377,"count":9}]} ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. +The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`. You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' + -d 'Sum(Range(age > 40), field="age")' ``` ``` response -{"results":[{"sum":191,"count":3}]} +{"results":[{"value":191,"count":3}]} ``` Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. @@ -499,42 +527,42 @@ To find the minimum age of all patients, run a `Min` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(frame="measurements", field="age")' + -d 'Min(field="age")' ``` ``` response -{"results":[{"min":19,"count":1}]} +{"results":[{"value":19,"count":1}]} ``` -The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. +The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value. You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' + -d 'Min(Range(age > 40), field="age")' ``` ``` response -{"results":[{"min":57,"count":1}]} +{"results":[{"value":57,"count":1}]} ``` To find the maximum age of all patients, run a `Max` query: ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(frame="measurements", field="age")' + -d 'Max(field="age")' ``` ``` response -{"results":[{"max":71,"count":1}]} +{"results":[{"value":71,"count":1}]} ``` -The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. +The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value. You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' + -d 'Max(Range(age < 40), field="age")' ``` ``` response -{"results":[{"max":34,"count":1}]} +{"results":[{"value":34,"count":1}]} ``` ### Storing Row and Column Attributes @@ -549,28 +577,28 @@ curl localhost:10101/index/books \ -X POST ``` ``` response -{} +{"success":true} ``` -Next, create a frame in the `books` index called `members` which will represent library members who have read books. +Next, create a field in the `books` index called `members` which will represent library members who have read books. ``` request -curl localhost:10101/index/books/frame/members \ +curl localhost:10101/index/books/field/members \ -X POST \ -d '{}' ``` ``` response -{} +{"success":true} ``` Now, let's add some books to our index. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960) - SetColumnAttrs(col=2, name="No Name in the Street", year=1972) - SetColumnAttrs(col=3, name="The Tipping Point", year=2000) - SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003) - SetColumnAttrs(col=5, name="The Forever War", year=2008)' + -d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(2, name="No Name in the Street", year=1972) + SetColumnAttrs(3, name="The Tipping Point", year=2000) + SetColumnAttrs(4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(5, name="The Forever War", year=2008)' ``` ``` response {"results":[null,null,null,null,null]} @@ -580,11 +608,11 @@ And add some members. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith") - SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins") - SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks") - SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez") - SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")' + -d 'SetRowAttrs(members, 10001, fullName="John Smith") + SetRowAttrs(members, 10002, fullName="Sue Perkins") + SetRowAttrs(members, 10003, fullName="Jennifer Hawks") + SetRowAttrs(members, 10004, fullName="Pedro Vazquez") + SetRowAttrs(members, 10005, fullName="Pat Washington")' ``` ``` response {"results":[null,null,null,null,null]} @@ -594,29 +622,29 @@ At this point we can query one of the `member` records by querying that row. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response -{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} +{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetBit(frame="members", row=10001, col=3) - SetBit(frame="members", row=10001, col=5) - SetBit(frame="members", row=10002, col=1) - SetBit(frame="members", row=10002, col=2) - SetBit(frame="members", row=10002, col=4) - SetBit(frame="members", row=10003, col=3) - SetBit(frame="members", row=10004, col=4) - SetBit(frame="members", row=10004, col=5) - SetBit(frame="members", row=10005, col=1) - SetBit(frame="members", row=10005, col=2) - SetBit(frame="members", row=10005, col=3) - SetBit(frame="members", row=10005, col=4) - SetBit(frame="members", row=10005, col=5)' + -d 'Set(3, members=10001) + Set(5, members=10001) + Set(1, members=10002) + Set(2, members=10002) + Set(4, members=10002) + Set(3, members=10003) + Set(4, members=10004) + Set(5, members=10004) + Set(1, members=10005) + Set(2, members=10005) + Set(3, members=10005) + Set(4, members=10005) + Set(5, members=10005)' ``` ``` response {"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} @@ -626,22 +654,22 @@ Now pull the record for `Sue Perkins` again. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response -{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} +{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]} ``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. +Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Bitmap(frame="members", row=10002)' + -d 'Row(members=10002)' ``` ``` response { - "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], + "results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}], "columnAttrs":[ {"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}}, {"id":2,"attrs":{"name":"No Name in the Street","year":1972}}, @@ -655,11 +683,11 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`, ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))' + -d 'Intersect(Row(members=10002), Row(members=10004))' ``` ``` response { - "results":[{"attrs":{},"bits":[4]}], + "results":[{"attrs":{},"columns":[4]}], "columnAttrs":[ {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}} ] diff --git a/docs/webui.md b/docs/webui.md index 07acd2ab4..a07132852 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -9,13 +9,27 @@ nav = [ ## WebUI -The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101). +A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status. -This can be used for constructing queries and viewing the cluster status. +### Installation + +Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac. + +Installing on a Mac with Homebrew is simple; just run: + +``` +brew install pilosa-webui +``` + +You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running: + +``` +make install +``` ### Console -The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. +The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. Each query's result will be displayed in the Output section along with the query time. @@ -29,14 +43,14 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create index ` - `:delete index ` - `:use ` -- `:create frame ` -- `:delete frame ` +- `:create field ` +- `:delete field ` -Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). +Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field). -- `:create frame cacheSize=10000` +- `:create field cacheSize=10000` ### Cluster Admin -Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. +Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields. diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go new file mode 100644 index 000000000..d8367c468 --- /dev/null +++ b/encoding/proto/proto.go @@ -0,0 +1,1037 @@ +package proto + +import ( + "fmt" + "sort" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// Serializer implements pilosa.Serializer for protobufs. +type Serializer struct{} + +// Marshal turns pilosa messages into protobuf serialized bytes. +func (Serializer) Marshal(m pilosa.Message) ([]byte, error) { + pm := encodeToProto(m) + if pm == nil { + return nil, errors.New("passed invalid pilosa.Message") + } + buf, err := proto.Marshal(pm) + return buf, errors.Wrap(err, "marshalling") +} + +// Unmarshal takes byte slices and protobuf deserializes them into a pilosa Message. +func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + msg := &internal.CreateShardMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateShardMessage") + } + decodeCreateShardMessage(msg, mt) + return nil + case *pilosa.CreateIndexMessage: + msg := &internal.CreateIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateIndexMessage") + } + decodeCreateIndexMessage(msg, mt) + return nil + case *pilosa.DeleteIndexMessage: + msg := &internal.DeleteIndexMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteIndexMessage") + } + decodeDeleteIndexMessage(msg, mt) + return nil + case *pilosa.CreateFieldMessage: + msg := &internal.CreateFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateFieldMessage") + } + decodeCreateFieldMessage(msg, mt) + return nil + case *pilosa.DeleteFieldMessage: + msg := &internal.DeleteFieldMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteFieldMessage") + } + decodeDeleteFieldMessage(msg, mt) + return nil + case *pilosa.CreateViewMessage: + msg := &internal.CreateViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling CreateViewMessage") + } + decodeCreateViewMessage(msg, mt) + return nil + case *pilosa.DeleteViewMessage: + msg := &internal.DeleteViewMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling DeleteViewMessage") + } + decodeDeleteViewMessage(msg, mt) + return nil + case *pilosa.ClusterStatus: + msg := &internal.ClusterStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ClusterStatus") + } + decodeClusterStatus(msg, mt) + return nil + case *pilosa.ResizeInstruction: + msg := &internal.ResizeInstruction{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstruction") + } + decodeResizeInstruction(msg, mt) + return nil + case *pilosa.ResizeInstructionComplete: + msg := &internal.ResizeInstructionComplete{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ResizeInstructionComplete") + } + decodeResizeInstructionComplete(msg, mt) + return nil + case *pilosa.SetCoordinatorMessage: + msg := &internal.SetCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling SetCoordinatorMessage") + } + decodeSetCoordinatorMessage(msg, mt) + return nil + case *pilosa.UpdateCoordinatorMessage: + msg := &internal.UpdateCoordinatorMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage") + } + decodeUpdateCoordinatorMessage(msg, mt) + return nil + case *pilosa.NodeStateMessage: + msg := &internal.NodeStateMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStateMessage") + } + decodeNodeStateMessage(msg, mt) + return nil + case *pilosa.RecalculateCaches: + msg := &internal.RecalculateCaches{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling RecalculateCaches") + } + decodeRecalculateCaches(msg, mt) + return nil + case *pilosa.NodeEvent: + msg := &internal.NodeEventMessage{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeEvent") + } + decodeNodeEventMessage(msg, mt) + return nil + case *pilosa.NodeStatus: + msg := &internal.NodeStatus{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling NodeStatus") + } + decodeNodeStatus(msg, mt) + return nil + case *pilosa.Node: + msg := &internal.Node{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling Node") + } + decodeNode(msg, mt) + return nil + case *pilosa.QueryRequest: + msg := &internal.QueryRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryRequest") + } + decodeQueryRequest(msg, mt) + return nil + case *pilosa.QueryResponse: + msg := &internal.QueryResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling QueryResponse") + } + decodeQueryResponse(msg, mt) + return nil + case *pilosa.ImportRequest: + msg := &internal.ImportRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRequest") + } + decodeImportRequest(msg, mt) + return nil + case *pilosa.ImportValueRequest: + msg := &internal.ImportValueRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportValueRequest") + } + decodeImportValueRequest(msg, mt) + return nil + case *pilosa.ImportResponse: + msg := &internal.ImportResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportResponse") + } + decodeImportResponse(msg, mt) + return nil + case *pilosa.BlockDataRequest: + msg := &internal.BlockDataRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataRequest") + } + decodeBlockDataRequest(msg, mt) + return nil + case *pilosa.BlockDataResponse: + msg := &internal.BlockDataResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling BlockDataResponse") + } + decodeBlockDataResponse(msg, mt) + return nil + default: + panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) + } +} + +func encodeToProto(m pilosa.Message) proto.Message { + switch mt := m.(type) { + case *pilosa.CreateShardMessage: + return encodeCreateShardMessage(mt) + case *pilosa.CreateIndexMessage: + return encodeCreateIndexMessage(mt) + case *pilosa.DeleteIndexMessage: + return encodeDeleteIndexMessage(mt) + case *pilosa.CreateFieldMessage: + return encodeCreateFieldMessage(mt) + case *pilosa.DeleteFieldMessage: + return encodeDeleteFieldMessage(mt) + case *pilosa.CreateViewMessage: + return encodeCreateViewMessage(mt) + case *pilosa.DeleteViewMessage: + return encodeDeleteViewMessage(mt) + case *pilosa.ClusterStatus: + return encodeClusterStatus(mt) + case *pilosa.ResizeInstruction: + return encodeResizeInstruction(mt) + case *pilosa.ResizeInstructionComplete: + return encodeResizeInstructionComplete(mt) + case *pilosa.SetCoordinatorMessage: + return encodeSetCoordinatorMessage(mt) + case *pilosa.UpdateCoordinatorMessage: + return encodeUpdateCoordinatorMessage(mt) + case *pilosa.NodeStateMessage: + return encodeNodeStateMessage(mt) + case *pilosa.RecalculateCaches: + return encodeRecalculateCaches(mt) + case *pilosa.NodeEvent: + return encodeNodeEventMessage(mt) + case *pilosa.NodeStatus: + return encodeNodeStatus(mt) + case *pilosa.Node: + return encodeNode(mt) + case *pilosa.QueryRequest: + return encodeQueryRequest(mt) + case *pilosa.QueryResponse: + return encodeQueryResponse(mt) + case *pilosa.ImportRequest: + return encodeImportRequest(mt) + case *pilosa.ImportValueRequest: + return encodeImportValueRequest(mt) + case *pilosa.ImportResponse: + return encodeImportResponse(mt) + case *pilosa.BlockDataRequest: + return encodeBlockDataRequest(mt) + case *pilosa.BlockDataResponse: + return encodeBlockDataResponse(mt) + } + return nil +} + +func encodeBlockDataRequest(m *pilosa.BlockDataRequest) *internal.BlockDataRequest { + return &internal.BlockDataRequest{ + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + Block: m.Block, + } +} +func encodeBlockDataResponse(m *pilosa.BlockDataResponse) *internal.BlockDataResponse { + return &internal.BlockDataResponse{ + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + } +} + +func encodeImportResponse(m *pilosa.ImportResponse) *internal.ImportResponse { + return &internal.ImportResponse{ + Err: m.Err, + } +} + +func encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { + return &internal.ImportRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, + } +} + +func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { + return &internal.ImportValueRequest{ + Index: m.Index, + Field: m.Field, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + } +} + +func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { + return &internal.QueryRequest{ + Query: m.Query, + Shards: m.Shards, + ColumnAttrs: m.ColumnAttrs, + Remote: m.Remote, + ExcludeRowAttrs: m.ExcludeRowAttrs, + ExcludeColumns: m.ExcludeColumns, + } +} + +func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(m.Results)), + ColumnAttrSets: encodeColumnAttrSets(m.ColumnAttrSets), + } + + for i := range m.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := m.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = queryResultTypeRow + pb.Results[i].Row = encodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = queryResultTypePairs + pb.Results[i].Pairs = encodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = queryResultTypeValCount + pb.Results[i].ValCount = encodeValCount(result) + case uint64: + pb.Results[i].Type = queryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = queryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = queryResultTypeNil + } + } + + if m.Err != nil { + pb.Err = m.Err.Error() + } + + return pb +} + +func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstruction { + return &internal.ResizeInstruction{ + JobID: m.JobID, + Node: encodeNode(m.Node), + Coordinator: encodeNode(m.Coordinator), + Sources: encodeResizeSources(m.Sources), + Schema: encodeSchema(m.Schema), + ClusterStatus: encodeClusterStatus(m.ClusterStatus), + } +} + +func encodeResizeSources(srcs []*pilosa.ResizeSource) []*internal.ResizeSource { + new := make([]*internal.ResizeSource, 0, len(srcs)) + for _, src := range srcs { + new = append(new, encodeResizeSource(src)) + } + return new +} + +func encodeResizeSource(m *pilosa.ResizeSource) *internal.ResizeSource { + return &internal.ResizeSource{ + Node: encodeNode(m.Node), + Index: m.Index, + Field: m.Field, + View: m.View, + Shard: m.Shard, + } +} + +func encodeSchema(m *pilosa.Schema) *internal.Schema { + return &internal.Schema{ + Indexes: encodeIndexInfos(m.Indexes), + } +} + +func encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index { + new := make([]*internal.Index, 0, len(idxs)) + for _, idx := range idxs { + new = append(new, encodeIndexInfo(idx)) + } + return new +} + +func encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { + return &internal.Index{ + Name: idx.Name, + Fields: encodeFieldInfos(idx.Fields), + } +} + +func encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { + new := make([]*internal.Field, 0, len(fs)) + for _, f := range fs { + new = append(new, encodeFieldInfo(f)) + } + return new +} + +func encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { + ifield := &internal.Field{ + Name: f.Name, + Meta: encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), + } + + for _, viewinfo := range f.Views { + ifield.Views = append(ifield.Views, viewinfo.Name) + } + return ifield +} + +func encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +// encodeNodes converts a slice of Nodes into its internal representation. +func encodeNodes(a []*pilosa.Node) []*internal.Node { + other := make([]*internal.Node, len(a)) + for i := range a { + other[i] = encodeNode(a[i]) + } + return other +} + +// encodeNode converts a Node into its internal representation. +func encodeNode(n *pilosa.Node) *internal.Node { + return &internal.Node{ + ID: n.ID, + URI: encodeURI(n.URI), + IsCoordinator: n.IsCoordinator, + } +} + +func encodeURI(u pilosa.URI) *internal.URI { + return &internal.URI{ + Scheme: u.Scheme, + Host: u.Host, + Port: uint32(u.Port), + } +} + +func encodeClusterStatus(m *pilosa.ClusterStatus) *internal.ClusterStatus { + return &internal.ClusterStatus{ + State: m.State, + ClusterID: m.ClusterID, + Nodes: encodeNodes(m.Nodes), + } +} + +func encodeCreateShardMessage(m *pilosa.CreateShardMessage) *internal.CreateShardMessage { + return &internal.CreateShardMessage{ + Index: m.Index, + Shard: m.Shard, + } +} + +func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { + return &internal.CreateIndexMessage{ + Index: m.Index, + Meta: encodeIndexMeta(m.Meta), + } +} + +func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { + return &internal.IndexMeta{ + Keys: m.Keys, + } +} + +func encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *internal.DeleteIndexMessage { + return &internal.DeleteIndexMessage{ + Index: m.Index, + } +} + +func encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { + return &internal.CreateFieldMessage{ + Index: m.Index, + Field: m.Field, + Meta: encodeFieldOptions(m.Meta), + } +} + +func encodeDeleteFieldMessage(m *pilosa.DeleteFieldMessage) *internal.DeleteFieldMessage { + return &internal.DeleteFieldMessage{ + Index: m.Index, + Field: m.Field, + } +} + +func encodeCreateViewMessage(m *pilosa.CreateViewMessage) *internal.CreateViewMessage { + return &internal.CreateViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeDeleteViewMessage(m *pilosa.DeleteViewMessage) *internal.DeleteViewMessage { + return &internal.DeleteViewMessage{ + Index: m.Index, + Field: m.Field, + View: m.View, + } +} + +func encodeResizeInstructionComplete(m *pilosa.ResizeInstructionComplete) *internal.ResizeInstructionComplete { + return &internal.ResizeInstructionComplete{ + JobID: m.JobID, + Node: encodeNode(m.Node), + Error: m.Error, + } +} + +func encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage { + return &internal.SetCoordinatorMessage{ + New: encodeNode(m.New), + } +} + +func encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage { + return &internal.UpdateCoordinatorMessage{ + New: encodeNode(m.New), + } +} + +func encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage { + return &internal.NodeStateMessage{ + NodeID: m.NodeID, + State: m.State, + } +} + +func encodeNodeEventMessage(m *pilosa.NodeEvent) *internal.NodeEventMessage { + return &internal.NodeEventMessage{ + Event: uint32(m.Event), + Node: encodeNode(m.Node), + } +} + +func encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus { + return &internal.NodeStatus{ + Node: encodeNode(m.Node), + MaxShards: &internal.MaxShards{Standard: m.MaxShards}, + Schema: encodeSchema(m.Schema), + } +} + +func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCaches { + return &internal.RecalculateCaches{} +} + +func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { + m.JobID = ri.JobID + m.Node = &pilosa.Node{} + decodeNode(ri.Node, m.Node) + m.Coordinator = &pilosa.Node{} + decodeNode(ri.Coordinator, m.Coordinator) + m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) + decodeResizeSources(ri.Sources, m.Sources) + m.Schema = &pilosa.Schema{} + decodeSchema(ri.Schema, m.Schema) + m.ClusterStatus = &pilosa.ClusterStatus{} + decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) +} + +func decodeResizeSources(srcs []*internal.ResizeSource, m []*pilosa.ResizeSource) { + for i := range srcs { + m[i] = &pilosa.ResizeSource{} + decodeResizeSource(srcs[i], m[i]) + } +} + +func decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) { + m.Node = &pilosa.Node{} + decodeNode(rs.Node, m.Node) + m.Index = rs.Index + m.Field = rs.Field + m.View = rs.View + m.Shard = rs.Shard +} + +func decodeSchema(s *internal.Schema, m *pilosa.Schema) { + m.Indexes = make([]*pilosa.IndexInfo, len(s.Indexes)) + decodeIndexes(s.Indexes, m.Indexes) +} + +func decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) { + for i := range idxs { + m[i] = &pilosa.IndexInfo{} + decodeIndex(idxs[i], m[i]) + } +} + +func decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { + m.Name = idx.Name + m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) + decodeFields(idx.Fields, m.Fields) +} + +func decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { + for i := range fs { + m[i] = &pilosa.FieldInfo{} + decodeField(fs[i], m[i]) + } +} + +func decodeField(f *internal.Field, m *pilosa.FieldInfo) { + m.Name = f.Name + m.Options = pilosa.FieldOptions{} + decodeFieldOptions(f.Meta, &m.Options) + m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) + for _, viewname := range f.Views { + m.Views = append(m.Views, &pilosa.ViewInfo{Name: viewname}) + } +} + +func decodeFieldOptions(options *internal.FieldOptions, m *pilosa.FieldOptions) { + m.Type = options.Type + m.CacheType = options.CacheType + m.CacheSize = options.CacheSize + m.Min = options.Min + m.Max = options.Max + m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + m.Keys = options.Keys +} + +func decodeNodes(a []*internal.Node, m []*pilosa.Node) { + for i := range a { + m[i] = &pilosa.Node{} + decodeNode(a[i], m[i]) + } +} + +func decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) { + m.State = cs.State + m.ClusterID = cs.ClusterID + m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) + decodeNodes(cs.Nodes, m.Nodes) +} + +func decodeNode(node *internal.Node, m *pilosa.Node) { + m.ID = node.ID + decodeURI(node.URI, &m.URI) + m.IsCoordinator = node.IsCoordinator +} + +func decodeURI(i *internal.URI, m *pilosa.URI) { + m.Scheme = i.Scheme + m.Host = i.Host + m.Port = uint16(i.Port) +} + +func decodeCreateShardMessage(pb *internal.CreateShardMessage, m *pilosa.CreateShardMessage) { + m.Index = pb.Index + m.Shard = pb.Shard +} + +func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { + m.Index = pb.Index + m.Meta = &pilosa.IndexOptions{} + decodeIndexMeta(pb.Meta, m.Meta) +} + +func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { + m.Keys = pb.Keys +} + +func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { + m.Index = pb.Index +} + +func decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.Meta = &pilosa.FieldOptions{} + decodeFieldOptions(pb.Meta, m.Meta) +} + +func decodeDeleteFieldMessage(pb *internal.DeleteFieldMessage, m *pilosa.DeleteFieldMessage) { + m.Index = pb.Index + m.Field = pb.Field +} + +func decodeCreateViewMessage(pb *internal.CreateViewMessage, m *pilosa.CreateViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *pilosa.DeleteViewMessage) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View +} + +func decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) { + m.JobID = pb.JobID + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.Error = pb.Error +} + +func decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) { + m.New = &pilosa.Node{} + decodeNode(pb.New, m.New) +} + +func decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) { + m.NodeID = pb.NodeID + m.State = pb.State +} + +func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) { + m.Event = pilosa.NodeEventType(pb.Event) + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) +} + +func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { + m.Node = &pilosa.Node{} + decodeNode(pb.Node, m.Node) + m.MaxShards = pb.MaxShards.Standard + m.Schema = &pilosa.Schema{} + decodeSchema(pb.Schema, m.Schema) +} + +func decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {} + +func decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) { + m.Query = pb.Query + m.Shards = pb.Shards + m.ColumnAttrs = pb.ColumnAttrs + m.Remote = pb.Remote + m.ExcludeRowAttrs = pb.ExcludeRowAttrs + m.ExcludeColumns = pb.ExcludeColumns +} + +func decodeImportRequest(pb *internal.ImportRequest, m *pilosa.ImportRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs + m.RowKeys = pb.RowKeys + m.ColumnKeys = pb.ColumnKeys + m.Timestamps = pb.Timestamps +} + +func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Shard = pb.Shard + m.ColumnIDs = pb.ColumnIDs + m.ColumnKeys = pb.ColumnKeys + m.Values = pb.Values +} + +func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { + m.Err = pb.Err +} + +func decodeBlockDataRequest(pb *internal.BlockDataRequest, m *pilosa.BlockDataRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.View = pb.View + m.Shard = pb.Shard + m.Block = pb.Block +} + +func decodeBlockDataResponse(pb *internal.BlockDataResponse, m *pilosa.BlockDataResponse) { + m.RowIDs = pb.RowIDs + m.ColumnIDs = pb.ColumnIDs +} + +func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { + m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets)) + decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets) + if pb.Err == "" { + m.Err = nil + } else { + m.Err = errors.New(pb.Err) + } + m.Results = make([]interface{}, len(pb.Results)) + decodeQueryResults(pb.Results, m.Results) + +} + +func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { + for i := range pb { + m[i] = &pilosa.ColumnAttrSet{} + decodeColumnAttrSet(pb[i], m[i]) + } +} + +func decodeColumnAttrSet(pb *internal.ColumnAttrSet, m *pilosa.ColumnAttrSet) { + m.ID = pb.ID + m.Key = pb.Key + m.Attrs = decodeAttrs(pb.Attrs) +} + +func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { + for i := range pb { + m[i] = decodeQueryResult(pb[i]) + } +} + +// QueryResult types. +const ( + queryResultTypeNil uint32 = iota + queryResultTypeRow + queryResultTypePairs + queryResultTypeValCount + queryResultTypeUint64 + queryResultTypeBool +) + +func decodeQueryResult(pb *internal.QueryResult) interface{} { + switch pb.Type { + case queryResultTypeRow: + return decodeRow(pb.Row) + case queryResultTypePairs: + return decodePairs(pb.Pairs) + case queryResultTypeValCount: + return decodeValCount(pb.ValCount) + case queryResultTypeUint64: + return pb.N + case queryResultTypeBool: + return pb.Changed + case queryResultTypeNil: + return nil + } + panic(fmt.Sprintf("unknown type: %d", pb.Type)) +} + +// DecodeRow converts r from its internal representation. +func decodeRow(pr *internal.Row) *pilosa.Row { + if pr == nil { + return nil + } + + r := pilosa.NewRow() + r.Attrs = decodeAttrs(pr.Attrs) + for _, v := range pr.Columns { + r.SetBit(v) + } + return r +} + +func decodeAttrs(pb []*internal.Attr) map[string]interface{} { + m := make(map[string]interface{}, len(pb)) + for i := range pb { + key, value := decodeAttr(pb[i]) + m[key] = value + } + return m +} + +const ( + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 +) + +func decodeAttr(attr *internal.Attr) (key string, value interface{}) { + switch attr.Type { + case attrTypeString: + return attr.Key, attr.StringValue + case attrTypeInt: + return attr.Key, attr.IntValue + case attrTypeBool: + return attr.Key, attr.BoolValue + case attrTypeFloat: + return attr.Key, attr.FloatValue + default: + return attr.Key, nil + } +} + +func decodePairs(a []*internal.Pair) []pilosa.Pair { + other := make([]pilosa.Pair, len(a)) + for i := range a { + other[i] = decodePair(a[i]) + } + return other +} + +func decodePair(pb *internal.Pair) pilosa.Pair { + return pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + } +} + +func decodeValCount(pb *internal.ValCount) pilosa.ValCount { + return pilosa.ValCount{ + Val: pb.Val, + Count: pb.Count, + } +} + +func encodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) + for i := range a { + other[i] = encodeColumnAttrSet(a[i]) + } + return other +} + +func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), + } +} + +func encodeRow(r *pilosa.Row) *internal.Row { + if r == nil { + return nil + } + + return &internal.Row{ + Columns: r.Columns(), + Attrs: encodeAttrs(r.Attrs), + } +} + +func encodePairs(a pilosa.Pairs) []*internal.Pair { + other := make([]*internal.Pair, len(a)) + for i := range a { + other[i] = encodePair(a[i]) + } + return other +} + +func encodePair(p pilosa.Pair) *internal.Pair { + return &internal.Pair{ + ID: p.ID, + Key: p.Key, + Count: p.Count, + } +} + +func encodeValCount(vc pilosa.ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, + } +} + +func encodeAttrs(m map[string]interface{}) []*internal.Attr { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + a := make([]*internal.Attr, len(keys)) + for i := range keys { + a[i] = encodeAttr(keys[i], m[keys[i]]) + } + return a +} + +// encodeAttr converts a key/value pair into an Attr internal representation. +func encodeAttr(key string, value interface{}) *internal.Attr { + pb := &internal.Attr{Key: key} + switch value := value.(type) { + case string: + pb.Type = attrTypeString + pb.StringValue = value + case float64: + pb.Type = attrTypeFloat + pb.FloatValue = value + case uint64: + pb.Type = attrTypeInt + pb.IntValue = int64(value) + case int64: + pb.Type = attrTypeInt + pb.IntValue = value + case bool: + pb.Type = attrTypeBool + pb.BoolValue = value + } + return pb +} diff --git a/enterprise/b/btree.go b/enterprise/b/btree.go index 3d4c09888..044411bd7 100644 --- a/enterprise/b/btree.go +++ b/enterprise/b/btree.go @@ -56,23 +56,23 @@ func init() { var ( btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} + btEPool = btEpool{sync.Pool{New: func() interface{} { return &enumerator{} }}} + btTPool = btTpool{sync.Pool{New: func() interface{} { return &tree{} }}} btXPool = sync.Pool{New: func() interface{} { return &x{} }} ) type btTpool struct{ sync.Pool } -func (p *btTpool) get(cmp Cmp) *Tree { - x := p.Get().(*Tree) +func (p *btTpool) get(cmp Cmp) *tree { + x := p.Get().(*tree) x.cmp = cmp return x } type btEpool struct{ sync.Pool } -func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *Tree, ver int64) *Enumerator { - x := p.Get().(*Enumerator) +func (p *btEpool) get(err error, hit bool, i int, k uint64, q *d, t *tree, ver int64) *enumerator { + x := p.Get().(*enumerator) x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver return x } @@ -98,26 +98,26 @@ type ( v *roaring.Container } - // Enumerator captures the state of enumerating a tree. It is returned + // enumerator captures the state of enumerating a tree. It is returned // from the Seek* methods. The enumerator is aware of any mutations // made to the tree in the process of enumerating it and automatically // resumes the enumeration at the proper key, if possible. // - // However, once an Enumerator returns io.EOF to signal "no more + // However, once an enumerator returns io.EOF to signal "no more // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an Enumerator is "sticky" (idempotent). - Enumerator struct { + // other words, io.EOF from an enumerator is "sticky" (idempotent). + enumerator struct { err error hit bool i int k uint64 q *d - t *Tree + t *tree ver int64 } - // Tree is a B+tree. - Tree struct { + // tree is a B+tree. + tree struct { c int cmp Cmp first *d @@ -140,9 +140,9 @@ type ( var ( // R/O zero values zd d zde de - ze Enumerator + ze enumerator zk uint64 - zt Tree + zt tree zx x zxe xe ) @@ -233,14 +233,14 @@ func (l *d) mvR(r *d, c int) { // ----------------------------------------------------------------------- Tree -// TreeNew returns a newly created, empty Tree. The compare function is used +// treeNew returns a newly created, empty Tree. The compare function is used // for key collation. -func TreeNew(cmp Cmp) *Tree { +func treeNew(cmp Cmp) *tree { return btTPool.get(cmp) } // Clear removes all K/V pairs from the tree. -func (t *Tree) Clear() { +func (t *tree) Clear() { if t.r == nil { return } @@ -252,13 +252,13 @@ func (t *Tree) Clear() { // Close performs Clear and recycles t to a pool for possible later reuse. No // references to t should exist or such references must not be used afterwards. -func (t *Tree) Close() { +func (t *tree) Close() { t.Clear() *t = zt btTPool.Put(t) } -func (t *Tree) cat(p *x, q, r *d, pi int) { +func (t *tree) cat(p *x, q, r *d, pi int) { t.ver++ q.mvL(r, r.c) if r.n != nil { @@ -286,7 +286,7 @@ func (t *Tree) cat(p *x, q, r *d, pi int) { t.r = q } -func (t *Tree) catX(p, q, r *x, pi int) { +func (t *tree) catX(p, q, r *x, pi int) { t.ver++ q.x[q.c].k = p.x[pi].k copy(q.x[q.c+1:], r.x[:r.c]) @@ -320,7 +320,7 @@ func (t *Tree) catX(p, q, r *x, pi int) { // Delete removes the k's KV pair, if it exists, in which case Delete returns // true. -func (t *Tree) Delete(k uint64) (ok bool) { +func (t *tree) Delete(k uint64) (ok bool) { pi := -1 var p *x q := t.r @@ -370,7 +370,7 @@ func (t *Tree) Delete(k uint64) (ok bool) { } } -func (t *Tree) extract(q *d, i int) { // (r *container) { +func (t *tree) extract(q *d, i int) { // (r *container) { t.ver++ //r = q.d[i].v // prepared for Extract q.c-- @@ -381,7 +381,7 @@ func (t *Tree) extract(q *d, i int) { // (r *container) { t.c-- } -func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { +func (t *tree) find(q interface{}, k uint64) (i int, ok bool) { var mk uint64 l := 0 switch x := q.(type) { @@ -419,7 +419,7 @@ func (t *Tree) find(q interface{}, k uint64) (i int, ok bool) { // First returns the first item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k uint64, v *roaring.Container) { +func (t *tree) First() (k uint64, v *roaring.Container) { if q := t.first; q != nil { q := &q.d[0] k, v = q.k, q.v @@ -429,7 +429,7 @@ func (t *Tree) First() (k uint64, v *roaring.Container) { // Get returns the value associated with k and true if it exists. Otherwise Get // returns (zero-value, false). -func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) { +func (t *tree) Get(k uint64) (v *roaring.Container, ok bool) { q := t.r if q == nil { return @@ -455,7 +455,7 @@ func (t *Tree) Get(k uint64) (v *roaring.Container, ok bool) { } } -func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { +func (t *tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { t.ver++ c := q.c if i < c { @@ -470,7 +470,7 @@ func (t *Tree) insert(q *d, i int, k uint64, v *roaring.Container) *d { // Last returns the last item of the tree in the key collating order, or // (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k uint64, v *roaring.Container) { +func (t *tree) Last() (k uint64, v *roaring.Container) { if q := t.last; q != nil { q := &q.d[q.c-1] k, v = q.k, q.v @@ -479,11 +479,11 @@ func (t *Tree) Last() (k uint64, v *roaring.Container) { } // Len returns the number of items in the tree. -func (t *Tree) Len() int { +func (t *tree) Len() int { return t.c } -func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { +func (t *tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ l, r := p.siblings(pi) @@ -528,7 +528,7 @@ func (t *Tree) overflow(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { // Seek returns an Enumerator positioned on an item such that k >= item's key. // ok reports if k == item.key The Enumerator's position is possibly after the // last item in the tree. -func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) { +func (t *tree) Seek(k uint64) (e *enumerator, ok bool) { q := t.r if q == nil { e = btEPool.get(nil, false, 0, k, nil, t, t.ver) @@ -558,7 +558,7 @@ func (t *Tree) Seek(k uint64) (e *Enumerator, ok bool) { // SeekFirst returns an enumerator positioned on the first KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekFirst() (e *Enumerator, err error) { +func (t *tree) SeekFirst() (e *enumerator, err error) { q := t.first if q == nil { return nil, io.EOF @@ -569,7 +569,7 @@ func (t *Tree) SeekFirst() (e *Enumerator, err error) { // SeekLast returns an enumerator positioned on the last KV pair in the tree, // if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekLast() (e *Enumerator, err error) { +func (t *tree) SeekLast() (e *enumerator, err error) { q := t.last if q == nil { return nil, io.EOF @@ -579,7 +579,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) { } // Set sets the value associated with k. -func (t *Tree) Set(k uint64, v *roaring.Container) { +func (t *tree) Set(k uint64, v *roaring.Container) { //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //defer func() { // dbg("--- POST\n%s\n====\n", t.dump()) @@ -645,7 +645,7 @@ func (t *Tree) Set(k uint64, v *roaring.Container) { // tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. -func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { +func (t *tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (newV *roaring.Container, write bool)) (oldV *roaring.Container, written bool) { pi := -1 var p *x q := t.r @@ -712,7 +712,7 @@ func (t *Tree) Put(k uint64, upd func(oldV *roaring.Container, exists bool) (new } } -func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { +func (t *tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.ver++ r := btDPool.Get().(*d) if q.n != nil { @@ -747,7 +747,7 @@ func (t *Tree) split(p *x, q *d, pi, i int, k uint64, v *roaring.Container) { t.insert(q, i, k, v) } -func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { +func (t *tree) splitX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ r := btXPool.Get().(*x) copy(r.x[:], q.x[kx+1:]) @@ -771,7 +771,7 @@ func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { return q, i } -func (t *Tree) underflow(p *x, q *d, pi int) { +func (t *tree) underflow(p *x, q *d, pi int) { t.ver++ l, r := p.siblings(pi) @@ -796,7 +796,7 @@ func (t *Tree) underflow(p *x, q *d, pi int) { t.cat(p, q, r, pi) } -func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { +func (t *tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { t.ver++ var l, r *x @@ -850,7 +850,7 @@ func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { // Close recycles e to a pool for possible later reuse. No references to e // should exist or such references must not be used afterwards. -func (e *Enumerator) Close() { +func (e *enumerator) Close() { *e = ze btEPool.Put(e) } @@ -858,7 +858,7 @@ func (e *Enumerator) Close() { // Next returns the currently enumerated item, if it exists and moves to the // next item in the key collation order. If there is no item to return, err == // io.EOF is returned. -func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) { +func (e *enumerator) Next() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } @@ -886,7 +886,7 @@ func (e *Enumerator) Next() (k uint64, v *roaring.Container, err error) { return } -func (e *Enumerator) next() error { +func (e *enumerator) next() error { if e.q == nil { e.err = io.EOF return io.EOF @@ -906,7 +906,7 @@ func (e *Enumerator) next() error { // Prev returns the currently enumerated item, if it exists and moves to the // previous item in the key collation order. If there is no item to return, err // == io.EOF is returned. -func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) { +func (e *enumerator) Prev() (k uint64, v *roaring.Container, err error) { if err = e.err; err != nil { return } @@ -941,7 +941,7 @@ func (e *Enumerator) Prev() (k uint64, v *roaring.Container, err error) { return } -func (e *Enumerator) prev() error { +func (e *enumerator) prev() error { if e.q == nil { e.err = io.EOF return io.EOF diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index ccb856d18..95fcb09b3 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -27,28 +27,28 @@ func cmp(a, b uint64) int { return int(a - b) } -type BTreeContainers struct { - tree *Tree +type bTreeContainers struct { + tree *tree lastKey uint64 lastContainer *roaring.Container } -func NewBTreeContainers() *BTreeContainers { - return &BTreeContainers{ - tree: TreeNew(cmp), +func newBTreeContainers() *bTreeContainers { + return &bTreeContainers{ + tree: treeNew(cmp), } } func NewBTreeBitmap(a ...uint64) *roaring.Bitmap { b := &roaring.Bitmap{ - Containers: NewBTreeContainers(), + Containers: newBTreeContainers(), } b.Add(a...) return b } -func (btc *BTreeContainers) Get(key uint64) *roaring.Container { +func (btc *bTreeContainers) Get(key uint64) *roaring.Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer @@ -64,7 +64,7 @@ func (btc *BTreeContainers) Get(key uint64) *roaring.Container { return c } -func (btc *BTreeContainers) Put(key uint64, c *roaring.Container) { +func (btc *bTreeContainers) Put(key uint64, c *roaring.Container) { // If a mapped container is added to the tree, reset the // lastContainer cache so that the cache is not pointing // at a read-only mmap. @@ -93,16 +93,16 @@ type updater struct { mapped bool } -func (btc *BTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { a := updater{key, containerType, n, mapped} btc.tree.Put(key, a.update) } -func (btc *BTreeContainers) Remove(key uint64) { +func (btc *bTreeContainers) Remove(key uint64) { btc.tree.Delete(key) } -func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { +func (btc *bTreeContainers) GetOrCreate(key uint64) *roaring.Container { // Check the last* cache for same container. if key == btc.lastKey && btc.lastContainer != nil { return btc.lastContainer @@ -121,8 +121,18 @@ func (btc *BTreeContainers) GetOrCreate(key uint64) *roaring.Container { return btc.lastContainer } -func (btc *BTreeContainers) Clone() roaring.Containers { - nbtc := NewBTreeContainers() +func (btc *bTreeContainers) Count() (n uint64) { + e, _ := btc.tree.Seek(0) + _, c, err := e.Next() + for err != io.EOF { + n += uint64(c.N()) + _, c, err = e.Next() + } + return +} + +func (btc *bTreeContainers) Clone() roaring.Containers { + nbtc := newBTreeContainers() itr, err := btc.tree.SeekFirst() if err == io.EOF { @@ -138,7 +148,7 @@ func (btc *BTreeContainers) Clone() roaring.Containers { return nbtc } -func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) { +func (btc *bTreeContainers) Last() (key uint64, c *roaring.Container) { if btc.tree.Len() == 0 { return 0, nil } @@ -146,34 +156,34 @@ func (btc *BTreeContainers) Last() (key uint64, c *roaring.Container) { return k, v } -func (btc *BTreeContainers) Size() int { +func (btc *bTreeContainers) Size() int { return btc.tree.Len() } -func (btc *BTreeContainers) Reset() { - btc.tree = TreeNew(cmp) +func (btc *bTreeContainers) Reset() { + btc.tree = treeNew(cmp) btc.lastKey = 0 btc.lastContainer = nil } -func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { +func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) { e, ok := btc.tree.Seek(key) if ok { found = true } - return &BTCIterator{ + return &btcIterator{ e: e, }, found } -type BTCIterator struct { - e *Enumerator +type btcIterator struct { + e *enumerator key uint64 val *roaring.Container } -func (i *BTCIterator) Next() bool { +func (i *btcIterator) Next() bool { k, v, err := i.e.Next() if err == io.EOF { @@ -184,7 +194,7 @@ func (i *BTCIterator) Next() bool { return true } -func (i *BTCIterator) Value() (uint64, *roaring.Container) { +func (i *btcIterator) Value() (uint64, *roaring.Container) { if i.val == nil { return 0, nil } diff --git a/event.go b/event.go index 5df69361b..0d5e59e99 100644 --- a/event.go +++ b/event.go @@ -14,8 +14,7 @@ package pilosa -// NodeEventType are the types of events that can be sent from the -// ChannelEventDelegate. +// NodeEventType are the types of node events. type NodeEventType int const ( @@ -29,27 +28,3 @@ type NodeEvent struct { Event NodeEventType Node *Node } - -// EventHandler is the interface for the pilosa object which knows how to -// handle broadcast messages. (Hint: this is implemented by pilosa.Server) -type EventHandler interface { - ReceiveEvent(e *NodeEvent) error -} - -// EventReceiver is the interface for the object which will listen for and -// decode broadcast messages before passing them to pilosa to handle. The -// implementation of this could be an http server which listens for messages, -// gets the protobuf payload, and then passes it to -// EventHandler.ReceiveMessage. -type EventReceiver interface { - // Start starts listening for broadcast messages - it should return - // immediately, spawning a goroutine if necessary. - Start(EventHandler) error -} - -type nopEventReceiver struct{} - -func (n *nopEventReceiver) Start(e EventHandler) error { return nil } - -// NopEventReceiver is a no-op implementation of the EventReceiver. -var NopEventReceiver = &nopEventReceiver{} diff --git a/executor.go b/executor.go index 97f42acf1..afea60112 100644 --- a/executor.go +++ b/executor.go @@ -17,56 +17,79 @@ package pilosa import ( "context" "fmt" - "net/http" "sort" "time" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) -// DefaultFrame is the frame used if one is not specified. +// defaultField is the field used if one is not specified. const ( - DefaultFrame = "general" + defaultField = "general" - // MinThreshold is the lowest count to use in a Top-N operation when + // defaultMinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. - MinThreshold = 1 + defaultMinThreshold = 1 columnLabel = "col" rowLabel = "row" ) -// Executor recursively executes calls in a PQL query across all slices. -type Executor struct { +// executor recursively executes calls in a PQL query across all shards. +type executor struct { Holder *Holder // Local hostname & cluster configuration. Node *Node - Cluster *Cluster + Cluster *cluster // Client used for remote requests. - client InternalClient + client InternalQueryClient - // Maximum number of SetBit() or ClearBit() commands per request. + // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int + + // Stores key/id translation data. + TranslateStore TranslateStore } -// NewExecutor returns a new instance of Executor. -func NewExecutor(remoteClient *http.Client) *Executor { - return &Executor{ - client: NewInternalHTTPClientFromURI(nil, remoteClient), +// executorOption is a functional option type for pilosa.Executor +type executorOption func(e *executor) error + +func optExecutorInternalQueryClient(c InternalQueryClient) executorOption { + return func(e *executor) error { + e.client = c + return nil } } +// newExecutor returns a new instance of Executor. +func newExecutor(opts ...executorOption) *executor { + e := &executor{ + client: newNopInternalQueryClient(), + } + for _, opt := range opts { + err := opt(e) + if err != nil { + panic(err) + } + } + return e +} + // Execute executes a PQL query. -func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { +func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { // Verify that an index is set. if index == "" { return nil, ErrIndexRequired } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { return nil, ErrTooManyWrites @@ -74,26 +97,49 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Default options. if opt == nil { - opt = &ExecOptions{} + opt = &execOptions{} } - // Don't bother calculating slices for query types that don't require it. - needsSlices := needsSlices(q.Calls) + // Translate query keys to ids, if necessary. + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } + } - // If slices are specified, then use that value for slices. If slices aren't + results, err := e.execute(ctx, index, q, shards, opt) + if err != nil { + return nil, err + } + + // Translate response objects from ids to keys, if necessary. + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { + // Don't bother calculating shards for query types that don't require it. + needsShards := needsShards(q.Calls) + + // If shards are specified, then use that value for shards. If shards aren't // specified, then include all of them. - if len(slices) == 0 && needsSlices { - // Round up the number of slices. + if len(shards) == 0 && needsShards { + // Round up the number of shards. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - maxSlice := idx.MaxSlice() + maxShard := idx.maxShard() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) + // Generate a slice of all shards. + shards = make([]uint64, maxShard+1) + for i := range shards { + shards[i] = uint64(i) } } @@ -105,7 +151,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - v, err := e.executeCall(ctx, index, call, slices, opt) + v, err := e.executeCall(ctx, index, call, shards, opt) if err != nil { return nil, err } @@ -115,7 +161,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } // executeCall executes a call. -func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { +func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } @@ -124,37 +170,35 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s switch c.Name { case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeSum(ctx, index, c, slices, opt) + return e.executeSum(ctx, index, c, shards, opt) case "Min": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMin(ctx, index, c, slices, opt) + return e.executeMin(ctx, index, c, shards, opt) case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMax(ctx, index, c, slices, opt) - case "ClearBit": + return e.executeMax(ctx, index, c, shards, opt) + case "Clear": return e.executeClearBit(ctx, index, c, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeCount(ctx, index, c, slices, opt) - case "SetBit": - return e.executeSetBit(ctx, index, c, opt) - case "SetFieldValue": - return nil, e.executeSetFieldValue(ctx, index, c, opt) + return e.executeCount(ctx, index, c, shards, opt) + case "Set": + return e.executeSet(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": return nil, e.executeSetColumnAttrs(ctx, index, c, opt) case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeTopN(ctx, index, c, slices, opt) + return e.executeTopN(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeBitmapCall(ctx, index, c, slices, opt) + return e.executeBitmapCall(ctx, index, c, shards, opt) } } // validateCallArgs ensures that the value types in call.Args are expected. -func (e *Executor) validateCallArgs(c *pql.Call) error { +func (e *executor) validateCallArgs(c *pql.Call) error { if _, ok := c.Args["ids"]; ok { switch v := c.Args["ids"].(type) { case []int64, []uint64: @@ -173,10 +217,8 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Sum(): frame required") - } else if field := c.Args["field"]; field == "" { +func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Sum(): field required") } @@ -185,17 +227,17 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeSumCountSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSumCountShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Add(v.(ValCount)) + return other.add(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -207,11 +249,9 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other, nil } -// executeFieldMin executes a Min() call. -func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Min(): frame required") - } else if field := c.Args["field"]; field == "" { +// executeMin executes a Min() call. +func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Min(): field required") } @@ -220,17 +260,17 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMinSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMinShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Smaller(v.(ValCount)) + return other.smaller(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -242,11 +282,9 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal return other, nil } -// executeFieldMax executes a Max() call. -func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Max(): frame required") - } else if field := c.Args["field"]; field == "" { +// executeMax executes a Max() call. +func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Max(): field required") } @@ -255,17 +293,17 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMaxSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMaxShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(ValCount) - return other.Larger(v.(ValCount)) + return other.larger(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -278,10 +316,10 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Row, error) { +func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeBitmapCallSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeBitmapCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -294,22 +332,22 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return other } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "map reduce") } - // Attach attributes for Bitmap() calls. + // Attach attributes for Row() calls. // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. row, _ := other.(*Row) - if c.Name == "Bitmap" { + if c.Name == "Row" { if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { idx := e.Holder.Index(index) if idx != nil { - if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { + if columnID, ok, err := c.UintArg("_" + columnLabel); ok && err == nil { attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, errors.Wrap(err, "getting column attrs") @@ -318,9 +356,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else if err != nil { return nil, err } else { - frame, _ := c.Args["frame"].(string) - if fr := idx.Frame(frame); fr != nil { - rowID, _, err := c.UintArg(rowLabel) + // field, _ := c.Args["field"].(string) + fieldName, _ := c.FieldArg() + if fr := idx.Field(fieldName); fr != nil { + rowID, _, err := c.UintArg(fieldName) if err != nil { return nil, errors.Wrap(err, "getting row") } @@ -336,145 +375,142 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } if opt.ExcludeColumns { - row.segments = []RowSegment{} + row.segments = []rowSegment{} } return row, nil } -// executeBitmapCallSlice executes a bitmap call for a single slice. -func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBitmapCallShard executes a bitmap call for a single shard. +func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { switch c.Name { - case "Bitmap": - return e.executeBitmapSlice(ctx, index, c, slice) + case "Row": + return e.executeBitmapShard(ctx, index, c, shard) case "Difference": - return e.executeDifferenceSlice(ctx, index, c, slice) + return e.executeDifferenceShard(ctx, index, c, shard) case "Intersect": - return e.executeIntersectSlice(ctx, index, c, slice) + return e.executeIntersectShard(ctx, index, c, shard) case "Range": - return e.executeRangeSlice(ctx, index, c, slice) + return e.executeRangeShard(ctx, index, c, shard) case "Union": - return e.executeUnionSlice(ctx, index, c, slice) + return e.executeUnionShard(ctx, index, c, shard) case "Xor": - return e.executeXorSlice(ctx, index, c, slice) + return e.executeXorShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } } -// executeSumCountSlice calculates the sum and count for fields on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeSumCountShard calculates the sum and count for bsiGroups on a shard. +func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, errors.Wrap(err, "executing bitmap call") } filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } - vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ - Val: int64(vsum) + (int64(vcount) * field.Min), + Val: int64(vsum) + (int64(vcount) * bsig.Min), Count: int64(vcount), }, nil } -// executeFieldMinSlice calculates the min for fields on a slice. -func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMinShard calculates the min for bsiGroups on a shard. +func (e *executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } - fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmin) + field.Min, + Val: int64(fmin) + bsig.Min, Count: int64(fcount), }, nil } -// executeFieldMaxSlice calculates the max for fields on a slice. -func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMaxShard calculates the max for bsiGroups on a shard. +func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } - fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmax) + field.Min, + Val: int64(fmax) + bsig.Min, Count: int64(fcount), }, nil } @@ -482,7 +518,7 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -493,7 +529,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s } // Execute original query. - pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) + pairs, err := e.executeTopNShards(ctx, index, c, shards, opt) if err != nil { return nil, errors.Wrap(err, "finding top results") } @@ -510,7 +546,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) + trimmedList, err := e.executeTopNShards(ctx, index, other, shards, opt) if err != nil { return nil, errors.Wrap(err, "retrieving full counts") } @@ -521,10 +557,10 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s return trimmedList, nil } -func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeTopNSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeTopNShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -533,7 +569,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return Pairs(other).Add(v.([]Pair)) } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -545,32 +581,32 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return results, nil } -// executeTopNSlice executes a TopN call for a single slice. -func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { - frame, _ := c.Args["frame"].(string) +// executeTopNShard executes a TopN call for a single shard. +func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { + field, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } - field, _ := c.Args["field"].(string) + attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } minThreshold, _, err := c.UintArg("threshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } - filters, _ := c.Args["filters"].([]interface{}) + attrValues, _ := c.Args["attrValues"].([]interface{}) tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } // Retrieve bitmap used to intersect. var src *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -579,42 +615,42 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca return nil, errors.New("TopN() can only have one input bitmap") } - // Set default frame. - if frame == "" { - frame = DefaultFrame + // Set default field. + if field == "" { + field = defaultField } - f := e.Holder.Fragment(index, frame, ViewStandard, slice) + f := e.Holder.fragment(index, field, viewStandard, shard) if f == nil { return nil, nil } if minThreshold <= 0 { - minThreshold = MinThreshold + minThreshold = defaultMinThreshold } if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.Top(TopOptions{ + return f.top(topOptions{ N: int(n), Src: src, RowIDs: rowIDs, - FilterField: field, - FilterValues: filters, + FilterName: attrName, + FilterValues: attrValues, MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, }) } -// executeDifferenceSlice executes a difference() call for a local slice. -func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeDifferenceShard executes a difference() call for a local shard. +func (e *executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Difference query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -625,50 +661,50 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c * other = other.Difference(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } -func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - // Fetch frame & row label based on argument. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Fetch field & row label based on argument. + fieldName, err := c.FieldArg() + if err != nil { + return nil, errors.New("Row() argument required: field") } - f := e.Holder.Frame(index, frame) + f := e.Holder.Field(index, fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } - rowID, rowOK, rowErr := c.UintArg(rowLabel) + rowID, rowOK, rowErr := c.UintArg(fieldName) if rowErr != nil { - return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr) + return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) } if !rowOK { - return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) + return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, frame, ViewStandard, slice) + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { return NewRow(), nil } - return frag.Row(rowID), nil + return frag.row(rowID), nil } -// executeIntersectSlice executes a intersect() call for a local slice. -func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeIntersectShard executes a intersect() call for a local shard. +func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Intersect query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -676,24 +712,24 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p if i == 0 { other = row } else { - other = other.Intersect(row) + other = other.intersect(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } -// executeRangeSlice executes a range() call for a local slice. -func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Handle field ranges differently. +// executeRangeShard executes a range() call for a local shard. +func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + // Handle bsiGroup ranges differently. if c.HasConditionArg() { - return e.executeFieldRangeSlice(ctx, index, c, slice) + return e.executeBSIGroupRangeShard(ctx, index, c, shard) } - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Parse field. + fieldName, err := c.FieldArg() + if err != nil { + return nil, errors.New("Range() argument required: field") } // Retrieve column label. @@ -702,23 +738,23 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return nil, ErrIndexNotFound } - // Retrieve base frame. - f := idx.Frame(frame) + // Retrieve base field. + f := idx.Field(fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Read row & column id. - rowID, rowOK, err := c.UintArg(rowLabel) + rowID, rowOK, err := c.UintArg(fieldName) if err != nil { - return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) + return nil, fmt.Errorf("executeRangeShard - reading row: %v", err) } if !rowOK { return nil, fmt.Errorf("Range() must specify %q", rowLabel) } // Parse start time. - startTimeStr, ok := c.Args["start"].(string) + startTimeStr, ok := c.Args["_start"].(string) if !ok { return nil, errors.New("Range() start time required") } @@ -728,7 +764,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Parse end time. - endTimeStr, ok := c.Args["end"].(string) + endTimeStr, ok := c.Args["_end"].(string) if !ok { return nil, errors.New("Range() end time required") } @@ -743,46 +779,32 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return &Row{}, nil } - // Union bitmaps across all time-based subframes. + // Union bitmaps across all time-based views. row := &Row{} - for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, frame, view, slice) + for _, view := range viewsByTimeRange(viewStandard, startTime, endTime, q) { + f := e.Holder.fragment(index, fieldName, view, shard) if f == nil { continue } - row = row.Union(f.Row(rowID)) + row = row.Union(f.row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil } -// executeFieldRangeSlice executes a range(field) call for a local slice. -func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - f := e.Holder.Frame(index, frame) - if f == nil { - return nil, ErrFrameNotFound - } - - // Remove frame field. - args := pql.CopyArgs(c.Args) - delete(args, "frame") - - // Only one conditional field should remain. - if len(args) == 0 { +// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. +func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + // Only one conditional should be present. + if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") - } else if len(args) > 1 { + } else if len(c.Args) > 1 { return nil, errors.New("Range(): too many arguments") } - // Extract condition field. + // Extract conditional. var fieldName string var cond *pql.Condition - for k, v := range args { + for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) @@ -790,28 +812,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * fieldName, cond = k, vv } - // EQ null (not implemented: flip frag.FieldNotNull with max ColumnID) - // NEQ null frag.FieldNotNull() - // BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween() - // BETWEEN a,b(out) BETWEEN/frag.FieldNotNull() - // EQ frag.FieldRange - // NEQ frag.FieldRange + f := e.Holder.Field(index, fieldName) + if f == nil { + return nil, ErrFieldNotFound + } + + // EQ null (not implemented: flip frag.NotNull with max ColumnID) + // NEQ null frag.NotNull() + // BETWEEN a,b(in) BETWEEN/frag.RangeBetween() + // BETWEEN a,b(out) BETWEEN/frag.NotNull() + // EQ frag.RangeOp + // NEQ frag.RangeOp // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } - return frag.FieldNotNull(field.BitDepth()) + return frag.notNull(bsig.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -826,33 +853,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } // The reason we don't just call: - // return f.FieldRangeBetween(fieldName, predicates[0], predicates[1]) - // here is because we need the call to be slice-specific. + // return f.RangeBetween(fieldName, predicates[0], predicates[1]) + // here is because we need the call to be shard-specific. - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1]) + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } // If the query is asking for the entire valid range, just return - // the not-null bitmap for the field. - if predicates[0] <= field.Min && predicates[1] >= field.Max { - return frag.FieldNotNull(field.BitDepth()) + // the not-null bitmap for the bsiGroup. + if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { + return frag.notNull(bsig.BitDepth()) } - return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } else { @@ -862,44 +889,44 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return nil, errors.New("Range(): conditions only support integer values") } - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } - baseValue, outOfRange := field.BaseValue(cond.Op, value) + baseValue, outOfRange := bsig.baseValue(cond.Op, value) if outOfRange && cond.Op != pql.NEQ { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } - // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. - if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || - (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { - return frag.FieldNotNull(field.BitDepth()) + // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. + if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || + (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { + return frag.notNull(bsig.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.FieldNotNull(field.BitDepth()) + return frag.notNull(bsig.BitDepth()) } - f.Stats.Count("range:field", 1, 1.0) - return frag.FieldRange(cond.Op, field.BitDepth(), baseValue) + f.Stats.Count("range:bsigroup", 1, 1.0) + return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) } } -// executeUnionSlice executes a union() call for a local slice. -func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeUnionShard executes a union() call for a local shard. +func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -910,15 +937,15 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C other = other.Union(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } -// executeXorSlice executes a xor() call for a local slice. -func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeXorShard executes a xor() call for a local shard. +func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -929,12 +956,12 @@ func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Cal other = other.Xor(row) } } - other.InvalidateCount() + other.invalidateCount() return other, nil } // executeCount executes a count() call. -func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { +func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { if len(c.Children) == 0 { return 0, errors.New("Count() requires an input bitmap") } else if len(c.Children) > 1 { @@ -942,8 +969,8 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + mapFn := func(shard uint64) (interface{}, error) { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err } @@ -956,7 +983,7 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, return other + v.(uint64) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return 0, err } @@ -965,49 +992,49 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, return n, nil } -// executeClearBit executes a ClearBit() call. -func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) - if !ok { - return false, errors.New("ClearBit() frame required") +// executeClearBit executes a Clear() call. +func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("Clear() argument required: field") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(fieldName) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg(fieldName) if err != nil { - return false, fmt.Errorf("reading ClearBit() row: %v", err) + return false, fmt.Errorf("reading Clear() row: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel) } - colID, ok, err := c.UintArg(columnLabel) + colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { - return false, fmt.Errorf("reading ClearBit() column: %v", err) + return false, fmt.Errorf("reading Clear() column: %v", err) } else if !ok { - return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) + return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel) } - return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt) + return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } -// executeClearBitView executes a ClearBit() call for a single view. -func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth +// executeClearBitField executes a Clear() call for a single view. +func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.ClearBit(view, rowID, colID, nil) + val, err := f.ClearBit(rowID, colID) if err != nil { return false, err } else if val { @@ -1030,60 +1057,73 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql return ret, nil } -// executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) - if !ok { - return false, errors.New("SetBit() field required: frame") +// executeSet executes a Set() call. +func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("Set() argument required: field") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(fieldName) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } - // Read fields using labels. - rowID, ok, err := c.UintArg(rowLabel) + // Read colID using labels. + colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { - return false, fmt.Errorf("reading SetBit() row: %v", err) + return false, fmt.Errorf("reading Set() column: %v", err) } else if !ok { - return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) + return false, fmt.Errorf("Set() column argument '%v' required", columnLabel) } - colID, ok, err := c.UintArg(columnLabel) - if err != nil { - return false, fmt.Errorf("reading SetBit() column: %v", err) - } else if !ok { - return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) - } - - var timestamp *time.Time - sTimestamp, ok := c.Args["timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) + if f.Type() == FieldTypeInt { + // Read remaining fields using labels. + rowVal, ok, err := c.IntArg(fieldName) if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) } - timestamp = &t - } - return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt) + return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) + } else { + // Read remaining fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) + } } -// executeSetBitView executes a SetBit() call for a specific view. -func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth +// executeSetBitField executes a Set() call for a specific field. +func (e *executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *execOptions) (bool, error) { + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.SetBit(view, rowID, colID, timestamp) + val, err := f.SetBit(rowID, colID, timestamp) if err != nil { return false, err } else if val { @@ -1107,87 +1147,53 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C return ret, nil } -// executeSetFieldValue executes a SetFieldValue() call. -func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) - if !ok { - return errors.New("SetFieldValue() frame required") - } +// executeSetValueField executes a Set() call for a specific int field. +func (e *executor) executeSetValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, value int64, opt *execOptions) (bool, error) { + shard := colID / ShardWidth + ret := false - // Retrieve frame. - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ErrFrameNotFound - } - - // Parse labels. - columnID, ok, err := c.UintArg(columnLabel) - if err != nil { - return fmt.Errorf("reading SetFieldValue() column: %v", err) - } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) - } - - // Copy args and remove reserved fields. - args := pql.CopyArgs(c.Args) - delete(args, "frame") - // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. - delete(args, columnLabel) - - // Set values. - for name, value := range args { - switch value := value.(type) { - case int64: - if _, err := frame.SetFieldValue(columnID, name, value); err != nil { - return err + for _, node := range e.Cluster.shardNodes(index, shard) { + // Update locally if host matches. + if node.ID == e.Node.ID { + val, err := f.SetValue(colID, value) + if err != nil { + return false, err + } else if val { + ret = true } - default: - return ErrInvalidFieldValueType + continue + } + + // Do not forward call if this is already being forwarded. + if opt.Remote { + continue + } + + // Forward call to remote node otherwise. + if res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + return false, err + } else { + ret = res[0].(bool) } } - frame.Stats.Count("SetFieldValue", 1, 1.0) - - // Do not forward call if this is already being forwarded. - if opt.Remote { - return nil - } - - // Execute on remote nodes in parallel. - nodes := Nodes(e.Cluster.Nodes).FilterID(e.Node.ID) - resp := make(chan error, len(nodes)) - for _, node := range nodes { - go func(node *Node) { - _, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) - resp <- err - }(node) - } - - // Return first error. - for range nodes { - if err := <-resp; err != nil { - return err - } - } - - return nil + return ret, nil } // executeSetRowAttrs executes a SetRowAttrs() call. -func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) +func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { + fieldName, ok := c.Args["_field"].(string) if !ok { - return errors.New("SetRowAttrs() frame required") + return errors.New("SetRowAttrs() field required") } - // Retrieve frame. - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ErrFrameNotFound + // Retrieve field. + field := e.Holder.Field(index, fieldName) + if field == nil { + return ErrFieldNotFound } // Parse labels. - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg("_" + rowLabel) if err != nil { return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { @@ -1196,14 +1202,14 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, "_field") + delete(attrs, "_"+rowLabel) // Set attributes. - if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { + if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1231,22 +1237,22 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. } // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. -func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { - // Collect attributes by frame/id. +func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *execOptions) ([]interface{}, error) { + // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["_field"].(string) if !ok { - return nil, errors.New("SetRowAttrs() frame required") + return nil, errors.New("SetRowAttrs() field required") } - // Retrieve frame. - f := e.Holder.Frame(index, frame) + // Retrieve field. + f := e.Holder.Field(index, field) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } - rowID, ok, err := c.UintArg(rowLabel) + rowID, ok, err := c.UintArg("_" + rowLabel) if err != nil { return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { @@ -1255,20 +1261,20 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") - delete(attrs, rowLabel) + delete(attrs, "_field") + delete(attrs, "_"+rowLabel) - // Create frame group, if not exists. - frameMap := m[frame] - if frameMap == nil { - frameMap = make(map[uint64]map[string]interface{}) - m[frame] = frameMap + // Create field group, if not exists. + fieldMap := m[field] + if fieldMap == nil { + fieldMap = make(map[uint64]map[string]interface{}) + m[field] = fieldMap } // Set or merge attributes. - attr := frameMap[rowID] + attr := fieldMap[rowID] if attr == nil { - frameMap[rowID] = cloneAttrs(attrs) + fieldMap[rowID] = cloneAttrs(attrs) } else { for k, v := range attrs { attr[k] = v @@ -1276,19 +1282,19 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } } - // Bulk insert attributes by frame. - for name, frameMap := range m { - // Retrieve frame. - frame := e.Holder.Frame(index, name) - if frame == nil { - return nil, ErrFrameNotFound + // Bulk insert attributes by field. + for name, fieldMap := range m { + // Retrieve field. + field := e.Holder.Field(index, name) + if field == nil { + return nil, ErrFieldNotFound } // Set attributes. - if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { + if err := field.RowAttrStore().SetBulkAttrs(fieldMap); err != nil { return nil, err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1318,22 +1324,22 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } // executeSetColumnAttrs executes a SetColumnAttrs() call. -func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { +func (e *executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { // Retrieve index. idx := e.Holder.Index(index) if idx == nil { return ErrIndexNotFound } - col, okCol, errCol := c.UintArg(columnLabel) + col, okCol, errCol := c.UintArg("_" + columnLabel) if errCol != nil || !okCol { return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, columnLabel) - delete(attrs, "frame") + delete(attrs, "_"+columnLabel) + delete(attrs, "field") // Set attributes. if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { @@ -1365,12 +1371,12 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return nil } -// exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +// exec executes a PQL query remotely for a set of shards on a node. +func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *execOptions) (results []interface{}, err error) { // Encode request object. - pbreq := &internal.QueryRequest{ + pbreq := &QueryRequest{ Query: q.String(), - Slices: slices, + Shards: shards, Remote: true, } @@ -1379,65 +1385,32 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * return nil, err } - // Return an error, if specified on response. - if err := decodeError(pb.Err); err != nil { - return nil, err - } - - // Return appropriate data for the query. - results = make([]interface{}, len(q.Calls)) - for i, call := range q.Calls { - var v interface{} - var err error - - switch call.Name { - case "Average", "Sum": - v, err = decodeValCount(pb.Results[i].GetValCount()), nil - case "TopN": - v, err = decodePairs(pb.Results[i].GetPairs()), nil - case "Count": - v, err = pb.Results[i].N, nil - case "SetBit": - v, err = pb.Results[i].Changed, nil - case "ClearBit": - v, err = pb.Results[i].Changed, nil - case "SetRowAttrs": - case "SetColumnAttrs": - default: - v, err = decodeRow(pb.Results[i].GetRow()), nil - } - if err != nil { - return nil, err - } - - results[i] = v - } - return results, nil + return pb.Results, pb.Err } -// slicesByNode returns a mapping of nodes to slices. -// Returns errSliceUnavailable if a slice cannot be allocated to a node. -func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) { +// shardsByNode returns a mapping of nodes to shards. +// Returns errShardUnavailable if a shard cannot be allocated to a node. +func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) { m := make(map[*Node][]uint64) loop: - for _, slice := range slices { - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, shard := range shards { + for _, node := range e.Cluster.shardNodes(index, shard) { if Nodes(nodes).Contains(node) { - m[node] = append(m[node], slice) + m[node] = append(m[node], shard) continue loop } } - return nil, errSliceUnavailable + return nil, errShardUnavailable } return m, nil } // mapReduce maps and reduces data across the cluster. // -// If a mapping of slices to a node fails then the slices are resplit across +// If a mapping of shards to a node fails then the shards are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. -func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { +func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { ch := make(chan mapResponse) // Wrap context with a cancel to kill goroutines on exit. @@ -1452,33 +1425,34 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} + nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)} } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { return nil, errors.Wrap(err, "starting mapper") } // Iterate over all map responses and reduce. var result interface{} - var maxSlice int + var maxShard int for { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, errors.Wrap(ctx.Err(), "context done") case resp := <-ch: // On error retry against remaining nodes. If an error returns then // the context will cancel and cause all open goroutines to return. + if resp.err != nil { // Filter out unavailable nodes. nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { - return nil, err + return nil, errors.Wrap(err, "calling mapper") } continue } @@ -1486,32 +1460,32 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, // Reduce value. result = reduceFn(result, resp.result) - // If all slices have been processed then return. - maxSlice += len(resp.slices) - if maxSlice >= len(slices) { + // If all shards have been processed then return. + maxShard += len(resp.shards) + if maxShard >= len(shards) { return result, nil } } } } -func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { - // Group slices together by nodes. - m, err := e.slicesByNode(nodes, index, slices) +func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { + // Group shards together by nodes. + m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return err + return errors.Wrap(err, "shards by node") } // Execute each node in a separate goroutine. - for n, nodeSlices := range m { - go func(n *Node, nodeSlices []uint64) { - resp := mapResponse{node: n, slices: nodeSlices} + for n, nodeShards := range m { + go func(n *Node, nodeShards []uint64) { + resp := mapResponse{node: n, shards: nodeShards} - // Send local slices to mapper, otherwise remote exec. + // Send local shards to mapper, otherwise remote exec. if n.ID == e.Node.ID { - resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) + resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, opt) if len(results) > 0 { resp.result = results[0] } @@ -1523,30 +1497,30 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod case <-ctx.Done(): case ch <- resp: } - }(n, nodeSlices) + }(n, nodeShards) } return nil } // mapperLocal performs map & reduce entirely on the local node. -func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { - ch := make(chan mapResponse, len(slices)) +func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { + ch := make(chan mapResponse, len(shards)) - for _, slice := range slices { - go func(slice uint64) { - result, err := mapFn(slice) + for _, shard := range shards { + go func(shard uint64) { + result, err := mapFn(shard) // Return response to the channel. select { case <-ctx.Done(): case ch <- mapResponse{result: result, err: err}: } - }(slice) + }(shard) } // Reduce results - var maxSlice int + var maxShard int var result interface{} for { select { @@ -1557,33 +1531,136 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu return nil, resp.err } result = reduceFn(result, resp.result) - maxSlice++ + maxShard++ } - // Exit once all slices are processed. - if maxSlice == len(slices) { + // Exit once all shards are processed. + if maxShard == len(shards) { return result, nil } } } -// errSliceUnavailable is a marker error if no nodes are available. -var errSliceUnavailable = errors.New("slice unavailable") +func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { + var colKey, rowKey, fieldName string + if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + // Positional args in new PQL syntax require special handling here. + colKey = "_" + columnLabel + fieldName, _ = c.FieldArg() + rowKey = fieldName + } else { + colKey = "col" + fieldName = callArgString(c, "field") + rowKey = "row" + } + // Translate column key. + if idx.Keys() { + if c.Args[colKey] != nil && !isString(c.Args[colKey]) { + return errors.New("column value must be a string when index 'keys' option enabled") + } + if value := callArgString(c, colKey); value != "" { + ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) + if err != nil { + return err + } + c.Args[colKey] = ids[0] + } + } else { + if isString(c.Args[colKey]) { + return errors.New("string 'col' value not allowed unless index 'keys' option enabled") + } + } -type mapFunc func(slice uint64) (interface{}, error) + // Translate row key, if field is specified & key exists. + if fieldName != "" { + field := idx.Field(fieldName) + if field == nil { + return ErrFieldNotFound + } + if field.keys() { + if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { + return errors.New("row value must be a string when field 'keys' option enabled") + } + if value := callArgString(c, rowKey); value != "" { + ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) + if err != nil { + return err + } + c.Args[rowKey] = ids[0] + } + } else { + if isString(c.Args[rowKey]) { + return errors.New("string 'row' value not allowed unless field 'keys' option enabled") + } + } + } + + // Translate child calls. + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return err + } + } + + return nil +} + +func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { + switch result := result.(type) { + case *Row: + if idx.Keys() { + other := &Row{Attrs: result.Attrs} + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + key, err := e.TranslateStore.TranslateColumnToString(index, col) + if err != nil { + return nil, err + } + other.Keys = append(other.Keys, key) + } + } + return other, nil + } + + case []Pair: + if fieldName := callArgString(call, "_field"); fieldName != "" { + field := idx.Field(fieldName) + if field == nil { + return nil, ErrFieldNotFound + } + if field.keys() { + other := make([]Pair, len(result)) + for i := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) + if err != nil { + return nil, err + } + other[i] = Pair{Key: key, Count: result[i].Count} + } + return other, nil + } + } + } + return result, nil +} + +// errShardUnavailable is a marker error if no nodes are available. +var errShardUnavailable = errors.New("shard unavailable") + +type mapFunc func(shard uint64) (interface{}, error) type reduceFunc func(prev, v interface{}) interface{} type mapResponse struct { node *Node - slices []uint64 + shards []uint64 result interface{} err error } -// ExecOptions represents an execution context for a single Execute() call. -type ExecOptions struct { +// execOptions represents an execution context for a single Execute() call. +type execOptions struct { Remote bool ExcludeRowAttrs bool ExcludeColumns bool @@ -1612,13 +1689,13 @@ func hasOnlySetRowAttrs(calls []*pql.Call) bool { return true } -func needsSlices(calls []*pql.Call) bool { +func needsShards(calls []*pql.Call) bool { if len(calls) == 0 { return false } for _, call := range calls { switch call.Name { - case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs": + case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue case "Count", "TopN": return true @@ -1636,29 +1713,15 @@ type ValCount struct { Count int64 `json:"count"` } -func (vc *ValCount) Add(other ValCount) ValCount { +func (vc *ValCount) add(other ValCount) ValCount { return ValCount{ Val: vc.Val + other.Val, Count: vc.Count + other.Count, } } -func encodeValCount(vc ValCount) *internal.ValCount { - return &internal.ValCount{ - Val: vc.Val, - Count: vc.Count, - } -} - -func decodeValCount(pb *internal.ValCount) ValCount { - return ValCount{ - Val: pb.Val, - Count: pb.Count, - } -} - -// Smaller returns the smaller of the two ValCounts. -func (vc *ValCount) Smaller(other ValCount) ValCount { +// smaller returns the smaller of the two ValCounts. +func (vc *ValCount) smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } @@ -1668,8 +1731,8 @@ func (vc *ValCount) Smaller(other ValCount) ValCount { } } -// Larger returns the larger of the two ValCounts. -func (vc *ValCount) Larger(other ValCount) ValCount { +// larger returns the larger of the two ValCounts. +func (vc *ValCount) larger(other ValCount) ValCount { if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } @@ -1678,3 +1741,17 @@ func (vc *ValCount) Larger(other ValCount) ValCount { Count: vc.Count, } } + +func callArgString(call *pql.Call, key string) string { + value, ok := call.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} + +func isString(v interface{}) bool { + _, ok := v.(string) + return ok +} diff --git a/executor_test.go b/executor_test.go index 0807228ac..41341b814 100644 --- a/executor_test.go +++ b/executor_test.go @@ -19,279 +19,400 @@ import ( "fmt" "reflect" "strconv" + "strings" "testing" + "time" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" ) // Ensure a bitmap query can be executed. func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) + f, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), - ), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { t.Fatal(err) } if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeColumns: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("Column", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), - ), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { t.Fatal(err) } - if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } }) + + t.Run("Keys", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } + + _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `` + + `Set("foo", f="bar")` + "\n" + + `Set("foo", f="baz")` + "\n" + + `Set("bat", f="bar")` + "\n" + + `Set("aaa", f="bbb")` + "\n", + }) + if err != nil { + t.Fatalf("querying: %v", err) + } + + if results, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `Row(f="bar")`, + }); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(results.Results, []interface{}{ + &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { + t.Fatal(diff) + } + }) } // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty difference query behaves properly. func TestExecutor_Execute_Empty_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference()`}); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty intersect query behaves properly. func TestExecutor_Execute_Empty_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect()`}); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty union query behaves properly. func TestExecutor_Execute_Empty_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union()`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a xor query can be executed. func TestExecutor_Execute_Xor(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+2) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { t.Fatal(err) - } else if res[0] != uint64(3) { - t.Fatalf("unexpected n: %d", res[0]) + } else if res.Results[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res.Results[0]) } } // Ensure a set query can be executed. func TestExecutor_Execute_SetBit(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("ID", func(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if n := f.Row(11).Count(); n != 0 { - t.Fatalf("unexpected bitmap count: %d", n) - } + t.Run("OK", func(t *testing.T) { + hldr.ClearBit("i", "f", 11, 1) + if n := hldr.Row("i", "f", 11).Count(); n != 0 { + t.Fatalf("unexpected bitmap count: %d", n) + } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { - t.Fatal(err) - } else { - if !res[0].(bool) { - t.Fatalf("expected column changed") - } - } + if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil { + t.Fatal(err) + } else { + if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + } - if n := f.Row(11).Count(); n != 1 { - t.Fatalf("unexpected bitmap count: %d", n) - } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { - t.Fatal(err) - } else { - if res[0].(bool) { - t.Fatalf("expected column unchanged") - } + if n := hldr.Row("i", "f", 11).Count(); n != 1 { + t.Fatalf("unexpected bitmap count: %d", n) + } + if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil { + t.Fatal(err) + } else { + if res.Results[0].(bool) { + t.Fatalf("expected column unchanged") + } + } + }) + + t.Run("ErrInvalidColValueType", func(t *testing.T) { + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=1)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { + t.Fatalf("The error is: '%v'", err) + } + }) + + t.Run("ErrInvalidRowValueType", func(t *testing.T) { + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2, f="bar")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { + t.Fatal(err) + } + }) + }) + + t.Run("Keys", func(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + + t.Run("OK", func(t *testing.T) { + hldr.SetBit("i", "f", 1, 0) + if n := hldr.Row("i", "f", 11).Count(); n != 0 { + t.Fatalf("unexpected bitmap count: %d", n) + } + + if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil { + t.Fatal(err) + } else { + if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + } + + if n := hldr.Row("i", "f", 11).Count(); n != 1 { + t.Fatalf("unexpected bitmap count: %d", n) + } + if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil { + t.Fatal(err) + } else { + if res.Results[0].(bool) { + t.Fatalf("expected column unchanged") + } + } + }) + + t.Run("ErrInvalidColValueType", func(t *testing.T) { + if err := index.DeleteField("f"); err != nil { + t.Fatal(err) + } + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `column value must be a string when index 'keys' option enabled` { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidRowValueType", func(t *testing.T) { + index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{}) + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `row value must be a string when field 'keys' option enabled` { + t.Fatal(err) + } + }) + }) +} + +// Ensure old PQL syntax doesn't break anything too badly. +func TestExecutor_Execute_OldPQL(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err) } } -// Ensure a SetFieldValue() query can be executed. -func TestExecutor_Execute_SetFieldValue(t *testing.T) { +// Ensure a SetValue() query can be executed. +func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - // Create frames. + // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, - }, - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - // Set field values. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + // Set bsiGroup values. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { t.Fatal(err) } - f := hldr.Frame("i", "f") - if value, exists, err := f.FieldValue(10, "field0"); err != nil { + f := hldr.Field("i", "f") + if value, exists, err := f.Value(10); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -299,15 +420,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.FieldValue(10, "field1"); err != nil { - t.Fatal(err) - } else if !exists { - t.Fatal("expected value to exist") - } else if value != 2 { - t.Fatalf("unexpected value: %v", value) - } - - if value, exists, err := f.FieldValue(100, "field0"); err != nil { + if value, exists, err := f.Value(100); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -317,41 +430,29 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { }) t.Run("", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, - }, - }); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { t.Fatal(err) } - t.Run("ErrFrameRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `field not found` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrColumnFieldRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrColumnFieldValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrInvalidFieldValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) @@ -360,34 +461,34 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Ensure a SetRowAttrs() query can be executed. func TestExecutor_Execute_SetRowAttrs(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - // Create frames. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - // Set two fields on f/10. - // Also set fields on other bitmaps and frames to test isolation. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + // Set two attrs on f/10. + // Also set attrs on other bitmaps and fields to test isolation. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { t.Fatal(err) } - f := hldr.Frame("i", "f") + f := hldr.Field("i", "f") if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { @@ -397,64 +498,108 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + t.Run("ID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - // Set columns for rows 0, 10, & 20 across two slices. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=1) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, row=10, col=0) - SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, row=0, col=0) - `), nil, nil); err != nil { - t.Fatal(err) - } - - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - - t.Run("Standard", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(1, f=0) + Set(` + strconv.Itoa(ShardWidth) + `, f=0) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=0) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=0) + Set(0, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=20) + Set(0, other=0) + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ {ID: 0, Count: 5}, {ID: 10, Count: 2}, }) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) + + t.Run("Keys", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("a", f="foo") + Set("b", f="foo") + Set("c", f="foo") + Set("d", f="foo") + Set("e", f="foo") + Set("a", f="bar") + Set("b", f="bar") + Set("b", f="baz") + Set("a", other="foo") + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(result.Results, []interface{}{ + []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + }); diff != "" { + t.Fatal(diff) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + // Set columns for rows 0, 10, & 20 across two shards. + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, 2) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 1, ShardWidth+2) + hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -463,32 +608,32 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Ensure func TestExecutor_Execute_TopN_fill_small(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 0, 2*ShardWidth) + hldr.SetBit("i", "f", 0, 3*ShardWidth) + hldr.SetBit("i", "f", 0, 4*ShardWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.SetBit("i", "f", 2, ShardWidth) + hldr.SetBit("i", "f", 2, ShardWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*ShardWidth) + hldr.SetBit("i", "f", 3, 2*ShardWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*ShardWidth) + hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -497,33 +642,34 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + // Set columns for rows 0, 10, & 20 across two shards. + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth) + hldr.SetBit("i", "f", 20, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth+2) // Create an intersecting row. - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.SetBit("i", "other", 100, ShardWidth) + hldr.SetBit("i", "other", 100, ShardWidth+1) + hldr.SetBit("i", "other", 100, ShardWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, {ID: 10, Count: 2}, {ID: 0, Count: 1}, @@ -534,20 +680,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { //Ensure TopN handles Attribute filters func TestExecutor_Execute_TopN_Attr(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, ShardWidth) - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -557,20 +702,20 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { //Ensure TopN handles Attribute filters with source bitmap func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, ShardWidth) + + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -579,39 +724,39 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure Min() and Max() queries can be executed. func TestExecutor_Execute_MinMax(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, - }, - }); err != nil { + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=3) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetBit(frame=f, row=1, col=1) - SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + t.Fatal(err) + } - SetFieldValue(frame=f, foo=20, col=0) - SetFieldValue(frame=f, foo=-5, col=1) - SetFieldValue(frame=f, foo=-5, col=2) - SetFieldValue(frame=f, foo=10, col=3) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - `), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(3, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + Set(1, x=1) + Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) + + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) + `}); err != nil { t.Fatal(err) } @@ -622,20 +767,20 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: -5, cnt: 2}, - {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Min(frame=f, field=foo)` + pql = `Min(field=f)` } else { - pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -648,20 +793,20 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Max(frame=f, field=foo)` + pql = `Max(field=f)` } else { - pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -670,58 +815,58 @@ func TestExecutor_Execute_MinMax(t *testing.T) { // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, - }); err != nil { + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, - }); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } - SetFieldValue(frame=f, foo=20, bar=2000, col=0) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) - `), nil, nil); err != nil { + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + `}); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -729,217 +874,217 @@ func TestExecutor_Execute_Sum(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - TimeQuantum: pilosa.TimeQuantum("YMDH"), - }); err != nil { + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { t.Fatal(err) } // Set columns. - if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") + cc := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) - SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") - `), nil, nil); err != nil { + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) + t.Run("Clear", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f=1)`}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) } -// Ensure a Range(field) query can be executed. -func TestExecutor_Execute_FieldRange(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) +// Ensure a Range(bsiGroup) query can be executed. +func TestExecutor_Execute_BSIGroupRange(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, - }); err != nil { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, - }); err != nil { + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, - }, - }); err != nil { + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } - SetFieldValue(frame=f, foo=20, bar=2000, col=50) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) - SetFieldValue(frame=edge, foo=100, col=0) - SetFieldValue(frame=edge, foo=-100, col=1) - `), nil, nil); err != nil { + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) + + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) + `}); err != nil { t.Fatal(err) } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != null)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo != 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns()) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo < 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo <= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo > 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo >= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) - // Ensure that the FieldNotNull code path gets run. - t.Run("FieldNotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { + // Ensure that the NotNull code path gets run. + t.Run("NotNull", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 0)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge < 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) - } - }) - - t.Run("ErrFrameNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -200)`}); err != nil { t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -947,288 +1092,153 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - c := test.NewCluster(2) + c := test.MustRunCluster(t, 2, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + ) + defer c.Close() + hldr0 := test.Holder{Holder: c[0].Server.Holder()} + hldr1 := test.Holder{Holder: c[1].Server.Holder()} - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { - t.Fatal(err) + t.Fatalf("creating index: %v", err) } - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", row=10)` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - - // Set columns in slice 0 & 2. - r := pilosa.NewRow( - (0*SliceWidth)+1, - (0*SliceWidth)+2, - (2*SliceWidth)+4, - ) - return []interface{}{r}, nil + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field: %v", err) } - // Create local executor data. - // The local node owns slice 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr1.MustSetBits("i", "f", 10, ShardWidth+1, ShardWidth+2, (3*ShardWidth)+4) + hldr0.SetBit("i", "f", 10, 1) - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 1, ShardWidth + 2, (3 * ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } -} -// Ensure a remote query can return a count. -func TestExecutor_Execute_Remote_Count(t *testing.T) { - c := test.NewCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(10)}, nil - } - - // Create local executor data. The local node owns slice 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) - - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { - t.Fatal(err) - } else if res[0] != uint64(12) { - t.Fatalf("unexpected n: %d", res[0]) - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - c := test.NewCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10)` { - t.Fatalf("unexpected query: %s", query.String()) + t.Run("Count", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + t.Fatal(err) + } else if res.Results[0] != uint64(4) { + t.Fatalf("unexpected n: %d", res.Results[0]) } - remoteCalled = true - return []interface{}{nil}, nil - } + }) - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - c := test.NewCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { - t.Fatalf("unexpected query: %s", query.String()) - } - remoteCalled = true - return []interface{}{nil}, nil - } - - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create frame. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if err := f.SetTimeQuantum("Y"); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can return a top-n query. -func TestExecutor_Execute_Remote_TopN(t *testing.T) { - c := test.NewCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if !reflect.DeepEqual(slices, []uint64{1, 3}) { - t.Fatalf("unexpected slices: %+v", slices) + t.Run("Remote SetBit", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) } - // Query should be executed twice. Once to get the top bitmaps for the - // slices and a second time to get the counts for a set of bitmaps. - switch remoteExecN { - case 0: - if query.String() != `TopN(frame="f", n=3)` { - t.Fatalf("unexpected query(0): %s", query.String()) - } - case 1: - if query.String() != `TopN(frame="f", ids=[0,10,30], n=3)` { - t.Fatalf("unexpected query(1): %s", query.String()) - } - default: - t.Fatalf("too many remote exec calls") + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } - remoteExecN++ + }) - // Return pair counts. - return []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, - {ID: 30, Count: 2}, - }}, nil - } + t.Run("remote with timestamp", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + if err != nil { + t.Fatalf("creating field: %v", err) + } - // Create local executor data on slice 2 & 4. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 30, Count: 4}, - {ID: 10, Count: 2}, - }}) { - t.Fatalf("unexpected results: %s", spew.Sdump(res)) - } + if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns()) + } + }) + + t.Run("remote topn", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` +Set(500001, fn=5) +Set(1500001, fn=5) +Set(2500001, fn=5) +Set(3500001, fn=5) +Set(1500001, fn=3) +Set(1500002, fn=3) +Set(3500003, fn=3) +Set(500001, fn=4) +Set(4500001, fn=4) +`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } + err := c[0].API.RecalculateCaches(context.Background()) + if err != nil { + t.Fatalf("recalcing caches: %v", err) + } + + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `TopN(fn, n=3)`, + }); err != nil { + t.Fatalf("topn querying: %v", err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 5, Count: 4}, + {ID: 3, Count: 3}, + {ID: 4, Count: 2}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + }) + + t.Run("remote setrowattrs", func(t *testing.T) { + if _, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)`, + }); err != nil { + t.Fatalf("setrowattrs querying: %v", err) + } else if attrst, err := hldr0.RowAttrStore("i", "f").Attrs(10); err != nil || !attrst["bat"].(bool) || attrst["baz"].(int64) != 123 { + t.Fatalf("wrong attrs: %v", attrst) + } + }) } // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - e.MaxWritesPerRequest = 3 - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { + c := test.MustNewCluster(t, 1) + c[0].Config.MaxWritesPerRequest = 3 + err := c.Start() + if err != nil { + t.Fatal(err) + } + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set() Clear() Set() Set()`}); errors.Cause(err) != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } -// Ensure SetColumnAttrs doesn't save `frame` as an attribute -func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() +// Ensure SetColumnAttrs doesn't save `field` as an attribute +func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FrameOptions{}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatalf("creating field: %v", err) + } targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) + // SetColumnAttrs call should exclude the field attribute + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -1240,12 +1250,12 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { t.Fatalf("%#v != %#v", targetAttrs, attrs) } - // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) + // SetColumnAttrs call should not break if field is not specified + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -1258,3 +1268,64 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { } } + +func TestExecutor_Time_Clear_Quantums(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + var rangeTests = []struct { + quantum pilosa.TimeQuantum + expected []uint64 + }{ + {quantum: "Y", expected: []uint64{3, 4, 5, 6}}, + {quantum: "M", expected: []uint64{3, 4, 6}}, + {quantum: "D", expected: []uint64{3, 4, 5, 6}}, + {quantum: "H", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "YM", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "MD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "MDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "DH", expected: []uint64{3, 4, 5, 6, 7}}, + } + populateBatch := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + ` + clearColumn := `Clear( 2, f=1)` + rangeCheckQuery := `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)` + + for i, tt := range rangeTests { + t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) { + // Create index. + indexName := strings.ToLower(string(tt.quantum)) + index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil { + t.Fatal(err) + } + // Populate + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: populateBatch}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: clearColumn}); err != nil { + t.Fatal(err) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: rangeCheckQuery}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, tt.expected) { + t.Fatalf("unexpected columns: %+v", columns) + } + + }) + } + +} diff --git a/field.go b/field.go new file mode 100644 index 000000000..a435148d3 --- /dev/null +++ b/field.go @@ -0,0 +1,1310 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" +) + +// Default field settings. +const ( + DefaultFieldType = FieldTypeSet + + DefaultCacheType = CacheTypeRanked + + // Default ranked field cache + DefaultCacheSize = 50000 + + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 + +) + +// Field types. +const ( + FieldTypeSet = "set" + FieldTypeInt = "int" + FieldTypeTime = "time" +) + +// Field represents a container for views. +type Field struct { + mu sync.RWMutex + path string + index string + name string + + viewMap map[string]*view + + // Row attribute storage and cache + rowAttrStore AttrStore + + broadcaster broadcaster + Stats StatsClient + + // Field options. + options FieldOptions + + bsiGroups []*bsiGroup + + logger Logger +} + +// FieldOption is a functional option type for pilosa.fieldOptions. +type FieldOption func(fo *FieldOptions) error + +func OptFieldKeys() FieldOption { + return func(fo *FieldOptions) error { + fo.Keys = true + return nil + } +} + +func OptFieldTypeDefault() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeSet + fo.CacheType = DefaultCacheType + fo.CacheSize = DefaultCacheSize + return nil + } +} + +func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeSet + fo.CacheType = cacheType + fo.CacheSize = cacheSize + return nil + } +} + +func OptFieldTypeInt(min, max int64) FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + if min > max { + return ErrInvalidBSIGroupRange + } + fo.Type = FieldTypeInt + fo.Min = min + fo.Max = max + return nil + } +} + +func OptFieldTypeTime(timeQuantum TimeQuantum) FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + if !timeQuantum.Valid() { + return ErrInvalidTimeQuantum + } + fo.Type = FieldTypeTime + fo.TimeQuantum = timeQuantum + return nil + } +} + +// NewField returns a new instance of field. +func NewField(path, index, name string, opts FieldOption) (*Field, error) { + err := validateName(name) + if err != nil { + return nil, err + } + + // Apply functional option. + fo := FieldOptions{} + err = opts(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + + f := &Field{ + path: path, + index: index, + name: name, + + viewMap: make(map[string]*view), + + rowAttrStore: nopStore, + + broadcaster: NopBroadcaster, + Stats: NopStatsClient, + + options: applyDefaultOptions(fo), + + logger: NopLogger, + } + return f, nil +} + +// Name returns the name the field was initialized with. +func (f *Field) Name() string { return f.name } + +// Index returns the index name the field was initialized with. +func (f *Field) Index() string { return f.index } + +// Path returns the path the field was initialized with. +func (f *Field) Path() string { return f.path } + +// RowAttrStore returns the attribute storage. +func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } + +// maxShard returns the max shard in the field. +func (f *Field) maxShard() uint64 { + f.mu.RLock() + defer f.mu.RUnlock() + + var max uint64 + for _, view := range f.viewMap { + if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max { + max = viewMaxShard + } + } + return max +} + +// Type returns the field type. +func (f *Field) Type() string { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Type +} + +// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultCacheSize 50000 +func (f *Field) SetCacheSize(v uint32) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == 0 || f.options.CacheSize == v { + return nil + } + + // Persist meta data to disk on change. + f.options.CacheSize = v + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving") + } + + return nil +} + +// CacheSize returns the ranked field cache size. +func (f *Field) CacheSize() uint32 { + f.mu.RLock() + v := f.options.CacheSize + f.mu.RUnlock() + return v +} + +// Options returns all options for this field. +func (f *Field) Options() FieldOptions { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options +} + +// Open opens and initializes the field. +func (f *Field) Open() error { + if err := func() error { + // Ensure the field's path exists. + if err := os.MkdirAll(f.path, 0777); err != nil { + return errors.Wrap(err, "creating field dir") + } + + if err := f.loadMeta(); err != nil { + return errors.Wrap(err, "loading meta") + } + + // Apply the field options loaded from meta. + if err := f.applyOptions(f.options); err != nil { + return errors.Wrap(err, "applying options") + } + + if err := f.openViews(); err != nil { + return errors.Wrap(err, "opening views") + } + + if err := f.rowAttrStore.Open(); err != nil { + return errors.Wrap(err, "opening attrstore") + } + + return nil + }(); err != nil { + f.Close() + return err + } + + return nil +} + +// openViews opens and initializes the views inside the field. +func (f *Field) openViews() error { + file, err := os.Open(filepath.Join(f.path, "views")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "opening view directory") + } + defer file.Close() + + fis, err := file.Readdir(0) + if err != nil { + return errors.Wrap(err, "reading directory") + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + + name := filepath.Base(fi.Name()) + view := f.newView(f.viewPath(name), name) + if err := view.open(); err != nil { + return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) + } + view.rowAttrStore = f.rowAttrStore + f.viewMap[view.name] = view + } + + return nil +} + +// loadMeta reads meta data for the field, if any. +func (f *Field) loadMeta() error { + var pb internal.FieldOptions + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "reading meta") + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return errors.Wrap(err, "unmarshaling") + } + } + + // Copy metadata fields. + f.options.Type = pb.Type + f.options.CacheType = pb.CacheType + f.options.CacheSize = pb.CacheSize + f.options.Min = pb.Min + f.options.Max = pb.Max + f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) + f.options.Keys = pb.Keys + + return nil +} + +// saveMeta writes meta data for the field. +func (f *Field) saveMeta() error { + // Marshal metadata. + fo := f.options + buf, err := proto.Marshal(fo.encode()) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { + return errors.Wrap(err, "writing meta") + } + + return nil +} + +// applyOptions configures the field based on opt. +func (f *Field) applyOptions(opt FieldOptions) error { + switch opt.Type { + case FieldTypeSet, "": + f.options.Type = FieldTypeSet + if opt.CacheType != "" { + f.options.CacheType = opt.CacheType + } + if opt.CacheSize != 0 { + f.options.CacheSize = opt.CacheSize + } + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + f.options.Keys = opt.Keys + case FieldTypeInt: + f.options.Type = opt.Type + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = opt.Min + f.options.Max = opt.Max + f.options.TimeQuantum = "" + f.options.Keys = opt.Keys + + // Create new bsiGroup. + bsig := &bsiGroup{ + Name: f.name, + Type: bsiGroupTypeInt, + Min: opt.Min, + Max: opt.Max, + } + // Validate bsiGroup. + if err := bsig.validate(); err != nil { + return err + } + if err := f.createBSIGroup(bsig); err != nil { + return errors.Wrap(err, "creating bsigroup") + } + case FieldTypeTime: + f.options.Type = opt.Type + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + f.options.Keys = opt.Keys + // Set the time quantum. + if err := f.setTimeQuantum(opt.TimeQuantum); err != nil { + f.Close() + return errors.Wrap(err, "setting time quantum") + } + default: + return errors.New("invalid field type") + } + + return nil +} + +// Close closes the field and its views. +func (f *Field) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + + // Close the attribute store. + if f.rowAttrStore != nil { + _ = f.rowAttrStore.Close() + } + + // Close all views. + for _, view := range f.viewMap { + if err := view.close(); err != nil { + return err + } + } + f.viewMap = make(map[string]*view) + + return nil +} + +// keys returns true if the field uses string keys. +func (f *Field) keys() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Keys +} + +// bsiGroup returns a bsiGroup by name. +func (f *Field) bsiGroup(name string) *bsiGroup { + f.mu.RLock() + defer f.mu.RUnlock() + for _, bsig := range f.bsiGroups { + if bsig.Name == name { + return bsig + } + } + return nil +} + +// hasBSIGroup returns true if a bsiGroup exists on the field. +func (f *Field) hasBSIGroup(name string) bool { + for _, bsig := range f.bsiGroups { + if bsig.Name == name { + return true + } + } + return false +} + +// createBSIGroup creates a new bsiGroup on the field. +func (f *Field) createBSIGroup(bsig *bsiGroup) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Append bsiGroup. + if err := f.addBSIGroup(bsig); err != nil { + return err + } + f.saveMeta() + return nil +} + +// addBSIGroup adds a single bsiGroup to bsiGroups. +func (f *Field) addBSIGroup(bsig *bsiGroup) error { + if err := bsig.validate(); err != nil { + return errors.Wrap(err, "validating bsigroup") + } else if f.hasBSIGroup(bsig.Name) { + return ErrBSIGroupExists + } + + // Add bsiGroup to list. + f.bsiGroups = append(f.bsiGroups, bsig) + + // Sort bsiGroups by name. + sort.Slice(f.bsiGroups, func(i, j int) bool { + return f.bsiGroups[i].Name < f.bsiGroups[j].Name + }) + + return nil +} + +// deleteBSIGroupAndView deletes an existing bsiGroup on the schema. +func (f *Field) deleteBSIGroupAndView(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Remove bsiGroup. + if err := f.deleteBSIGroup(name); err != nil { + return err + } + + // Remove views. + viewName := viewBSIGroupPrefix + name + if view := f.viewMap[viewName]; view != nil { + delete(f.viewMap, viewName) + + if err := view.close(); err != nil { + return errors.Wrap(err, "closing view") + } else if err := os.RemoveAll(view.path); err != nil { + return errors.Wrap(err, "deleting directory") + } + } + + return nil +} + +// deleteBSIGroup removes a single bsiGroup from bsiGroups. +func (f *Field) deleteBSIGroup(name string) error { + for i, bsig := range f.bsiGroups { + if bsig.Name == name { + copy(f.bsiGroups[i:], f.bsiGroups[i+1:]) + f.bsiGroups, f.bsiGroups[len(f.bsiGroups)-1] = f.bsiGroups[:len(f.bsiGroups)-1], nil + return nil + } + } + return ErrBSIGroupNotFound +} + +// TimeQuantum returns the time quantum for the field. +func (f *Field) TimeQuantum() TimeQuantum { + f.mu.Lock() + defer f.mu.Unlock() + return f.options.TimeQuantum +} + +// setTimeQuantum sets the time quantum for the field. +func (f *Field) setTimeQuantum(q TimeQuantum) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on field. + f.options.TimeQuantum = q + + // Persist meta data to disk. + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving meta") + } + + return nil +} + +// RowTime gets the row at the particular time with the granularity specified by +// the quantum. +func (f *Field) RowTime(rowID uint64, time time.Time, quantum string) (*Row, error) { + if !TimeQuantum(quantum).Valid() { + return nil, ErrInvalidTimeQuantum + } + viewname := viewsByTime(viewStandard, time, TimeQuantum(quantum[len(quantum)-1:]))[0] + view := f.view(viewname) + if view == nil { + return nil, errors.Errorf("view with quantum %v not found.", quantum) + } + return view.row(rowID), nil +} + +// viewPath returns the path to a view in the field. +func (f *Field) viewPath(name string) string { + return filepath.Join(f.path, "views", name) +} + +// view returns a view in the field by name. +func (f *Field) view(name string) *view { + f.mu.RLock() + defer f.mu.RUnlock() + return f.unprotectedView(name) +} + +func (f *Field) unprotectedView(name string) *view { return f.viewMap[name] } + +// views returns a list of all views in the field. +func (f *Field) views() []*view { + f.mu.RLock() + defer f.mu.RUnlock() + + other := make([]*view, 0, len(f.viewMap)) + for _, view := range f.viewMap { + other = append(other, view) + } + return other +} + +// viewNames returns a list of all views (as a string) in the field. +func (f *Field) viewNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + + other := make([]string, 0, len(f.viewMap)) + for viewName, _ := range f.viewMap { + other = append(other, viewName) + } + return other +} + +// recalculateCaches recalculates caches on every view in the field. +func (f *Field) recalculateCaches() { + for _, view := range f.views() { + view.recalculateCaches() + } +} + +// createViewIfNotExists returns the named view, creating it if necessary. +// Additionally, a CreateViewMessage is sent to the cluster. +func (f *Field) createViewIfNotExists(name string) (*view, error) { + view, created, err := f.createViewIfNotExistsBase(name) + if err != nil { + return nil, err + } + + if created { + // Broadcast view creation to the cluster. + err = f.broadcaster.SendSync( + &CreateViewMessage{ + Index: f.index, + Field: f.name, + View: name, + }) + if err != nil { + return nil, errors.Wrap(err, "sending CreateView message") + } + } + + return view, nil +} + +// createViewIfNotExistsBase returns the named view, creating it if necessary. +// The returned bool indicates whether the view was created or not. +func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if view := f.viewMap[name]; view != nil { + return view, false, nil + } + view := f.newView(f.viewPath(name), name) + + if err := view.open(); err != nil { + return nil, false, errors.Wrap(err, "opening view") + } + view.rowAttrStore = f.rowAttrStore + f.viewMap[view.name] = view + + return view, true, nil +} + +func (f *Field) newView(path, name string) *view { + view := newView(path, f.index, f.name, name, f.options.CacheSize) + view.cacheType = f.options.CacheType + view.logger = f.logger + view.rowAttrStore = f.rowAttrStore + view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) + view.broadcaster = f.broadcaster + return view +} + +// deleteView removes the view from the field. +func (f *Field) deleteView(name string) error { + view := f.viewMap[name] + if view == nil { + return ErrInvalidView + } + + // Close data files before deletion. + if err := view.close(); err != nil { + return errors.Wrap(err, "closing view") + } + + // Delete view directory. + if err := os.RemoveAll(view.path); err != nil { + return errors.Wrap(err, "deleting directory") + } + + delete(f.viewMap, name) + + return nil +} + +// Row returns a row of the standard view. +func (f *Field) Row(rowID uint64) (*Row, error) { + if f.Type() != FieldTypeSet { + return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) + } + view := f.view(viewStandard) + if view == nil { + return nil, ErrInvalidView + } + return view.row(rowID), nil +} + +// SetBit sets a bit on a view within the field. +func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { + viewName := viewStandard + + // Retrieve view. Exit if it doesn't exist. + view, err := f.createViewIfNotExists(viewName) + if err != nil { + return changed, errors.Wrap(err, "creating view") + } + + // Set non-time bit. + if v, err := view.setBit(rowID, colID); err != nil { + return changed, errors.Wrap(err, "setting on view") + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then set bits across all views for the quantum. + for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { + view, err := f.createViewIfNotExists(subname) + if err != nil { + return changed, errors.Wrapf(err, "creating view %s", subname) + } + + if c, err := view.setBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "setting on view %s", subname) + } else if c { + changed = true + } + } + + return changed, nil +} + +// ClearBit clears a bit within the field. +func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { + viewName := viewStandard + + // Retrieve view. Exit if it doesn't exist. + view, present := f.viewMap[viewName] + if !present { + return changed, errors.Wrap(err, "clearing missing view") + + } + + // Clear non-time bit. + if v, err := view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrap(err, "clearing on view") + } else if v { + changed = v + } + if len(f.viewMap) == 1 { // assuming no time views + return changed, nil + } + lastViewNameSize := 0 + level := 0 + skipAbove := maxInt + for _, view := range f.allTimeViewsSortedByQuantum() { + if lastViewNameSize < len(view.name) { + level++ + } else if lastViewNameSize > len(view.name) { + level-- + } + if level < skipAbove { + if changed, err = view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "clearing on view %s", view.name) + } + if !changed { + skipAbove = level + 1 + } else { + skipAbove = maxInt + } + } + lastViewNameSize = len(view.name) + } + + return changed, nil +} + +func groupCompare(a, b string, offset int) (lt, eq bool) { + if len(a) > offset { + a = a[:offset] + } + if len(b) > offset { + b = b[:offset] + } + v := strings.Compare(a, b) + return v < 0, v == 0 +} + +func (f *Field) allTimeViewsSortedByQuantum() (me []*view) { + me = make([]*view, len(f.viewMap), len(f.viewMap)) + prefix := viewStandard + "_" + offset := len(viewStandard) + 1 + i := 0 + for _, v := range f.viewMap { + if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views + me[i] = v + i++ + } + } + me = me[:i] + year := strings.Index(me[0].name, "_") + 4 + month := year + 2 + day := month + 2 + sort.Slice(me, func(i, j int) (lt bool) { + var eq bool + // group by quantum from year to hour + if lt, eq = groupCompare(me[i].name, me[j].name, year); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, month); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, day); eq { + lt = strings.Compare(me[i].name, me[j].name) > 0 + } + } + } + return + }) + return +} + +// Value reads a field value for a column. +func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { + bsig := f.bsiGroup(f.name) + if bsig == nil { + return 0, false, ErrBSIGroupNotFound + } + + // Fetch target view. + view := f.view(viewBSIGroupPrefix + f.name) + if view == nil { + return 0, false, nil + } + + v, exists, err := view.value(columnID, bsig.BitDepth()) + if err != nil { + return 0, false, err + } else if !exists { + return 0, false, nil + } + return int64(v) + bsig.Min, true, nil +} + +// SetValue sets a field value for a column. +func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { + // Fetch bsiGroup and validate value. + bsig := f.bsiGroup(f.name) + if bsig == nil { + return false, ErrBSIGroupNotFound + } else if value < bsig.Min { + return false, ErrBSIGroupValueTooLow + } else if value > bsig.Max { + return false, ErrBSIGroupValueTooHigh + } + + // Fetch target view. + view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + + // Determine base value to store. + baseValue := uint64(value - bsig.Min) + + return view.setValue(columnID, bsig.BitDepth(), baseValue) +} + +// Sum returns the sum and count for a field. +// An optional filtering row can be provided. +func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.view(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vsum, vcount, err := view.sum(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil +} + +// Min returns the min for a field. +// An optional filtering row can be provided. +func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.view(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.min(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + bsig.Min, int64(vcount), nil +} + +// Max returns the max for a field. +// An optional filtering row can be provided. +func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.view(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.max(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + bsig.Min, int64(vcount), nil +} + +func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { + return nil, ErrBSIGroupNotFound + } else if predicate < bsig.Min || predicate > bsig.Max { + return nil, nil + } + + // Retrieve bsiGroup's view. + view := f.view(viewBSIGroupPrefix + name) + if view == nil { + return nil, nil + } + + baseValue, outOfRange := bsig.baseValue(op, predicate) + if outOfRange { + return NewRow(), nil + } + + return view.rangeOp(op, bsig.BitDepth(), baseValue) +} + +func (f *Field) rangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { + return nil, ErrBSIGroupNotFound + } else if predicateMin > predicateMax { + return nil, ErrInvalidBetweenValue + } + + // Retrieve bsiGroup's view. + view := f.view(viewBSIGroupPrefix + name) + if view == nil { + return nil, nil + } + + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicateMin, predicateMax) + if outOfRange { + return NewRow(), nil + } + + return view.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) +} + +// Import bulk imports data. +func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { + // Determine quantum if timestamps are set. + q := f.TimeQuantum() + if hasTime(timestamps) && q == "" { + return errors.New("time quantum not set in field") + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] + var timestamp *time.Time + if len(timestamps) > i { + timestamp = timestamps[i] + } + + var standard []string + if timestamp == nil { + standard = []string{viewStandard} + } else { + standard = viewsByTime(viewStandard, *timestamp, q) + // In order to match the logic of `SetBit()`, we want bits + // with timestamps to write to both time and standard views. + standard = append(standard, viewStandard) + } + + // Attach bit to each standard view. + for _, name := range standard { + key := importKey{View: name, Shard: columnID / ShardWidth} + data := dataByFragment[key] + data.RowIDs = append(data.RowIDs, rowID) + data.ColumnIDs = append(data.ColumnIDs, columnID) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + view, err := f.createViewIfNotExists(key.View) + if err != nil { + return errors.Wrap(err, "creating view") + } + + frag, err := view.CreateFragmentIfNotExists(key.Shard) + if err != nil { + return errors.Wrap(err, "creating view") + } + + if err := frag.bulkImport(data.RowIDs, data.ColumnIDs); err != nil { + return err + } + } + + return nil +} + +// importValue bulk imports range-encoded value data. +func (f *Field) importValue(columnIDs []uint64, values []int64) error { + viewName := viewBSIGroupPrefix + f.name + // Get the bsiGroup so we know bitDepth. + bsig := f.bsiGroup(f.name) + if bsig == nil { + return errors.Wrap(ErrBSIGroupNotFound, f.name) + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importValueData) + for i := range columnIDs { + columnID, value := columnIDs[i], values[i] + if int64(value) > bsig.Max { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) + } else if int64(value) < bsig.Min { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) + } + + // Attach value to each bsiGroup view. + for _, name := range []string{viewName} { + key := importKey{View: name, Shard: columnID / ShardWidth} + data := dataByFragment[key] + data.ColumnIDs = append(data.ColumnIDs, columnID) + data.Values = append(data.Values, value) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + + // The view must already exist (i.e. we can't create it) + // because we need to know bitDepth (based on min/max value). + view, err := f.createViewIfNotExists(key.View) + if err != nil { + return errors.Wrap(err, "creating view") + } + + frag, err := view.CreateFragmentIfNotExists(key.Shard) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + + baseValues := make([]uint64, len(data.Values)) + for i, value := range data.Values { + baseValues[i] = uint64(value - bsig.Min) + } + + if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { + return err + } + } + + return nil +} + +type fieldSlice []*Field + +func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldSlice) Len() int { return len(p) } +func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } + +// FieldInfo represents schema information for a field. +type FieldInfo struct { + Name string `json:"name"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` +} + +type fieldInfoSlice []*FieldInfo + +func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldInfoSlice) Len() int { return len(p) } +func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + +// FieldOptions represents options to set when initializing a field. +type FieldOptions struct { + Type string `json:"type,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Keys bool `json:"keys"` +} + +// applyDefaultOptions returns a new FieldOptions object +// with default values if o does not contain a valid type. +func applyDefaultOptions(o FieldOptions) FieldOptions { + if o.Type == "" { + return FieldOptions{ + Type: DefaultFieldType, + CacheType: DefaultCacheType, + CacheSize: DefaultCacheSize, + } + } + return o +} + +// encode converts o into its internal representation. +func (o *FieldOptions) encode() *internal.FieldOptions { + return encodeFieldOptions(o) +} + +func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +func (o *FieldOptions) MarshalJSON() ([]byte, error) { + switch o.Type { + case FieldTypeSet: + return json.Marshal(struct { + Type string `json:"type"` + CacheType string `json:"cacheType"` + CacheSize uint32 `json:"cacheSize"` + Keys bool `json:"keys"` + }{ + o.Type, + o.CacheType, + o.CacheSize, + o.Keys, + }) + case FieldTypeInt: + return json.Marshal(struct { + Type string `json:"type"` + Min int64 `json:"min"` + Max int64 `json:"max"` + Keys bool `json:"keys"` + }{ + o.Type, + o.Min, + o.Max, + o.Keys, + }) + case FieldTypeTime: + return json.Marshal(struct { + Type string `json:"type"` + TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` + }{ + o.Type, + o.TimeQuantum, + o.Keys, + }) + } + return nil, errors.New("invalid field type") +} + +// List of bsiGroup types. +const ( + bsiGroupTypeInt = "int" +) + +func isValidBSIGroupType(v string) bool { + switch v { + case bsiGroupTypeInt: + return true + default: + return false + } +} + +// bsiGroup represents a group of range-encoded rows on a field. +type bsiGroup struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` +} + +// BitDepth returns the number of bits required to store a value between min & max. +func (b *bsiGroup) BitDepth() uint { + for i := uint(0); i < 63; i++ { + if b.Max-b.Min < (1 << i) { + return i + } + } + return 63 +} + +// baseValue adjusts the value to align with the range for Field for a certain +// operation type. +// Note: There is an edge case for GT and LT where this returns a baseValue +// that does not fully encompass the range. +// ex: Field.Min = 0, Field.Max = 1023 +// baseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively +// exclude any columns with value = 1023. +// Note that in this case (because the range uses the full BitDepth 0 to 1023), +// we can't simply return 1024. +// In order to make this work, we effectively need to change the operator to LTE. +// Executor.executeBSIGroupRangeShard() takes this into account and returns +// `frag.FieldNotNull(bsig.BitDepth())` in such instances. +func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { + if op == pql.GT || op == pql.GTE { + if value > b.Max { + return baseValue, true + } else if value > b.Min { + baseValue = uint64(value - b.Min) + } + } else if op == pql.LT || op == pql.LTE { + if value < b.Min { + return baseValue, true + } else if value > b.Max { + baseValue = uint64(b.Max - b.Min) + } else { + baseValue = uint64(value - b.Min) + } + } else if op == pql.EQ || op == pql.NEQ { + if value < b.Min || value > b.Max { + return baseValue, true + } + baseValue = uint64(value - b.Min) + } + return baseValue, false +} + +// baseValueBetween adjusts the min/max value to align with the range for Field. +func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { + if max < b.Min || min > b.Max { + return baseValueMin, baseValueMax, true + } + // Adjust min/max to range. + if min > b.Min { + baseValueMin = uint64(min - b.Min) + } + // Make sure the high value of the BETWEEN does not exceed BitDepth. + if max > b.Max { + baseValueMax = uint64(b.Max - b.Min) + } else if max > b.Min { + baseValueMax = uint64(max - b.Min) + } + return baseValueMin, baseValueMax, false +} + +func (b *bsiGroup) validate() error { + if b.Name == "" { + return ErrBSIGroupNameRequired + } else if !isValidBSIGroupType(b.Type) { + return ErrInvalidBSIGroupType + } else if b.Min > b.Max { + return ErrInvalidBSIGroupRange + } + return nil +} + +// Cache types. +const ( + CacheTypeLRU = "lru" + CacheTypeRanked = "ranked" + CacheTypeNone = "none" +) + +// isValidCacheType returns true if v is a valid cache type. +func isValidCacheType(v string) bool { + switch v { + case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: + return true + default: + return false + } +} diff --git a/field_internal_test.go b/field_internal_test.go new file mode 100644 index 000000000..2d37ea25c --- /dev/null +++ b/field_internal_test.go @@ -0,0 +1,343 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "os" + "reflect" + "testing" + "time" + + "github.com/pilosa/pilosa/pql" +) + +// Ensure a bsiGroup can adjust to its baseValue. +func TestBSIGroup_BaseValue(t *testing.T) { + b0 := &bsiGroup{ + Name: "b0", + Type: bsiGroupTypeInt, + Min: -100, + Max: 900, + } + b1 := &bsiGroup{ + Name: "b1", + Type: bsiGroupTypeInt, + Min: 0, + Max: 1000, + } + + b2 := &bsiGroup{ + Name: "b2", + Type: bsiGroupTypeInt, + Min: 100, + Max: 1100, + } + + t.Run("Normal Condition", func(t *testing.T) { + + for _, tt := range []struct { + f *bsiGroup + op pql.Token + val int64 + expBaseValue uint64 + expOutOfRange bool + }{ + // LT + {b0, pql.LT, 5, 105, false}, + {b0, pql.LT, -8, 92, false}, + {b0, pql.LT, -108, 0, true}, + {b0, pql.LT, 1005, 1000, false}, + {b0, pql.LT, 0, 100, false}, + + {b1, pql.LT, 5, 5, false}, + {b1, pql.LT, -8, 0, true}, + {b1, pql.LT, 1005, 1000, false}, + {b1, pql.LT, 0, 0, false}, + + {b2, pql.LT, 5, 0, true}, + {b2, pql.LT, -8, 0, true}, + {b2, pql.LT, 105, 5, false}, + {b2, pql.LT, 1105, 1000, false}, + + // GT + {b0, pql.GT, -105, 0, false}, + {b0, pql.GT, 5, 105, false}, + {b0, pql.GT, 905, 0, true}, + {b0, pql.GT, 0, 100, false}, + + {b1, pql.GT, 5, 5, false}, + {b1, pql.GT, -8, 0, false}, + {b1, pql.GT, 1005, 0, true}, + {b1, pql.GT, 0, 0, false}, + + {b2, pql.GT, 5, 0, false}, + {b2, pql.GT, -8, 0, false}, + {b2, pql.GT, 105, 5, false}, + {b2, pql.GT, 1105, 0, true}, + + // EQ + {b0, pql.EQ, -105, 0, true}, + {b0, pql.EQ, 5, 105, false}, + {b0, pql.EQ, 905, 0, true}, + {b0, pql.EQ, 0, 100, false}, + + {b1, pql.EQ, 5, 5, false}, + {b1, pql.EQ, -8, 0, true}, + {b1, pql.EQ, 1005, 0, true}, + {b1, pql.EQ, 0, 0, false}, + + {b2, pql.EQ, 5, 0, true}, + {b2, pql.EQ, -8, 0, true}, + {b2, pql.EQ, 105, 5, false}, + {b2, pql.EQ, 1105, 0, true}, + } { + bv, oor := tt.f.baseValue(tt.op, tt.val) + if oor != tt.expOutOfRange { + t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) + } + } + }) + + t.Run("Betwween Condition", func(t *testing.T) { + for _, tt := range []struct { + f *bsiGroup + predMin int64 + predMax int64 + expBaseValueMin uint64 + expBaseValueMax uint64 + expOutOfRange bool + }{ + + {b0, -205, -105, 0, 0, true}, + {b0, -105, 80, 0, 180, false}, + {b0, 5, 20, 105, 120, false}, + {b0, 20, 1005, 120, 1000, false}, + {b0, 1005, 2000, 0, 0, true}, + + {b1, -105, -5, 0, 0, true}, + {b1, -5, 20, 0, 20, false}, + {b1, 5, 20, 5, 20, false}, + {b1, 20, 1005, 20, 1000, false}, + {b1, 1005, 2000, 0, 0, true}, + + {b2, 5, 95, 0, 0, true}, + {b2, 95, 120, 0, 20, false}, + {b2, 105, 120, 5, 20, false}, + {b2, 120, 1105, 20, 1000, false}, + {b2, 1105, 2000, 0, 0, true}, + } { + min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax) + if oor != tt.expOutOfRange { + t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { + t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) + } + } + }) +} + +// Ensure field can open and retrieve a view. +func TestField_DeleteView(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + defer f.Close() + + viewName := viewStandard + "_v" + + // Create view. + view, err := f.createViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + err = f.deleteView(viewName) + if err != nil { + t.Fatal(err) + } + + if f.view(viewName) != nil { + t.Fatal("view still exists in field") + } + + // Recreate view with same name, verify that the old view was not reused. + view2, err := f.createViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == view2 { + t.Fatal("failed to create new view") + } +} + +// TestField represents a test wrapper for Field. +type TestField struct { + *Field +} + +// NewTestField returns a new instance of TestField d/0. +func NewTestField(opts FieldOption) *TestField { + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + field, err := NewField(path, "i", "f", opts) + if err != nil { + panic(err) + } + return &TestField{Field: field} +} + +// MustOpenField returns a new, opened field at a temporary path. Panic on error. +func MustOpenField(opts FieldOption) *TestField { + f := NewTestField(opts) + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Close closes the field and removes the underlying data. +func (f *TestField) Close() error { + defer os.RemoveAll(f.Path()) + return f.Field.Close() +} + +// Reopen closes the index and reopens it. +func (f *TestField) Reopen() error { + var err error + if err := f.Field.Close(); err != nil { + return err + } + + path, index, name := f.Path(), f.Index(), f.Name() + f.Field, err = NewField(path, index, name, OptFieldTypeDefault()) + if err != nil { + return err + } + + if err := f.Open(); err != nil { + return err + } + return nil +} + +func (f *TestField) MustSetBit(row, col uint64, ts ...time.Time) { + if len(ts) == 0 { + _, err := f.Field.SetBit(row, col, nil) + if err != nil { + panic(err) + } + } + for _, t := range ts { + _, err := f.Field.SetBit(row, col, &t) + if err != nil { + panic(err) + } + } +} + +// Ensure field can open and retrieve a view. +func TestField_CreateViewIfNotExists(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + defer f.Close() + + // Create view. + view, err := f.createViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + // Retrieve existing view. + view2, err := f.createViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view != view2 { + t.Fatal("view mismatch") + } + + if view != f.view("v") { + t.Fatal("view mismatch") + } +} + +func TestField_SetTimeQuantum(t *testing.T) { + f := MustOpenField(OptFieldTypeTime(TimeQuantum(""))) + defer f.Close() + + // Set & retrieve time quantum. + if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload field and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + +func TestField_RowTime(t *testing.T) { + f := MustOpenField(OptFieldTypeTime(TimeQuantum(""))) + defer f.Close() + + if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } + + f.MustSetBit(1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC)) + f.MustSetBit(1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC)) + + if r, err := f.RowTime(1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) { + t.Fatalf("wrong columns: %#v", r.Columns()) + } + + if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "YM"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(r.Columns(), []uint64{3}) { + t.Fatalf("wrong columns: %#v", r.Columns()) + } + + if r, err := f.RowTime(1, time.Date(2010, time.February, 7, 13, 0, 0, 0, time.UTC), "M"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(r.Columns(), []uint64{3}) { + t.Fatalf("wrong columns: %#v", r.Columns()) + } + + if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC), "MD"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(r.Columns(), []uint64{1, 5}) { + t.Fatalf("wrong columns: %#v", r.Columns()) + } + + if r, err := f.RowTime(1, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC), "MDH"); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(r.Columns(), []uint64{5}) { + t.Fatalf("wrong columns: %#v", r.Columns()) + } + +} diff --git a/field_test.go b/field_test.go new file mode 100644 index 000000000..520179f52 --- /dev/null +++ b/field_test.go @@ -0,0 +1,187 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "io/ioutil" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// Ensure a field can set & read a bsiGroup value. +func TestField_SetValue(t *testing.T) { + t.Run("OK", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value on field. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if value != 21 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + + // Setting value should return no change. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if changed { + t.Fatal("expected no change") + } + }) + + t.Run("Overwrite", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Set different value. + if changed, err := f.SetValue(100, 23); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if value != 23 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + }) + + t.Run("ErrBSIGroupNotFound", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { + t.Fatalf("unexpected error: %s", err) + } + }) +} + +func TestField_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + field, err := pilosa.NewField(path, "i", ".meta", pilosa.OptFieldTypeDefault()) + if field != nil { + t.Fatalf("unexpected field name %s", err) + } +} + +// Ensure that field name validation is consistent. +func TestField_NameValidation(t *testing.T) { + validFieldNames := []string{ + "foo", + "hyphen-ated", + "under_score", + "abc123", + "trailing_", + } + invalidFieldNames := []string{ + "", + "123abc", + "x.y", + "_foo", + "-bar", + "abc def", + "camelCase", + "UPPERCASE", + "a12345678901234567890123456789012345678901234567890123456789012345", + } + + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + for _, name := range validFieldNames { + _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatalf("unexpected field name: %s %s", name, err) + } + } + for _, name := range invalidFieldNames { + _, err := pilosa.NewField(path, "i", name, pilosa.OptFieldTypeDefault()) + if err == nil { + t.Fatalf("expected error on field name: %s", name) + } + } +} diff --git a/fragment.go b/fragment.go index 8044f579b..8c5fbf787 100644 --- a/fragment.go +++ b/fragment.go @@ -25,7 +25,6 @@ import ( "hash" "io" "io/ioutil" - "net/http" "os" "sort" "sync" @@ -45,36 +44,34 @@ import ( ) const ( - // SliceWidth is the number of column IDs in a slice. - SliceWidth = 1048576 + // ShardWidth is the number of column IDs in a shard. + ShardWidth = 1048576 - // SnapshotExt is the file extension used for an in-process snapshot. - SnapshotExt = ".snapshotting" + // snapshotExt is the file extension used for an in-process snapshot. + snapshotExt = ".snapshotting" - // CopyExt is the file extension used for the temp file used while copying. - CopyExt = ".copying" + // copyExt is the file extension used for the temp file used while copying. + copyExt = ".copying" - // CacheExt is the file extension for persisted cache ids. - CacheExt = ".cache" + // cacheExt is the file extension for persisted cache ids. + cacheExt = ".cache" // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 + + // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. + defaultFragmentMaxOpN = 2000 ) -const ( - // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - DefaultFragmentMaxOpN = 2000 -) - -// Fragment represents the intersection of a frame and slice in an index. -type Fragment struct { +// fragment represents the intersection of a field and shard in an index. +type fragment struct { mu sync.RWMutex // Composite identifiers index string - frame string + field string view string - slice uint64 + shard uint64 // File-backed storage path string @@ -84,15 +81,15 @@ type Fragment struct { opN int // number of ops since snapshot // Cache for row counts. - CacheType string // passed in by frame - cache Cache + CacheType string // passed in by field + cache cache CacheSize uint32 // Stats reporting. maxRowID uint64 // Cache containing full rows (not just counts). - rowCache BitmapCache + rowCache bitmapCache // Cached checksums for each block. checksums map[int][]byte @@ -106,54 +103,35 @@ type Fragment struct { Logger Logger // Row attribute storage. - // This is set by the parent frame unless overridden for testing. + // This is set by the parent field unless overridden for testing. RowAttrStore AttrStore stats StatsClient } -// NewFragment returns a new instance of Fragment. -func NewFragment(path, index, frame, view string, slice uint64) *Fragment { - return &Fragment{ +// newFragment returns a new instance of Fragment. +func newFragment(path, index, field, view string, shard uint64) *fragment { + return &fragment{ path: path, index: index, - frame: frame, + field: field, view: view, - slice: slice, + shard: shard, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, Logger: NopLogger, - MaxOpN: DefaultFragmentMaxOpN, + MaxOpN: defaultFragmentMaxOpN, stats: NopStatsClient, } } -// Path returns the path the fragment was initialized with. -func (f *Fragment) Path() string { return f.path } - -// CachePath returns the path to the fragment's cache data. -func (f *Fragment) CachePath() string { return f.path + CacheExt } - -// Index returns the index that the fragment was initialized with. -func (f *Fragment) Index() string { return f.index } - -// Frame returns the frame the fragment was initialized with. -func (f *Fragment) Frame() string { return f.frame } - -// View returns the view the fragment was initialized with. -func (f *Fragment) View() string { return f.view } - -// Slice returns the slice the fragment was initialized with. -func (f *Fragment) Slice() uint64 { return f.slice } - -// Cache returns the fragment's cache. -// This is not safe for concurrent use. -func (f *Fragment) Cache() Cache { return f.cache } +// cachePath returns the path to the fragment's cache data. +func (f *fragment) cachePath() string { return f.path + cacheExt } // Open opens the underlying storage. -func (f *Fragment) Open() error { +func (f *fragment) Open() error { f.mu.Lock() defer f.mu.Unlock() @@ -173,7 +151,7 @@ func (f *Fragment) Open() error { // Read last bit to determine max row. pos := f.storage.Max() - f.maxRowID = pos / SliceWidth + f.maxRowID = pos / ShardWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) return nil @@ -186,8 +164,8 @@ func (f *Fragment) Open() error { } // openStorage opens the storage bitmap. -func (f *Fragment) openStorage() error { - // Create a roaring bitmap to serve as storage for the slice. +func (f *fragment) openStorage() error { + // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() } @@ -239,29 +217,29 @@ func (f *Fragment) openStorage() error { // Attach the file to the bitmap to act as a write-ahead log. f.storage.OpWriter = f.file - f.rowCache = &SimpleCache{make(map[uint64]*Row)} + f.rowCache = &simpleCache{make(map[uint64]*Row)} return nil } // openCache initializes the cache from row ids persisted to disk. -func (f *Fragment) openCache() error { - // Determine cache type from frame name. +func (f *fragment) openCache() error { + // Determine cache type from field name. switch f.CacheType { case CacheTypeRanked: f.cache = NewRankCache(f.CacheSize) case CacheTypeLRU: - f.cache = NewLRUCache(f.CacheSize) + f.cache = newLRUCache(f.CacheSize) case CacheTypeNone: - f.cache = NewNopCache() + f.cache = globalNopCache return nil default: return ErrInvalidCacheType } // Read cache data from disk. - path := f.CachePath() + path := f.cachePath() buf, err := ioutil.ReadFile(path) if os.IsNotExist(err) { return nil @@ -279,7 +257,7 @@ func (f *Fragment) openCache() error { // Read in all rows by ID. // This will cause them to be added to the cache. for _, id := range pb.IDs { - n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) + n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth) f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -288,13 +266,13 @@ func (f *Fragment) openCache() error { } // Close flushes the underlying storage, closes the file and unlocks it. -func (f *Fragment) Close() error { +func (f *fragment) Close() error { f.mu.Lock() defer f.mu.Unlock() return f.close() } -func (f *Fragment) close() error { +func (f *fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { f.Logger.Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) @@ -313,7 +291,7 @@ func (f *Fragment) close() error { return nil } -func (f *Fragment) closeStorage() error { +func (f *fragment) closeStorage() error { // Clear the storage bitmap so it doesn't access the closed mmap. //f.storage = roaring.NewBitmap() @@ -342,14 +320,14 @@ func (f *Fragment) closeStorage() error { return nil } -// Row returns a row by ID. -func (f *Fragment) Row(rowID uint64) *Row { +// row returns a row by ID. +func (f *fragment) row(rowID uint64) *Row { f.mu.Lock() defer f.mu.Unlock() - return f.row(rowID, true, true) + return f.unprotectedRow(rowID, true, true) } -func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { +func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { if checkRowCache { r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { @@ -359,19 +337,19 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by - data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth) + data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. // We Clone() data because otherwise row will contains pointers to containers in storage. // This causes unexpected results when we cache the row and try to use it later. row := &Row{ - segments: []RowSegment{{ + segments: []rowSegment{{ data: *data.Clone(), - slice: f.slice, + shard: f.shard, writable: false, }}, } - row.InvalidateCount() + row.invalidateCount() if updateRowCache { f.rowCache.Add(rowID, row) @@ -380,15 +358,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a bit for a given column & row within the fragment. +// setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(rowID, columnID) + return f.unprotectedSetBit(rowID, columnID) } -func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -415,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from row cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.SetBit(columnID) // Update the cache. @@ -432,15 +410,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a bit for a given column & row within the fragment. +// clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { +func (f *fragment) clearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(rowID, columnID) + return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -467,8 +445,8 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from cache or fragment.storage. - row := f.row(rowID, true, true) - row.ClearBit(columnID) + row := f.unprotectedRow(rowID, true, true) + row.clearBit(columnID) // Update the cache. f.cache.Add(rowID, row.Count()) @@ -478,7 +456,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { +func (f *fragment) bit(rowID, columnID uint64) (bool, error) { pos, err := f.pos(rowID, columnID) if err != nil { return false, err @@ -486,8 +464,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// FieldValue uses a column of bits to read a multi-bit value. -func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (f *fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -510,20 +488,20 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi return value, true, nil } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (f *fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() for i := uint(0); i < bitDepth; i++ { if value&(1< uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := consider.Difference(row) count = x.Count() @@ -647,13 +625,13 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err return min, count, nil } -// FieldMax returns the max of a given field as well as the number of columns involved. +// max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.row(uint64(bitDepth)) if filter != nil { - consider = consider.Intersect(filter) + consider = consider.intersect(filter) } // If there are no columns to consider, return early. @@ -663,9 +641,9 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) - x := row.Intersect(consider) + x := row.intersect(consider) count = x.Count() if count > 0 { max += (1 << ii) @@ -678,33 +656,33 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err return max, count, nil } -// FieldRange returns bitmaps with a field value encoding matching the predicate. -func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. +func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: - return f.fieldRangeEQ(bitDepth, predicate) + return f.rangeEQ(bitDepth, predicate) case pql.NEQ: - return f.fieldRangeNEQ(bitDepth, predicate) + return f.rangeNEQ(bitDepth, predicate) case pql.LT, pql.LTE: - return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + return f.rangeLT(bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) + return f.rangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } } -func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 if bit == 1 { - b = b.Intersect(row) + b = b.intersect(row) } else { b = b.Difference(row) } @@ -713,12 +691,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Get the equal bitmap. - eq, err := f.fieldRangeEQ(bitDepth, predicate) + eq, err := f.rangeEQ(bitDepth, predicate) if err != nil { return nil, err } @@ -729,16 +707,16 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) return b, nil } -func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. @@ -777,13 +755,13 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(bitDepth)) +func (f *fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + b := f.row(uint64(bitDepth)) keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -805,27 +783,27 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep = keep.Union(b.Intersect(row)) + keep = keep.Union(b.intersect(row)) } } return b, nil } -// FieldNotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) { - return f.Row(uint64(bitDepth)), nil +// notNull returns the not-null row (stored at bitDepth). +func (f *fragment) notNull(bitDepth uint) (*Row, error) { + return f.row(uint64(bitDepth)), nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(bitDepth)) +// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. +func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { + b := f.row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 @@ -837,7 +815,7 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { - keep1 = keep1.Union(b.Intersect(row)) + keep1 = keep1.Union(b.intersect(row)) } } @@ -858,18 +836,18 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u } // pos translates the row ID and column ID into a position in the storage bitmap. -func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { - // Return an error if the column ID is out of the range of the fragment's slice. - minColumnID := f.slice * SliceWidth - if columnID < minColumnID || columnID >= minColumnID+SliceWidth { +func (f *fragment) pos(rowID, columnID uint64) (uint64, error) { + // Return an error if the column ID is out of the range of the fragment's shard. + minColumnID := f.shard * ShardWidth + if columnID < minColumnID || columnID >= minColumnID+ShardWidth { return 0, errors.New("column out of bounds") } - return Pos(rowID, columnID), nil + return pos(rowID, columnID), nil } -// ForEachBit executes fn for every bit set in the fragment. +// forEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { +func (f *fragment) forEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -881,15 +859,15 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { } // Invoke caller's function. - err = fn(i/SliceWidth, (f.slice*SliceWidth)+(i%SliceWidth)) + err = fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)) }) return err } -// Top returns the top rows from the fragment. +// top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. // If opt.FilterValues exist then the row attribute specified by field is matched. -func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { +func (f *fragment) top(opt topOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. pairs := f.topBitmapPairs(opt.RowIDs) @@ -900,7 +878,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Create a fast lookup of filter values. var filters map[interface{}]struct{} - if opt.FilterField != "" && len(opt.FilterValues) > 0 { + if opt.FilterName != "" && len(opt.FilterValues) > 0 { filters = make(map[interface{}]struct{}) for _, v := range opt.FilterValues { filters[v] = struct{}{} @@ -919,7 +897,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } // Iterate over rankings and add to results until we have enough. - results := &PairHeap{} + results := &pairHeap{} for _, pair := range pairs { rowID, cnt := pair.ID, pair.Count @@ -948,7 +926,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return nil, errors.Wrap(err, "getting attrs") } else if attr == nil { continue - } else if attrValue := attr[opt.FilterField]; attrValue == nil { + } else if attrValue := attr[opt.FilterName]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { continue @@ -960,7 +938,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Row(rowID)) + count = opt.Src.intersectionCount(f.row(rowID)) } if count == 0 { continue @@ -1004,7 +982,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.IntersectionCount(f.Row(rowID)) + count := opt.Src.intersectionCount(f.row(rowID)) if count < threshold { continue } @@ -1023,7 +1001,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return r, nil } -func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { +func (f *fragment) topBitmapPairs(rowIDs []uint64) []bitmapPair { // Don't retrieve from storage if CacheTypeNone. if f.CacheType == CacheTypeNone { return f.cache.Top() @@ -1037,32 +1015,32 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { } // Otherwise retrieve specific rows. - pairs := make([]BitmapPair, 0, len(rowIDs)) + pairs := make([]bitmapPair, 0, len(rowIDs)) for _, rowID := range rowIDs { // Look up cache first, if available. if n := f.cache.Get(rowID); n > 0 { - pairs = append(pairs, BitmapPair{ + pairs = append(pairs, bitmapPair{ ID: rowID, Count: n, }) continue } - row := f.Row(rowID) + row := f.row(rowID) if row.Count() > 0 { // Otherwise load from storage. - pairs = append(pairs, BitmapPair{ + pairs = append(pairs, bitmapPair{ ID: rowID, Count: row.Count(), }) } } - sort.Sort(BitmapPairs(pairs)) + sort.Sort(bitmapPairs(pairs)) return pairs } -// TopOptions represents options passed into the Top() function. -type TopOptions struct { +// topOptions represents options passed into the Top() function. +type topOptions struct { // Number of rows to return. N int @@ -1074,14 +1052,14 @@ type TopOptions struct { MinThreshold uint64 // Filter field name & values. - FilterField string + FilterName string FilterValues []interface{} TanimotoThreshold uint64 } // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. -func (f *Fragment) Checksum() []byte { +func (f *fragment) Checksum() []byte { h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) @@ -1089,22 +1067,15 @@ func (f *Fragment) Checksum() []byte { return h.Sum(nil) } -// BlockN returns the number of blocks in the fragment. -func (f *Fragment) BlockN() int { - f.mu.Lock() - defer f.mu.Unlock() - return int(f.storage.Max() / (HashBlockSize * SliceWidth)) -} - // InvalidateChecksums clears all cached block checksums. -func (f *Fragment) InvalidateChecksums() { +func (f *fragment) InvalidateChecksums() { f.mu.Lock() f.checksums = make(map[int][]byte) f.mu.Unlock() } // Blocks returns info for all blocks containing data. -func (f *Fragment) Blocks() []FragmentBlock { +func (f *fragment) Blocks() []FragmentBlock { f.mu.Lock() defer f.mu.Unlock() @@ -1122,16 +1093,16 @@ func (f *Fragment) Blocks() []FragmentBlock { if eof { return nil } - blockID := int(v / (HashBlockSize * SliceWidth)) + blockID := int(v / (HashBlockSize * ShardWidth)) for { // Check for multiple block checksums in a row. if n := f.readContiguousChecksums(&a, blockID); n > 0 { - itr.Seek(uint64(blockID+n) * HashBlockSize * SliceWidth) + itr.Seek(uint64(blockID+n) * HashBlockSize * ShardWidth) v, eof = itr.Next() if eof { break } - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) continue } @@ -1142,7 +1113,7 @@ func (f *Fragment) Blocks() []FragmentBlock { // Read all values for the block. for ; ; v, eof = itr.Next() { // Once we hit the next block, save the value for the next iteration. - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) if blockID != h.blockID || eof { break } @@ -1170,7 +1141,7 @@ func (f *Fragment) Blocks() []FragmentBlock { } // readContiguousChecksums appends multiple checksums in a row and returns the count added. -func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n int) { +func (f *fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n int) { for i := 0; ; i++ { chksum := f.checksums[blockID+i] if chksum == nil { @@ -1184,29 +1155,29 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as row & column ID pairs. -func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { +// blockData returns bits in a block as row & column ID pairs. +func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() - f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) { - rowIDs = append(rowIDs, i/SliceWidth) - columnIDs = append(columnIDs, i%SliceWidth) + f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) { + rowIDs = append(rowIDs, i/ShardWidth) + columnIDs = append(columnIDs, i%ShardWidth) }) return } -// MergeBlock compares the block's bits and computes a diff with another set of block bits. +// mergeBlock compares the block's bits and computes a diff with another set of block bits. // The state of a bit is determined by consensus from all blocks being considered. // // For example, if 3 blocks are compared and two have a set bit and one has a // cleared bit then the bit is considered cleared. The function returns the // diff per incoming block so that all can be in sync. -func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { +func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].RowIDs) != len(data[i].ColumnIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) + if len(data[i].rowIDs) != len(data[i].columnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs)) } } @@ -1214,26 +1185,26 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e defer f.mu.Unlock() // Track sets and clears for all blocks (including local). - sets = make([]PairSet, len(data)+1) - clears = make([]PairSet, len(data)+1) + sets = make([]pairSet, len(data)+1) + clears = make([]pairSet, len(data)+1) // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize - maxColumnID := uint64(SliceWidth) + maxColumnID := uint64(ShardWidth) // Create buffered iterator for local block. - itrs := make([]*BufIterator, 1, len(data)+1) - itrs[0] = NewBufIterator( - NewLimitIterator( - NewRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID, + itrs := make([]*bufIterator, 1, len(data)+1) + itrs[0] = newBufIterator( + newLimitIterator( + newRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID, ), ) // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) - itr = NewLimitIterator(itr, maxRowID, maxColumnID) - itrs = append(itrs, NewBufIterator(itr)) + var itr iterator = newSliceIterator(data[i].rowIDs, data[i].columnIDs) + itr = newLimitIterator(itr, maxRowID, maxColumnID) + itrs = append(itrs, newBufIterator(itr)) } // Seek to initial pair. @@ -1296,25 +1267,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) - sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + sets[i].rowIDs = append(sets[i].rowIDs, min.rowID) + sets[i].columnIDs = append(sets[i].columnIDs, min.columnID) } else { - clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) - clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + clears[i].rowIDs = append(sets[i].rowIDs, min.rowID) + clears[i].columnIDs = append(sets[i].columnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ColumnIDs { - if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { + for i := range sets[0].columnIDs { + if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.shard*ShardWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. - for i := range clears[0].ColumnIDs { - if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { + for i := range clears[0].columnIDs { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.shard*ShardWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1322,9 +1293,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of bits and then snapshots the storage. +// bulkImport bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { +func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of row ids and column ids. @@ -1373,7 +1344,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { // Import should ALWAYS have row() load a new row from fragment.storage // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) + f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count()) } f.cache.Invalidate() @@ -1392,8 +1363,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { return nil } -// ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { +// importValue bulk imports a set of range-encoded values. +func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1408,7 +1379,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error for i := range columnIDs { columnID, value := columnIDs[i], values[i] - _, err := f.importSetFieldValue(columnID, bitDepth, value) + _, err := f.importSetValue(columnID, bitDepth, value) if err != nil { return errors.Wrap(err, "setting") } @@ -1427,7 +1398,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error // incrementOpN increase the operation count by one. // If the count exceeds the maximum allowed then a snapshot is performed. -func (f *Fragment) incrementOpN() error { +func (f *fragment) incrementOpN() error { f.opN++ if f.opN <= f.MaxOpN { return nil @@ -1440,7 +1411,7 @@ func (f *Fragment) incrementOpN() error { } // Snapshot writes the storage bitmap to disk and reopens it. -func (f *Fragment) Snapshot() error { +func (f *fragment) Snapshot() error { f.mu.Lock() defer f.mu.Unlock() return f.snapshot() @@ -1451,14 +1422,14 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { stats.Histogram("snapshot", elapsed.Seconds(), 1.0) } -func (f *Fragment) snapshot() error { - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) +func (f *fragment) snapshot() error { + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) + completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. - snapshotPath := f.path + SnapshotExt + snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { return fmt.Errorf("create snapshot file: %s", err) @@ -1497,20 +1468,20 @@ func (f *Fragment) snapshot() error { } // RecalculateCache rebuilds the cache regardless of invalidate time delay. -func (f *Fragment) RecalculateCache() { +func (f *fragment) RecalculateCache() { f.mu.Lock() f.cache.Recalculate() f.mu.Unlock() } // FlushCache writes the cache data to disk. -func (f *Fragment) FlushCache() error { +func (f *fragment) FlushCache() error { f.mu.Lock() defer f.mu.Unlock() return f.flushCache() } -func (f *Fragment) flushCache() error { +func (f *fragment) flushCache() error { if f.cache == nil { return nil } @@ -1529,7 +1500,7 @@ func (f *Fragment) flushCache() error { } // Write to disk. - if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1537,7 +1508,7 @@ func (f *Fragment) flushCache() error { } // WriteTo writes the fragment's data to w. -func (f *Fragment) WriteTo(w io.Writer) (n int64, err error) { +func (f *fragment) WriteTo(w io.Writer) (n int64, err error) { // Force cache flush. if err := f.FlushCache(); err != nil { return 0, errors.Wrap(err, "flushing cache") @@ -1554,7 +1525,7 @@ func (f *Fragment) WriteTo(w io.Writer) (n int64, err error) { return 0, nil } -func (f *Fragment) writeStorageToArchive(tw *tar.Writer) error { +func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { // Open separate file descriptor to read from. file, err := os.Open(f.path) if err != nil { @@ -1598,12 +1569,12 @@ func (f *Fragment) writeStorageToArchive(tw *tar.Writer) error { return nil } -func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error { +func (f *fragment) writeCacheToArchive(tw *tar.Writer) error { f.mu.Lock() defer f.mu.Unlock() // Read cache into buffer. - buf, err := ioutil.ReadFile(f.CachePath()) + buf, err := ioutil.ReadFile(f.cachePath()) if os.IsNotExist(err) { return nil } else if err != nil { @@ -1628,7 +1599,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error { } // ReadFrom reads a data file from r and loads it into the fragment. -func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) { +func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -1660,9 +1631,9 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) { return 0, nil } -func (f *Fragment) readStorageFromArchive(r io.Reader) error { +func (f *fragment) readStorageFromArchive(r io.Reader) error { // Create a temporary file to copy into. - path := f.path + CopyExt + path := f.path + copyExt file, err := os.Create(path) if err != nil { return errors.Wrap(err, "creating directory") @@ -1692,12 +1663,12 @@ func (f *Fragment) readStorageFromArchive(r io.Reader) error { return nil } -func (f *Fragment) readCacheFromArchive(r io.Reader) error { +func (f *fragment) readCacheFromArchive(r io.Reader) error { // Slurp data from reader and write to disk. buf, err := ioutil.ReadAll(r) if err != nil { return errors.Wrap(err, "reading") - } else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + } else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1741,19 +1712,18 @@ func (h *blockHasher) WriteValue(v uint64) { h.hash.Write(h.buf[:]) } -// FragmentSyncer syncs a local fragment to one on a remote host. -type FragmentSyncer struct { - Fragment *Fragment +// fragmentSyncer syncs a local fragment to one on a remote host. +type fragmentSyncer struct { + Fragment *fragment - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *cluster Closing <-chan struct{} } // isClosing returns true if the closing channel is closed. -func (s *FragmentSyncer) isClosing() bool { +func (s *fragmentSyncer) isClosing() bool { select { case <-s.Closing: return true @@ -1762,11 +1732,11 @@ func (s *FragmentSyncer) isClosing() bool { } } -// SyncFragment compares checksums for the local and remote fragments and +// syncFragment compares checksums for the local and remote fragments and // then merges any blocks which have differences. -func (s *FragmentSyncer) SyncFragment() error { +func (s *fragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard) if len(nodes) == 1 { return nil } @@ -1782,8 +1752,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice()) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1842,13 +1811,13 @@ func (s *FragmentSyncer) SyncFragment() error { // syncBlock sends and receives all rows for a given block. // Returns an error if any remote hosts are unreachable. -func (s *FragmentSyncer) syncBlock(id int) error { +func (s *fragmentSyncer) syncBlock(id int) error { f := s.Fragment // Read pairs from each remote block. - var pairSets []PairSet - var clients []InternalClient - for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { + var uris []*URI + var pairSets []pairSet + for _, node := range s.Cluster.shardNodes(f.index, f.shard) { if s.Node.ID == node.ID { continue } @@ -1858,18 +1827,18 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - clients = append(clients, client) + uri := &node.URI + uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.shard, id) if err != nil { return errors.Wrap(err, "getting block") } - pairSets = append(pairSets, PairSet{ - ColumnIDs: columnIDs, - RowIDs: rowIDs, + pairSets = append(pairSets, pairSet{ + columnIDs: columnIDs, + rowIDs: rowIDs, }) } @@ -1879,36 +1848,36 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Merge blocks together. - sets, clears, err := f.MergeBlock(id, pairSets) + sets, clears, err := f.mergeBlock(id, pairSets) if err != nil { return errors.Wrap(err, "merging") } // Write updates to remote blocks. - for i := 0; i < len(clients); i++ { + for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] count := 0 // Ignore if there are no differences. - if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { + if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { continue } // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.ColumnIDs) + len(clear.ColumnIDs) - maxWrites := s.Cluster.MaxWritesPerRequest + total := len(set.columnIDs) + len(clear.columnIDs) + maxWrites := s.Cluster.maxWritesPerRequest if maxWrites <= 0 { maxWrites = 5000 } buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. - for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + for j := 0; j < len(set.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) count++ } - for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + for j := 0; j < len(clear.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) count++ } @@ -1920,11 +1889,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Execute query. - queryRequest := &internal.QueryRequest{ + queryRequest := &QueryRequest{ Query: buffers[k].String(), Remote: true, } - _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest) if err != nil { return errors.Wrap(err, "executing") } @@ -1942,10 +1911,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length row and column id lists. -type PairSet struct { - RowIDs []uint64 - ColumnIDs []uint64 +// pairSet is a list of equal length row and column id lists. +type pairSet struct { + rowIDs []uint64 + columnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1962,7 +1931,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the row position of a row/column pair. -func Pos(rowID, columnID uint64) uint64 { - return (rowID * SliceWidth) + (columnID % SliceWidth) +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { + return (rowID * ShardWidth) + (columnID % ShardWidth) } diff --git a/fragment_test.go b/fragment_internal_test.go similarity index 56% rename from fragment_test.go rename to fragment_internal_test.go index 5c5f03b16..e665a10ba 100644 --- a/fragment_test.go +++ b/fragment_internal_test.go @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "bytes" "flag" + "io/ioutil" "math" "reflect" "testing" "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" ) // Test flags @@ -35,82 +34,79 @@ var ( FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth - // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1); err != nil { + if _, err := f.setBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6); err != nil { + } else if _, err := f.setBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0); err != nil { + } else if _, err := f.setBit(121, 0); err != nil { t.Fatal(err) } // Verify counts on rows. - if n := f.Row(120).Count(); n != 2 { + if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(120).Count(); n != 2 { + } else if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.Row(1000).Count(); n != 1 { + if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } -// Ensure a fragment can set & read a field value. -func TestFragment_SetFieldValue(t *testing.T) { +// Ensure a fragment can set & read a value. +func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -119,7 +115,7 @@ func TestFragment_SetFieldValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -127,25 +123,25 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.SetFieldValue(100, 16, 2028); err != nil { + if changed, err := f.setValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -155,18 +151,18 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("NotExists", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 10, 20); err != nil { + if changed, err := f.setValue(100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.FieldValue(100, 11); err != nil { + if value, exists, err := f.value(100, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -185,7 +181,7 @@ func TestFragment_SetFieldValue(t *testing.T) { values[i] = values[i] % (1 << bitDepth) } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. @@ -195,14 +191,14 @@ func TestFragment_SetFieldValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.FieldValue(columnID, bitDepth) + v, exists, err := f.value(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -219,26 +215,26 @@ func TestFragment_SetFieldValue(t *testing.T) { }) } -// Ensure a fragment can sum field values. -func TestFragment_FieldSum(t *testing.T) { +// Ensure a fragment can sum values. +func TestFragment_Sum(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -248,7 +244,7 @@ func TestFragment_FieldSum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -258,45 +254,45 @@ func TestFragment_FieldSum(t *testing.T) { }) } -// Ensure a fragment can find the min and max of field values. -func TestFragment_FieldMinMax(t *testing.T) { +// Ensure a fragment can find the min and max of values. +func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(7000, bitDepth, 0); err != nil { t.Fatal(err) } t.Run("Min", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 0, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -308,19 +304,19 @@ func TestFragment_FieldMinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 2818, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -331,27 +327,27 @@ func TestFragment_FieldMinMax(t *testing.T) { }) } -// Ensure a fragment query for matching fields. -func TestFragment_FieldRange(t *testing.T) { +// Ensure a fragment query for matching values. +func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -359,22 +355,22 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("NEQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -382,47 +378,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields less than (ending with set column). - if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { + // Query for values less than (ending with set column). + if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than (ending with unset column). - if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { + // Query for values less than (ending with unset column). + if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with set column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { + // Query for values less than or equal to (ending with set column). + if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with unset column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { + // Query for values less than or equal to (ending with unset column). + if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -430,47 +426,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { + // Query for values greater than (ending with unset bit). + if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { + // Query for values greater than (ending with set bit). + if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { + // Query for values greater than or equal to (ending with unset bit). + if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { + // Query for values greater than or equal to (ending with set bit). + if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -478,47 +474,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { + // Query for values greater than (ending with unset column). + if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { + // Query for values greater than (ending with set column). + if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { + // Query for values greater than or equal to (ending with unset column). + if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { + // Query for values greater than or equal to (ending with set column). + if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -528,50 +524,50 @@ func TestFragment_FieldRange(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20); err != nil { + if _, err := f.setBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38); err != nil { + } else if _, err := f.setBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37); err != nil { + } else if _, err := f.setBit(2, 37); err != nil { t.Fatal(err) } // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { @@ -586,75 +582,75 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil { + if pairs, err := f.top(topOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[1] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. - f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) - f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) + f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) + f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{ + if pairs, err := f.top(topOptions{ N: 2, - FilterField: "x", + FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, }); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[0] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 1}) { + } else if pairs[1] != (Pair{ID: 101, Count: 1}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on various rows. - f.MustSetBits(100, 1, 10, 11, 12) // one intersection - f.MustSetBits(101, 1, 2, 3, 4) // three intersections - f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections - f.MustSetBits(103, 1000, 1001, 1002) // no intersection + f.mustSetBits(100, 1, 10, 11, 12) // one intersection + f.mustSetBits(101, 1, 2, 3, 4) // three intersections + f.mustSetBits(102, 1, 2, 4, 5, 6) // two intersections + f.mustSetBits(103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(topOptions{N: 3, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, {ID: 102, Count: 2}, {ID: 100, Count: 1}, @@ -669,11 +665,11 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow( + src := NewRow( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) @@ -681,15 +677,15 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { - f.MustSetBits(i, j) + f.mustSetBits(i, j) } } f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(topOptions{N: 10, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, {ID: 998, Count: 18}, {ID: 997, Count: 17}, @@ -707,18 +703,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, {ID: 100, Count: 3}, }) { @@ -728,83 +724,79 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(topOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) { + } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) } } // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { - slice := uint64(0) + shard := uint64(0) cacheSize := uint32(3) // Create Index. - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, cacheSize)) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.createViewIfNotExists(viewStandard) if err != nil { t.Fatal(err) } // Create fragment. - frag, err := view.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } // Close the storage so we can re-open it without encountering a flock. frag.Close() - f := &test.Fragment{ - Fragment: frag, - RowAttrStore: test.MustOpenAttrStore(), - } - f.Fragment.RowAttrStore = f.RowAttrStore + f := frag if err := f.Open(); err != nil { panic(err) } defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) - f.MustSetBits(103, 8, 9, 10, 11, 12, 13) - f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) - f.MustSetBits(105, 10, 11) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(103, 8, 9, 10, 11, 12, 13) + f.mustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.mustSetBits(105, 10, 11) f.RecalculateCache() - p := []pilosa.Pair{ + p := []Pair{ {ID: 104, Count: 7}, {ID: 103, Count: 6}, {ID: 102, Count: 5}, } // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + if pairs, err := f.top(topOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) - } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { + } else if pairs[0] != (Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) @@ -813,14 +805,14 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200); err != nil { + if _, err := f.setBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { + } else if _, err := f.setBit(HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -832,14 +824,14 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Retrieve initial checksum. - var prev []pilosa.FragmentBlock + var prev []FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0); err != nil { + if _, err := f.setBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -849,7 +841,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different row. - if _, err := f.SetBit(20, 0); err != nil { + if _, err := f.setBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -859,7 +851,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different column. - if _, err := f.SetBit(20, 100); err != nil { + if _, err := f.setBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -870,11 +862,11 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1); err != nil { + if _, err := f.setBit(100, 1); err != nil { t.Fatal(err) } @@ -888,31 +880,31 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) defer f.Close() // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*lruCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the fragment. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*lruCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -920,17 +912,17 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", OptFieldTypeSet(CacheTypeRanked, DefaultCacheSize)) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.createViewIfNotExists(viewStandard) if err != nil { t.Fatal(err) } @@ -943,29 +935,29 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*rankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the index. - if err := index.Reopen(); err != nil { + if err := index.reopen(); err != nil { t.Fatal(err) } // Re-fetch fragment. - f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0) + f = index.Field("f").view(viewStandard).Fragment(0) // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*rankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -973,20 +965,20 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f0 := mustOpenFragment("i", "f", viewStandard, 0, "") defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1); err != nil { + if _, err := f0.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2); err != nil { + } else if _, err := f0.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f0.ClearBit(1000, 1); err != nil { + } else if _, err := f0.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify cache is populated. - if n := f0.Cache().Len(); n != 1 { + if n := f0.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } @@ -998,7 +990,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f1 := mustOpenFragment("i", "f", viewStandard, 0, "") if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -1006,21 +998,21 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify cache is in other fragment. - if n := f1.Cache().Len(); n != 1 { + if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } // Verify data in other fragment. - if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns: %+v", a) } // Close and reopen the fragment & verify the data. - if err := f1.Reopen(); err != nil { + if err := f1.reopen(); err != nil { t.Fatal(err) - } else if n := f1.Cache().Len(); n != 1 { + } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1031,7 +1023,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1047,18 +1039,18 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() f.MaxOpN = math.MaxInt32 // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i)); err != nil { + if _, err := f.setBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i)); err != nil { + if _, err := f.setBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -1071,67 +1063,67 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { + if n := f.row(1).intersectionCount(f.row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } } func TestFragment_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(topOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(topOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) - } else if pairs[2] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[2] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[2]) } } func TestFragment_Snapshot_Run(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Set bits on the fragment. for i := uint64(1); i < 3; i++ { - if _, err := f.SetBit(1000, i); err != nil { + if _, err := f.setBit(1000, i); err != nil { t.Fatal(err) } } @@ -1139,14 +1131,14 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -1158,7 +1150,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := newFragment(*FragmentPath, "i", "f", viewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1177,7 +1169,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { } func BenchmarkFragment_FullSnapshot(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() // Generate some intersecting data. maxX := 1048576 / 2 @@ -1189,13 +1181,13 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { for row := 0; row < 100; row++ { val := 1 i := 0 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 i++ } - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } if row > max { @@ -1214,7 +1206,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { } func BenchmarkFragment_Import(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Close() maxX := 1048576 * 5 * 2 sz := maxX @@ -1223,7 +1215,7 @@ func BenchmarkFragment_Import(b *testing.B) { i := 0 for row := 0; row < 100; row++ { val := 1 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 @@ -1236,8 +1228,53 @@ func BenchmarkFragment_Import(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } } } + +///////////////////////////////////////////////////////////////////// + +// mustOpenFragment returns a new instance of Fragment with a temporary path. +func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + if cacheType == "" { + cacheType = DefaultCacheType + } + + f := newFragment(file.Name(), index, field, view, shard) + f.CacheType = cacheType + f.RowAttrStore = newMemAttrStore() + + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Reopen closes the fragment and reopens it as a new instance. +func (f *fragment) reopen() error { + if err := f.Close(); err != nil { + return err + } + if err := f.Open(); err != nil { + return err + } + return nil +} + +// mustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.setBit(rowID, columnID); err != nil { + panic(err) + } + } +} diff --git a/frame.go b/frame.go deleted file mode 100644 index ab005edcc..000000000 --- a/frame.go +++ /dev/null @@ -1,1156 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "sync" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - "github.com/pkg/errors" -) - -// Default frame settings. -const ( - DefaultCacheType = CacheTypeRanked - - // Default ranked frame cache - DefaultCacheSize = 50000 -) - -// Frame represents a container for views. -type Frame struct { - mu sync.RWMutex - path string - index string - name string - - views map[string]*View - - // Row attribute storage and cache - rowAttrStore AttrStore - - broadcaster Broadcaster - Stats StatsClient - - // Frame options. - cacheType string - cacheSize uint32 - timeQuantum TimeQuantum - fields []*Field - - Logger Logger -} - -// NewFrame returns a new instance of frame. -func NewFrame(path, index, name string) (*Frame, error) { - err := ValidateName(name) - if err != nil { - return nil, err - } - - return &Frame{ - path: path, - index: index, - name: name, - - views: make(map[string]*View), - - rowAttrStore: NopAttrStore, - - broadcaster: NopBroadcaster, - Stats: NopStatsClient, - - cacheType: DefaultCacheType, - cacheSize: DefaultCacheSize, - //timeQuantum - //fields - - Logger: NopLogger, - }, nil -} - -// Name returns the name the frame was initialized with. -func (f *Frame) Name() string { return f.name } - -// Index returns the index name the frame was initialized with. -func (f *Frame) Index() string { return f.index } - -// Path returns the path the frame was initialized with. -func (f *Frame) Path() string { return f.path } - -// RowAttrStore returns the attribute storage. -func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore } - -// MaxSlice returns the max slice in the frame. -func (f *Frame) MaxSlice() uint64 { - f.mu.RLock() - defer f.mu.RUnlock() - - var max uint64 - for _, view := range f.views { - if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { - max = viewMaxSlice - } - } - return max -} - -// CacheType returns the caching mode for the frame. -func (f *Frame) CacheType() string { - return f.cacheType -} - -// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. -// defaults to DefaultCacheSize 50000 -func (f *Frame) SetCacheSize(v uint32) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Ignore if no change occurred. - if v == 0 || f.cacheSize == v { - return nil - } - - // Persist meta data to disk on change. - f.cacheSize = v - if err := f.saveMeta(); err != nil { - return errors.Wrap(err, "saving") - } - - return nil -} - -// CacheSize returns the ranked frame cache size. -func (f *Frame) CacheSize() uint32 { - f.mu.Lock() - v := f.cacheSize - f.mu.Unlock() - return v -} - -// Options returns all options for this frame. -func (f *Frame) Options() FrameOptions { - f.mu.RLock() - defer f.mu.RUnlock() - return f.options() -} - -func (f *Frame) options() FrameOptions { - return FrameOptions{ - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: f.timeQuantum, - Fields: f.fields, - } -} - -// Open opens and initializes the frame. -func (f *Frame) Open() error { - if err := func() error { - // Ensure the frame's path exists. - if err := os.MkdirAll(f.path, 0777); err != nil { - return errors.Wrap(err, "creating frame dir") - } - - if err := f.loadMeta(); err != nil { - return errors.Wrap(err, "loading meta") - } - - if err := f.openViews(); err != nil { - return errors.Wrap(err, "opening views") - } - - if err := f.rowAttrStore.Open(); err != nil { - return errors.Wrap(err, "opening attrstore") - } - - return nil - }(); err != nil { - f.Close() - return err - } - - return nil -} - -// openViews opens and initializes the views inside the frame. -func (f *Frame) openViews() error { - file, err := os.Open(filepath.Join(f.path, "views")) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "opening view directory") - } - defer file.Close() - - fis, err := file.Readdir(0) - if err != nil { - return errors.Wrap(err, "reading directory") - } - - for _, fi := range fis { - if !fi.IsDir() { - continue - } - - name := filepath.Base(fi.Name()) - view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { - return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) - } - view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view - } - - return nil -} - -// loadMeta reads meta data for the frame, if any. -func (f *Frame) loadMeta() error { - var pb internal.FrameMeta - - // Read data from meta file. - buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) - if os.IsNotExist(err) { - f.cacheType = DefaultCacheType - f.cacheSize = DefaultCacheSize - f.timeQuantum = "" - //f.fields - return nil - } else if err != nil { - return errors.Wrap(err, "reading meta") - } else { - if err := proto.Unmarshal(buf, &pb); err != nil { - return errors.Wrap(err, "unmarshaling") - } - } - - // Copy metadata fields. - f.cacheType = pb.CacheType - if f.cacheType == "" { - f.cacheType = DefaultCacheType - } - f.cacheSize = pb.CacheSize - f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.fields = decodeFields(pb.Fields) - - return nil -} - -// saveMeta writes meta data for the frame. -func (f *Frame) saveMeta() error { - // Marshal metadata. - fo := f.options() - buf, err := proto.Marshal(fo.Encode()) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Write to meta file. - if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { - return errors.Wrap(err, "writing meta") - } - - return nil -} - -// Close closes the frame and its views. -func (f *Frame) Close() error { - f.mu.Lock() - defer f.mu.Unlock() - - // Close the attribute store. - if f.rowAttrStore != nil { - _ = f.rowAttrStore.Close() - } - - // Close all views. - for _, view := range f.views { - if err := view.Close(); err != nil { - return err - } - } - f.views = make(map[string]*View) - - return nil -} - -// Field returns a field by name. -func (f *Frame) Field(name string) *Field { - f.mu.RLock() - defer f.mu.RUnlock() - for _, field := range f.fields { - if field.Name == name { - return field - } - } - return nil -} - -// Fields returns the fields on the frame. -func (f *Frame) Fields() []*Field { - f.mu.RLock() - defer f.mu.RUnlock() - return f.fields -} - -// HasField returns true if a field exists on the frame. -func (f *Frame) HasField(name string) bool { - for _, fld := range f.fields { - if fld.Name == name { - return true - } - } - return false -} - -// CreateField creates a new field on the frame. -func (f *Frame) CreateField(field *Field) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Append field. - if err := f.addField(field); err != nil { - return err - } - f.saveMeta() - return nil -} - -// addField adds a single field to fields. -func (f *Frame) addField(field *Field) error { - if err := ValidateField(field); err != nil { - return errors.Wrap(err, "validating field") - } else if f.HasField(field.Name) { - return ErrFieldExists - } - - // Add field to list. - f.fields = append(f.fields, field) - - // Sort fields by name. - sort.Slice(f.fields, func(i, j int) bool { - return f.fields[i].Name < f.fields[j].Name - }) - - return nil -} - -// GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() ([]*Field, error) { - f.mu.RLock() - defer f.mu.RUnlock() - - err := f.loadMeta() - if err != nil { - return nil, errors.Wrap(err, "loading meta") - } - - return f.fields, nil -} - -// DeleteField deletes an existing field on the schema. -func (f *Frame) DeleteField(name string) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Remove field. - if err := f.deleteField(name); err != nil { - return err - } - - // Remove views. - viewName := ViewFieldPrefix + name - if view := f.views[viewName]; view != nil { - delete(f.views, viewName) - - if err := view.Close(); err != nil { - return errors.Wrap(err, "closing view") - } else if err := os.RemoveAll(view.Path()); err != nil { - return errors.Wrap(err, "deleting directory") - } - } - - return nil -} - -// deleteField removes a single field from fields. -func (f *Frame) deleteField(name string) error { - for i, field := range f.fields { - if field.Name == name { - copy(f.fields[i:], f.fields[i+1:]) - f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil - return nil - } - } - return ErrFieldNotFound -} - -// TimeQuantum returns the time quantum for the frame. -func (f *Frame) TimeQuantum() TimeQuantum { - f.mu.Lock() - defer f.mu.Unlock() - return f.timeQuantum -} - -// SetTimeQuantum sets the time quantum for the frame. -func (f *Frame) SetTimeQuantum(q TimeQuantum) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on frame. - f.timeQuantum = q - - // Persist meta data to disk. - if err := f.saveMeta(); err != nil { - return errors.Wrap(err, "saving meta") - } - - return nil -} - -// ViewPath returns the path to a view in the frame. -func (f *Frame) ViewPath(name string) string { - return filepath.Join(f.path, "views", name) -} - -// View returns a view in the frame by name. -func (f *Frame) View(name string) *View { - f.mu.RLock() - defer f.mu.RUnlock() - return f.view(name) -} - -func (f *Frame) view(name string) *View { return f.views[name] } - -// Views returns a list of all views in the frame. -func (f *Frame) Views() []*View { - f.mu.RLock() - defer f.mu.RUnlock() - - other := make([]*View, 0, len(f.views)) - for _, view := range f.views { - other = append(other, view) - } - return other -} - -// viewNames returns a list of all views (as a string) in the frame. -func (f *Frame) viewNames() []string { - f.mu.Lock() - defer f.mu.Unlock() - - other := make([]string, 0, len(f.views)) - for viewName, _ := range f.views { - other = append(other, viewName) - } - return other -} - -// RecalculateCaches recalculates caches on every view in the frame. -func (f *Frame) RecalculateCaches() { - for _, view := range f.Views() { - view.RecalculateCaches() - } -} - -// CreateViewIfNotExists returns the named view, creating it if necessary. -// Additionally, a CreateViewMessage is sent to the cluster. -func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { - - view, created, err := f.createViewIfNotExistsBase(name) - if err != nil { - return nil, err - } - - if created { - // Broadcast view creation to the cluster. - err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ - Index: f.index, - Frame: f.name, - View: name, - }) - if err != nil { - return nil, errors.Wrap(err, "sending CreateView message") - } - } - - return view, nil -} - -// createViewIfNotExistsBase returns the named view, creating it if necessary. -// The returned bool indicates whether the view was created or not. -func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { - f.mu.Lock() - defer f.mu.Unlock() - - if view := f.views[name]; view != nil { - return view, false, nil - } - - view := f.newView(f.ViewPath(name), name) - - if err := view.Open(); err != nil { - return nil, false, errors.Wrap(err, "opening view") - } - view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view - - return view, true, nil -} - -func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.index, f.name, name, f.cacheSize) - view.cacheType = f.cacheType - view.Logger = f.Logger - view.RowAttrStore = f.rowAttrStore - view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) - view.broadcaster = f.broadcaster - return view -} - -// DeleteView removes the view from the frame. -func (f *Frame) DeleteView(name string) error { - view := f.views[name] - if view == nil { - return ErrInvalidView - } - - // Close data files before deletion. - if err := view.Close(); err != nil { - return errors.Wrap(err, "closing view") - } - - // Delete view directory. - if err := os.RemoveAll(view.Path()); err != nil { - return errors.Wrap(err, "deleting directory") - } - - delete(f.views, name) - - return nil -} - -// SetBit sets a bit on a view within the frame. -func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { - // Validate view name. - if !IsValidView(name) { - return false, ErrInvalidView - } - - // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(name) - if err != nil { - return changed, errors.Wrap(err, "creating view") - } - - // Set non-time bit. - if v, err := view.SetBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") - } else if v { - changed = v - } - - // Exit early if no timestamp is specified. - if t == nil { - return changed, nil - } - - // If a timestamp is specified then set bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) - } - - if c, err := view.SetBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) - } else if c { - changed = true - } - } - - return changed, nil -} - -// ClearBit clears a bit within the frame. -func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { - // Validate view name. - if !IsValidView(name) { - return false, ErrInvalidView - } - - // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(name) - if err != nil { - return changed, errors.Wrap(err, "creating view") - } - - // Clear non-time bit. - if v, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") - } else if v { - changed = v - } - - // Exit early if no timestamp is specified. - if t == nil { - return changed, nil - } - - // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) - } - - if c, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) - } else if c { - changed = true - } - } - - return changed, nil -} - -// FieldValue reads a field value for a column. -func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) { - field := f.Field(name) - if field == nil { - return 0, false, ErrFieldNotFound - } - - // Fetch target view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, false, nil - } - - v, exists, err := view.FieldValue(columnID, field.BitDepth()) - if err != nil { - return 0, false, err - } else if !exists { - return 0, false, nil - } - return int64(v) + field.Min, true, nil -} - -// SetFieldValue sets a field value for a column. -func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) { - // Fetch field and validate value. - field := f.Field(name) - if field == nil { - return false, ErrFieldNotFound - } else if value < field.Min { - return false, ErrFieldValueTooLow - } else if value > field.Max { - return false, ErrFieldValueTooHigh - } - - // Fetch target view. - view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name) - if err != nil { - return false, errors.Wrap(err, "creating view") - } - - // Determine base value to store. - baseValue := uint64(value - field.Min) - - return view.SetFieldValue(columnID, field.BitDepth(), baseValue) -} - -// FieldSum returns the sum and count for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil -} - -// FieldMin returns the min for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vmin) + field.Min, int64(vcount), nil -} - -// FieldMax returns the max for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vmax) + field.Min, int64(vcount), nil -} - -func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, error) { - // Retrieve and validate field. - field := f.Field(name) - if field == nil { - return nil, ErrFieldNotFound - } else if predicate < field.Min || predicate > field.Max { - return nil, nil - } - - // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return nil, nil - } - - baseValue, outOfRange := field.BaseValue(op, predicate) - if outOfRange { - return NewRow(), nil - } - - return view.FieldRange(op, field.BitDepth(), baseValue) -} - -func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { - // Retrieve and validate field. - field := f.Field(name) - if field == nil { - return nil, ErrFieldNotFound - } else if predicateMin > predicateMax { - return nil, ErrInvalidBetweenValue - } - - // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return nil, nil - } - - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicateMin, predicateMax) - if outOfRange { - return NewRow(), nil - } - - return view.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) -} - -// Import bulk imports data. -func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { - // Determine quantum if timestamps are set. - q := f.TimeQuantum() - if hasTime(timestamps) && q == "" { - return errors.New("time quantum not set in either index or frame") - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importData) - for i := range rowIDs { - rowID, columnID := rowIDs[i], columnIDs[i] - var timestamp *time.Time - if len(timestamps) > i { - timestamp = timestamps[i] - } - - var standard []string - if timestamp == nil { - standard = []string{ViewStandard} - } else { - standard = ViewsByTime(ViewStandard, *timestamp, q) - // In order to match the logic of `SetBit()`, we want bits - // with timestamps to write to both time and standard views. - standard = append(standard, ViewStandard) - } - - // Attach bit to each standard view. - for _, name := range standard { - key := importKey{View: name, Slice: columnID / SliceWidth} - data := dataByFragment[key] - data.RowIDs = append(data.RowIDs, rowID) - data.ColumnIDs = append(data.ColumnIDs, columnID) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - view, err := f.CreateViewIfNotExists(key.View) - if err != nil { - return errors.Wrap(err, "creating view") - } - - frag, err := view.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return errors.Wrap(err, "creating view") - } - - if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { - return err - } - } - - return nil -} - -// ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - viewName := ViewFieldPrefix + fieldName - // Get the field so we know bitDepth. - field := f.Field(fieldName) - if field == nil { - return fmt.Errorf("Field does not exist: %s", fieldName) - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importValueData) - for i := range columnIDs { - columnID, value := columnIDs[i], values[i] - if int64(value) > field.Max { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooHigh, columnID, value) - } else if int64(value) < field.Min { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooLow, columnID, value) - } - - // Attach value to each field view. - for _, name := range []string{viewName} { - key := importKey{View: name, Slice: columnID / SliceWidth} - data := dataByFragment[key] - data.ColumnIDs = append(data.ColumnIDs, columnID) - data.Values = append(data.Values, value) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - - // The view must already exist (i.e. we can't create it) - // because we need to know bitDepth (based on min/max value). - view, err := f.CreateViewIfNotExists(key.View) - if err != nil { - return errors.Wrap(err, "creating view") - } - - frag, err := view.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - baseValues := make([]uint64, len(data.Values)) - for i, value := range data.Values { - baseValues[i] = uint64(value - field.Min) - } - - if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil { - return err - } - } - - return nil -} - -// encodeFrames converts a into its internal representation. -func encodeFrames(a []*Frame) []*internal.Frame { - other := make([]*internal.Frame, len(a)) - for i := range a { - other[i] = encodeFrame(a[i]) - } - return other -} - -// encodeFrame converts f into its internal representation. -func encodeFrame(f *Frame) *internal.Frame { - fo := f.options() - return &internal.Frame{ - Name: f.name, - Meta: fo.Encode(), - Views: f.viewNames(), - } -} - -type frameSlice []*Frame - -func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameSlice) Len() int { return len(p) } -func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } - -// FrameInfo represents schema information for a frame. -type FrameInfo struct { - Name string `json:"name"` - Options FrameOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` -} - -type frameInfoSlice []*FrameInfo - -func (p frameInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameInfoSlice) Len() int { return len(p) } -func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } - -// FrameOptions represents options to set when initializing a frame. -type FrameOptions struct { - CacheType string `json:"cacheType,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*Field `json:"fields,omitempty"` -} - -// Encode converts o into its internal representation. -func (o *FrameOptions) Encode() *internal.FrameMeta { - return encodeFrameOptions(o) -} - -func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { - if o == nil { - return nil - } - return &internal.FrameMeta{ - CacheType: o.CacheType, - CacheSize: o.CacheSize, - TimeQuantum: string(o.TimeQuantum), - Fields: encodeFields(o.Fields), - } -} - -func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { - if options == nil { - return nil - } - return &FrameOptions{ - CacheType: options.CacheType, - CacheSize: options.CacheSize, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Fields: decodeFields(options.Fields), - } -} - -// List of field data types. -const ( - FieldTypeInt = "int" -) - -func IsValidFieldType(v string) bool { - switch v { - case FieldTypeInt: - return true - default: - return false - } -} - -// Field represents a range field on a frame. -type Field struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -// BitDepth returns the number of bits required to store a value between min & max. -func (f *Field) BitDepth() uint { - for i := uint(0); i < 63; i++ { - if f.Max-f.Min < (1 << i) { - return i - } - } - return 63 -} - -// BaseValue adjusts the value to align with the range for Field for a certain -// operation type. -// Note: There is an edge case for GT and LT where this returns a baseValue -// that does not fully encompass the range. -// ex: Field.Min = 0, Field.Max = 1023 -// BaseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively -// exclude any columns with value = 1023. -// Note that in this case (because the range uses the full BitDepth 0 to 1023), -// we can't simply return 1024. -// In order to make this work, we effectively need to change the operator to LTE. -// Executor.executeFieldRangeSlice() takes this into account and returns -// `frag.FieldNotNull(field.BitDepth())` in such instances. -func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { - if op == pql.GT || op == pql.GTE { - if value > f.Max { - return baseValue, true - } else if value > f.Min { - baseValue = uint64(value - f.Min) - } - } else if op == pql.LT || op == pql.LTE { - if value < f.Min { - return baseValue, true - } else if value > f.Max { - baseValue = uint64(f.Max - f.Min) - } else { - baseValue = uint64(value - f.Min) - } - } else if op == pql.EQ || op == pql.NEQ { - if value < f.Min || value > f.Max { - return baseValue, true - } - baseValue = uint64(value - f.Min) - } - return baseValue, false -} - -// BaseValueBetween adjusts the min/max value to align with the range for Field. -func (f *Field) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { - if max < f.Min || min > f.Max { - return baseValueMin, baseValueMax, true - } - // Adjust min/max to range. - if min > f.Min { - baseValueMin = uint64(min - f.Min) - } - // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > f.Max { - baseValueMax = uint64(f.Max - f.Min) - } else if max > f.Min { - baseValueMax = uint64(max - f.Min) - } - return baseValueMin, baseValueMax, false -} - -func ValidateField(f *Field) error { - if f.Name == "" { - return ErrFieldNameRequired - } else if !IsValidFieldType(f.Type) { - return ErrInvalidFieldType - } else if f.Min > f.Max { - return ErrInvalidFieldRange - } - return nil -} - -func encodeFields(a []*Field) []*internal.Field { - if len(a) == 0 { - return nil - } - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -func decodeFields(a []*internal.Field) []*Field { - if len(a) == 0 { - return nil - } - other := make([]*Field, len(a)) - for i := range a { - other[i] = decodeField(a[i]) - } - return other -} - -func encodeField(f *Field) *internal.Field { - if f == nil { - return nil - } - return &internal.Field{ - Name: f.Name, - Type: f.Type, - Min: int64(f.Min), - Max: int64(f.Max), - } -} - -func decodeField(f *internal.Field) *Field { - if f == nil { - return nil - } - return &Field{ - Name: f.Name, - Type: f.Type, - Min: f.Min, - Max: f.Max, - } -} - -// importBitSet represents slices of row and column ids. -// This is used to sort data during import. -type importBitSet struct { - rowIDs, columnIDs []uint64 -} - -func (p importBitSet) Swap(i, j int) { - p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i] - p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i] -} -func (p importBitSet) Len() int { return len(p.rowIDs) } -func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } - -// Cache types. -const ( - CacheTypeLRU = "lru" - CacheTypeRanked = "ranked" - CacheTypeNone = "none" -) - -// IsValidCacheType returns true if v is a valid cache type. -func IsValidCacheType(v string) bool { - switch v { - case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: - return true - default: - return false - } -} diff --git a/frame_test.go b/frame_test.go deleted file mode 100644 index 78014bbce..000000000 --- a/frame_test.go +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "reflect" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" -) - -// Ensure frame can open and retrieve a view. -func TestFrame_CreateViewIfNotExists(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - // Create view. - view, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - // Retrieve existing view. - view2, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view != view2 { - t.Fatal("view mismatch") - } - - if view != f.View("v") { - t.Fatal("view mismatch") - } -} - -// Ensure frame can set its time quantum. -func TestFrame_SetTimeQuantum(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - // Set & retrieve time quantum. - if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload frame and verify that it is persisted. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - -// Ensure a frame can set & read a field value. -func TestFrame_SetFieldValue(t *testing.T) { - t.Run("OK", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value on first field. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Set value on same column but different field. - if changed, err := f.SetFieldValue(100, "field1", 25); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { - t.Fatal(err) - } else if value != 21 { - t.Fatalf("unexpected value: %d", value) - } else if !exists { - t.Fatal("expected value to exist") - } - - // Setting value should return no change. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if changed { - t.Fatal("expected no change") - } - }) - - t.Run("Overwrite", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Set different value. - if changed, err := f.SetFieldValue(100, "field0", 23); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { - t.Fatal(err) - } else if value != 23 { - t.Fatalf("unexpected value: %d", value) - } else if !exists { - t.Fatal("expected value to exist") - } - }) - - t.Run("ErrFieldNotFound", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrFieldValueTooLow", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrFieldValueTooHigh", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh { - t.Fatalf("unexpected error: %s", err) - } - }) -} - -func TestFrame_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - frame, err := pilosa.NewFrame(path, "i", ".meta") - if frame != nil { - t.Fatalf("unexpected frame name %s", err) - } -} - -// Ensure that frame name validation is consistent. -func TestFrame_NameValidation(t *testing.T) { - validFrameNames := []string{ - "foo", - "hyphen-ated", - "under_score", - "abc123", - "trailing_", - } - invalidFrameNames := []string{ - "", - "123abc", - "x.y", - "_foo", - "-bar", - "abc def", - "camelCase", - "UPPERCASE", - "a12345678901234567890123456789012345678901234567890123456789012345", - } - - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - for _, name := range validFrameNames { - _, err := pilosa.NewFrame(path, "i", name) - if err != nil { - t.Fatalf("unexpected frame name: %s %s", name, err) - } - } - for _, name := range invalidFrameNames { - _, err := pilosa.NewFrame(path, "i", name) - if err == nil { - t.Fatalf("expected error on frame name: %s", name) - } - } -} - -// Ensure frame can open and retrieve a view. -func TestFrame_DeleteView(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - viewName := pilosa.ViewStandard + "_v" - - // Create view. - view, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - err = f.DeleteView(viewName) - if err != nil { - t.Fatal(err) - } - - if f.View(viewName) != nil { - t.Fatal("view still exists in frame") - } - - // Recreate view with same name, verify that the old view was not reused. - view2, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == view2 { - t.Fatal("failed to create new view") - } -} - -// Ensure a field can adjust to its baseValue. -func TestField_BaseValue(t *testing.T) { - f0 := &pilosa.Field{ - Name: "f0", - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 900, - } - f1 := &pilosa.Field{ - Name: "f1", - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 1000, - } - - f2 := &pilosa.Field{ - Name: "f2", - Type: pilosa.FieldTypeInt, - Min: 100, - Max: 1100, - } - - t.Run("Normal Condition", func(t *testing.T) { - - for _, tt := range []struct { - f *pilosa.Field - op pql.Token - val int64 - expBaseValue uint64 - expOutOfRange bool - }{ - // LT - {f0, pql.LT, 5, 105, false}, - {f0, pql.LT, -8, 92, false}, - {f0, pql.LT, -108, 0, true}, - {f0, pql.LT, 1005, 1000, false}, - {f0, pql.LT, 0, 100, false}, - - {f1, pql.LT, 5, 5, false}, - {f1, pql.LT, -8, 0, true}, - {f1, pql.LT, 1005, 1000, false}, - {f1, pql.LT, 0, 0, false}, - - {f2, pql.LT, 5, 0, true}, - {f2, pql.LT, -8, 0, true}, - {f2, pql.LT, 105, 5, false}, - {f2, pql.LT, 1105, 1000, false}, - - // GT - {f0, pql.GT, -105, 0, false}, - {f0, pql.GT, 5, 105, false}, - {f0, pql.GT, 905, 0, true}, - {f0, pql.GT, 0, 100, false}, - - {f1, pql.GT, 5, 5, false}, - {f1, pql.GT, -8, 0, false}, - {f1, pql.GT, 1005, 0, true}, - {f1, pql.GT, 0, 0, false}, - - {f2, pql.GT, 5, 0, false}, - {f2, pql.GT, -8, 0, false}, - {f2, pql.GT, 105, 5, false}, - {f2, pql.GT, 1105, 0, true}, - - // EQ - {f0, pql.EQ, -105, 0, true}, - {f0, pql.EQ, 5, 105, false}, - {f0, pql.EQ, 905, 0, true}, - {f0, pql.EQ, 0, 100, false}, - - {f1, pql.EQ, 5, 5, false}, - {f1, pql.EQ, -8, 0, true}, - {f1, pql.EQ, 1005, 0, true}, - {f1, pql.EQ, 0, 0, false}, - - {f2, pql.EQ, 5, 0, true}, - {f2, pql.EQ, -8, 0, true}, - {f2, pql.EQ, 105, 5, false}, - {f2, pql.EQ, 1105, 0, true}, - } { - bv, oor := tt.f.BaseValue(tt.op, tt.val) - if oor != tt.expOutOfRange { - t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) - } - } - }) - - t.Run("Betwween Condition", func(t *testing.T) { - for _, tt := range []struct { - f *pilosa.Field - predMin int64 - predMax int64 - expBaseValueMin uint64 - expBaseValueMax uint64 - expOutOfRange bool - }{ - - {f0, -205, -105, 0, 0, true}, - {f0, -105, 80, 0, 180, false}, - {f0, 5, 20, 105, 120, false}, - {f0, 20, 1005, 120, 1000, false}, - {f0, 1005, 2000, 0, 0, true}, - - {f1, -105, -5, 0, 0, true}, - {f1, -5, 20, 0, 20, false}, - {f1, 5, 20, 5, 20, false}, - {f1, 20, 1005, 20, 1000, false}, - {f1, 1005, 2000, 0, 0, true}, - - {f2, 5, 95, 0, 0, true}, - {f2, 95, 120, 0, 20, false}, - {f2, 105, 120, 5, 20, false}, - {f2, 120, 1105, 20, 1000, false}, - {f2, 1105, 2000, 0, 0, true}, - } { - min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) - if oor != tt.expOutOfRange { - t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { - t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) - } - } - }) -} diff --git a/gcnotify/gcnotify.go b/gcnotify/gcnotify.go index 76953a378..54313403d 100644 --- a/gcnotify/gcnotify.go +++ b/gcnotify/gcnotify.go @@ -20,25 +20,25 @@ import ( ) // Ensure ActiveGCNotifier implements interface. -var _ pilosa.GCNotifier = &ActiveGCNotifier{} +var _ pilosa.GCNotifier = &activeGCNotifier{} -type ActiveGCNotifier struct { +type activeGCNotifier struct { gcn *gcnotifier.GCNotifier } // NewActiveGCNotifier creates an active GCNotifier. -func NewActiveGCNotifier() *ActiveGCNotifier { - return &ActiveGCNotifier{ +func NewActiveGCNotifier() *activeGCNotifier { + return &activeGCNotifier{ gcn: gcnotifier.New(), } } // Close implements the GCNotifier interface. -func (n *ActiveGCNotifier) Close() { +func (n *activeGCNotifier) Close() { n.gcn.Close() } // AfterGC implements the GCNotifier interface. -func (n *ActiveGCNotifier) AfterGC() <-chan struct{} { +func (n *activeGCNotifier) AfterGC() <-chan struct{} { return n.gcn.AfterGC() } diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 3310aeae1..8cdd77778 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -22,15 +22,15 @@ import ( var _ pilosa.SystemInfo = NewSystemInfo() -// SystemInfo is an implementation of pilosa.SystemInfo that uses gopsutil to collect information about the host OS. -type SystemInfo struct { +// systemInfo is an implementation of pilosa.systemInfo that uses gopsutil to collect information about the host OS. +type systemInfo struct { platform string family string osVersion string } // Uptime returns the system uptime in seconds. -func (s *SystemInfo) Uptime() (uptime uint64, err error) { +func (s *systemInfo) Uptime() (uptime uint64, err error) { hostInfo, err := host.Info() if err != nil { return 0, err @@ -39,7 +39,7 @@ func (s *SystemInfo) Uptime() (uptime uint64, err error) { } // collectPlatformInfo fetches and caches system platform information. -func (s *SystemInfo) collectPlatformInfo() error { +func (s *systemInfo) collectPlatformInfo() error { var err error if s.platform == "" { s.platform, s.family, s.osVersion, err = host.PlatformInformation() @@ -51,7 +51,7 @@ func (s *SystemInfo) collectPlatformInfo() error { } // Platform returns the system platform. -func (s *SystemInfo) Platform() (string, error) { +func (s *systemInfo) Platform() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -60,7 +60,7 @@ func (s *SystemInfo) Platform() (string, error) { } // Family returns the system family. -func (s *SystemInfo) Family() (string, error) { +func (s *systemInfo) Family() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -69,7 +69,7 @@ func (s *SystemInfo) Family() (string, error) { } // OSVersion returns the OS Version. -func (s *SystemInfo) OSVersion() (string, error) { +func (s *systemInfo) OSVersion() (string, error) { err := s.collectPlatformInfo() if err != nil { return "", err @@ -78,7 +78,7 @@ func (s *SystemInfo) OSVersion() (string, error) { } // MemFree returns the amount of free memory in bytes. -func (s *SystemInfo) MemFree() (uint64, error) { +func (s *systemInfo) MemFree() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -87,7 +87,7 @@ func (s *SystemInfo) MemFree() (uint64, error) { } // MemTotal returns the amount of total memory in bytes. -func (s *SystemInfo) MemTotal() (uint64, error) { +func (s *systemInfo) MemTotal() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -96,7 +96,7 @@ func (s *SystemInfo) MemTotal() (uint64, error) { } // MemUsed returns the amount of used memory in bytes. -func (s *SystemInfo) MemUsed() (uint64, error) { +func (s *systemInfo) MemUsed() (uint64, error) { memInfo, err := mem.VirtualMemory() if err != nil { return 0, err @@ -105,11 +105,11 @@ func (s *SystemInfo) MemUsed() (uint64, error) { } // KernelVersion returns the kernel version as a string. -func (s *SystemInfo) KernelVersion() (string, error) { +func (s *systemInfo) KernelVersion() (string, error) { return host.KernelVersion() } // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. -func NewSystemInfo() *SystemInfo { - return &SystemInfo{} +func NewSystemInfo() *systemInfo { + return &systemInfo{} } diff --git a/gossip/gossip.go b/gossip/gossip.go index 2da6e3e4c..7650cff86 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -15,69 +15,46 @@ package gossip import ( + "bytes" + "context" "fmt" "io/ioutil" "log" + "net" "strconv" "strings" "sync" "time" - "golang.org/x/sync/errgroup" - - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) // Ensure GossipMemberSet implements interfaces. -var _ pilosa.BroadcastReceiver = &GossipMemberSet{} -var _ pilosa.Gossiper = &GossipMemberSet{} -var _ memberlist.Delegate = &GossipMemberSet{} +var _ memberlist.Delegate = &gossipMemberSet{} -// GossipMemberSet represents a gossip implementation of MemberSet using memberlist. -type GossipMemberSet struct { +// gossipMemberSet represents a gossip implementation of MemberSet using memberlist. +type gossipMemberSet struct { mu sync.RWMutex - node *pilosa.Node memberlist *memberlist.Memberlist - handler pilosa.BroadcastHandler broadcasts *memberlist.TransmitLimitedQueue - statusHandler pilosa.StatusHandler - config *gossipConfig + papi *pilosa.API + config *gossipConfig Logger pilosa.Logger logger *log.Logger transport *Transport -} -// Start implements the BroadcastReceiver interface and sets the BroadcastHandler. -func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { - g.handler = h - return nil -} - -// GetBindAddr returns the gossip bind address based on config and auto bind port. -// This method is currently only used in a test scenario where a second node needs -// the auto-bind address of the first node to use as its gossip seed. -func (g *GossipMemberSet) GetBindAddr() string { - return fmt.Sprintf("%s:%d", g.config.memberlistConfig.BindAddr, g.config.memberlistConfig.BindPort) + gossipEventReceiver *gossipEventReceiver } // Open implements the MemberSet interface to start network activity. -func (g *GossipMemberSet) Open(n *pilosa.Node) error { - if g.handler == nil { - return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") - } - - g.node = n - - err := error(nil) +func (g *gossipMemberSet) Open() (err error) { g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() @@ -117,7 +94,7 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error { } // joinWithRetry wraps the standard memberlist Join function in a retry. -func (g *GossipMemberSet) joinWithRetry(hosts []string) error { +func (g *gossipMemberSet) joinWithRetry(hosts []string) error { err := retry(60, 2*time.Second, func() error { _, err := g.memberlist.Join(hosts) return err @@ -148,28 +125,30 @@ type gossipConfig struct { memberlistConfig *memberlist.Config } -// GossipMemberSetOption describes a functional option for GossipMemberSet. -type GossipMemberSetOption func(*GossipMemberSet) error +// gossipMemberSetOption describes a functional option for GossipMemberSet. +type gossipMemberSetOption func(*gossipMemberSet) error // WithTransport is a functional option for providing a transport to NewGossipMemberSet. -func WithTransport(transport *Transport) GossipMemberSetOption { - return func(g *GossipMemberSet) error { +func WithTransport(transport *Transport) gossipMemberSetOption { + return func(g *gossipMemberSet) error { g.transport = transport return nil } } // WithLogger is a functional option for providing a logger to NewGossipMemberSet. -func WithLogger(logger *log.Logger) GossipMemberSetOption { - return func(g *GossipMemberSet) error { +func WithLogger(logger *log.Logger) gossipMemberSetOption { + return func(g *gossipMemberSet) error { g.logger = logger return nil } } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - g := &GossipMemberSet{ +func NewGossipMemberSet(cfg Config, api *pilosa.API, options ...gossipMemberSetOption) (*gossipMemberSet, error) { + host := api.Node().URI.Host + g := &gossipMemberSet{ + papi: api, Logger: pilosa.NopLogger, } @@ -179,6 +158,8 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe return nil, errors.Wrap(err, "executing option") } } + ger := newGossipEventReceiver(g.logger, api) + g.gossipEventReceiver = ger if g.transport == nil { port, err := strconv.Atoi(cfg.Port) @@ -195,7 +176,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe g.transport = transport } - port := g.transport.Net.GetAutoBindPort() + port := g.transport.net.GetAutoBindPort() var gossipKey []byte var err error @@ -208,12 +189,12 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe // memberlist config conf := memberlist.DefaultWANConfig() - conf.Transport = g.transport.Net - conf.Name = name - conf.BindAddr = host + conf.Transport = g.transport.net + conf.Name = api.Node().ID + conf.BindAddr = api.Node().URI.Host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = pilosa.HostToIP(host) + conf.AdvertiseAddr = hostToIP(api.Node().URI.Host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -234,57 +215,12 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe gossipSeeds: cfg.Seeds, } - g.statusHandler = sh - return g, nil } -// SendSync implementation of the Broadcaster interface. -func (g *GossipMemberSet) SendSync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - mlist := g.memberlist - - // Direct sends the message directly to every node. - // An error from any node raises an error on the entire operation. - // - // Gossip uses the gossip protocol to eventually deliver the message - // to every node. - var eg errgroup.Group - for _, n := range mlist.Members() { - // Don't send the message to the local node. - if n == mlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return mlist.SendToTCP(node, msg) - }) - } - return eg.Wait() -} - -// SendAsync implementation of the Gossiper interface. -func (g *GossipMemberSet) SendAsync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) - return nil -} - // NodeMeta implementation of the memberlist.Delegate interface. -func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) +func (g *gossipMemberSet) NodeMeta(limit int) []byte { + buf, err := g.papi.Serializer.Marshal(g.papi.Node()) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} @@ -294,35 +230,30 @@ func (g *GossipMemberSet) NodeMeta(limit int) []byte { // NotifyMsg implementation of the memberlist.Delegate interface // called when a user-data message is received. -func (g *GossipMemberSet) NotifyMsg(b []byte) { - m, err := pilosa.UnmarshalMessage(b) +func (g *gossipMemberSet) NotifyMsg(b []byte) { + err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b)) if err != nil { - g.Logger.Printf("unmarshal message error: %s", err) - return - } - if err := g.handler.ReceiveMessage(m); err != nil { - g.Logger.Printf("receive message error: %s", err) - return + g.Logger.Printf("cluster message error: %s", err) } } // GetBroadcasts implementation of the memberlist.Delegate interface // called when user data messages can be broadcast. -func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { +func (g *gossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. -func (g *GossipMemberSet) LocalState(join bool) []byte { - pb, err := g.statusHandler.LocalStatus() - if err != nil { - g.Logger.Printf("error getting local state, err=%s", err) - return []byte{} +func (g *gossipMemberSet) LocalState(join bool) []byte { + m := &pilosa.NodeStatus{ + Node: g.papi.Node(), + MaxShards: g.papi.MaxShards(context.Background()), + Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } // Marshal nodestate data to bytes. - buf, err := proto.Marshal(pb) + buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer) if err != nil { g.Logger.Printf("error marshalling nodestate data, err=%s", err) return []byte{} @@ -332,59 +263,49 @@ func (g *GossipMemberSet) LocalState(join bool) []byte { // MergeRemoteState implementation of the memberlist.Delegate interface // receive and process the remote side's LocalState. -func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { - // Unmarshal nodestate data. - var pb internal.NodeStatus - if err := proto.Unmarshal(buf, &pb); err != nil { - g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) - return - } - err := g.statusHandler.HandleRemoteStatus(&pb) +func (g *gossipMemberSet) MergeRemoteState(buf []byte, join bool) { + err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)) if err != nil { g.Logger.Printf("merge state error: %s", err) } } -// GossipEventReceiver is used to enable an application to receive +// gossipEventReceiver is used to enable an application to receive // events about joins and leaves over a channel. // // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. -type GossipEventReceiver struct { - ch chan memberlist.NodeEvent - eventHandler pilosa.EventHandler +type gossipEventReceiver struct { + ch chan memberlist.NodeEvent + papi *pilosa.API - Logger pilosa.Logger + logger *log.Logger } -// NewGossipEventReceiver returns a new instance of GossipEventReceiver. -func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver { - return &GossipEventReceiver{ +// newGossipEventReceiver returns a new instance of GossipEventReceiver. +func newGossipEventReceiver(logger *log.Logger, papi *pilosa.API) *gossipEventReceiver { + ger := &gossipEventReceiver{ ch: make(chan memberlist.NodeEvent, 1), - Logger: logger, + logger: logger, + papi: papi, } + go ger.listen() + return ger } -func (g *GossipEventReceiver) NotifyJoin(n *memberlist.Node) { +func (g *gossipEventReceiver) NotifyJoin(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeJoin, n} } -func (g *GossipEventReceiver) NotifyLeave(n *memberlist.Node) { +func (g *gossipEventReceiver) NotifyLeave(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeLeave, n} } -func (g *GossipEventReceiver) NotifyUpdate(n *memberlist.Node) { +func (g *gossipEventReceiver) NotifyUpdate(n *memberlist.Node) { g.ch <- memberlist.NodeEvent{memberlist.NodeUpdate, n} } -// Start implements the pilosa.EventReceiver interface and sets the EventHandler. -func (g *GossipEventReceiver) Start(h pilosa.EventHandler) error { - g.eventHandler = h - go g.listen() - return nil -} - -func (g *GossipEventReceiver) listen() { +func (g *gossipEventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { e := <-g.ch @@ -400,46 +321,29 @@ func (g *GossipEventReceiver) listen() { } // Get the node from the event.Node meta data. - var n internal.Node - if err := proto.Unmarshal(e.Node.Meta, &n); err != nil { - panic("failed to unmarshal event node meta data") + var n pilosa.Node + if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil { + panic("failed to unmarshal event node meta into node") } - node := pilosa.DecodeNode(&n) ne := &pilosa.NodeEvent{ Event: nodeEventType, - Node: node, + Node: &n, } - if err := g.eventHandler.ReceiveEvent(ne); err != nil { - g.Logger.Printf("receive event error: %s", err) + buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer) + if err != nil { + panic(err) + } + if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil { + g.logger.Printf("receive event error: %s", err) } - } -} - -// broadcast represents an implementation of memberlist.Broadcast -type broadcast struct { - msg []byte - notify chan<- struct{} -} - -func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { - return false -} - -func (b *broadcast) Message() []byte { - return b.msg -} - -func (b *broadcast) Finished() { - if b.notify != nil { - close(b.notify) } } // Transport is a gossip transport for binding to a port. type Transport struct { //memberlist.Transport - Net *memberlist.NetTransport + net *memberlist.NetTransport URI *pilosa.URI } @@ -466,7 +370,7 @@ func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) } return &Transport{ - Net: net, + net: net, URI: uri, }, nil } @@ -580,3 +484,21 @@ type Config struct { Nodes int `toml:"nodes"` ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` } + +// hostToIP converts host to an IP4 address based on net.LookupIP(). +func hostToIP(host string) string { + // if host is not an IP addr, check net.LookupIP() + if net.ParseIP(host) == nil { + hosts, err := net.LookupIP(host) + if err != nil { + return host + } + for _, h := range hosts { + // this restricts pilosa to IP4 + if h.To4() != nil { + return h.String() + } + } + } + return host +} diff --git a/handler.go b/handler.go index 9ddabaeeb..9fc3af368 100644 --- a/handler.go +++ b/handler.go @@ -1,1058 +1,7 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( "encoding/json" - "expvar" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. - _ "net/http/pprof" - "reflect" - "runtime/debug" - "strconv" - "strings" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/internal" - - "github.com/pkg/errors" -) - -// Handler represents an HTTP handler. -type Handler struct { - Handler http.Handler - - FileSystem FileSystem - - Logger Logger - - // Keeps the query argument validators for each handler - validators map[string]*queryValidationSpec - - API *API - - AllowedOrigins []string -} - -// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. -// This is used for stats tagging. -var externalPrefixFlag = map[string]bool{ - "schema": true, - "query": true, - "import": true, - "export": true, - "index": true, - "frame": true, - "nodes": true, - "version": true, -} - -type errorResponse struct { - Error string `json:"error"` -} - -// HandlerOption is a functional option type for pilosa.Handler -type HandlerOption func(s *Handler) error - -func OptHandlerAllowedOrigins(origins []string) HandlerOption { - return func(h *Handler) error { - h.Handler = handlers.CORS( - handlers.AllowedOrigins(origins), - handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) - return nil - } -} - -// NewHandler returns a new instance of Handler with a default logger. -func NewHandler(opts ...HandlerOption) (*Handler, error) { - handler := &Handler{ - FileSystem: NopFileSystem, - Logger: NopLogger, - } - handler.Handler = NewRouter(handler) - handler.populateValidators() - - for _, opt := range opts { - err := opt(handler) - if err != nil { - return nil, errors.Wrap(err, "applying option") - } - } - - return handler, nil -} - -func (h *Handler) populateValidators() { - h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice") -} - -func (h *Handler) queryArgValidator(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - key := mux.CurrentRoute(r).GetName() - if validator, ok := h.validators[key]; ok { - if err := validator.validate(r.URL.Query()); err != nil { - // TODO: Return the response depending on the Accept header - response := errorResponse{Error: err.Error()} - body, err := json.Marshal(response) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - http.Error(w, string(body), http.StatusBadRequest) - return - } - } - next.ServeHTTP(w, r) - }) -} - -// NewRouter creates a new mux http router. -func NewRouter(handler *Handler) *mux.Router { - router := mux.NewRouter() - router.HandleFunc("/", handler.handleWebUI).Methods("GET") - router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") - router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - - // TODO: Apply MethodNotAllowed statuses to all endpoints. - // Ideally this would be automatic, as described in this (wontfix) ticket: - // https://github.com/gorilla/mux/issues/6 - // For now we just do it for the most commonly used handler, /query - router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - - router.Use(handler.queryArgValidator) - return router -} - -func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) -} - -// ServeHTTP handles an HTTP request. -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - w.WriteHeader(http.StatusInternalServerError) - stack := debug.Stack() - msg := "PANIC: %s\n%s" - h.Logger.Printf(msg, err, stack) - fmt.Fprintf(w, msg, err, stack) - } - }() - - t := time.Now() - h.Handler.ServeHTTP(w, r) - dif := time.Since(t) - - // Calculate per request StatsD metrics when the handler is fully configured. - statsTags := make([]string, 0, 3) - - longQueryTime := h.API.LongQueryTime() - if longQueryTime > 0 && dif > longQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - if stats != nil { - stats.Histogram("http."+endpointName, float64(dif), 0.1) - } -} - -func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { - // If user is using curl, don't chuck HTML at them - if strings.HasPrefix(r.UserAgent(), "curl") { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) - return - } - filesystem, err := h.FileSystem.New() - if err != nil { - _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") - return - } - http.FileServer(filesystem).ServeHTTP(w, r) -} - -// handleGetSchema handles GET /schema requests. -func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { - schema := h.API.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: schema, - }); err != nil { - h.Logger.Printf("write schema response error: %s", err) - } -} - -// handleGetStatus handles GET /status requests. -func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status := getStatusResponse{ - State: h.API.State(), - Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.LocalID(), - } - if err := json.NewEncoder(w).Encode(status); err != nil { - h.Logger.Printf("write status response error: %s", err) - } -} - -func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { - info := h.API.Info() - if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write info response error: %s", err) - } -} - -type getSchemaResponse struct { - Indexes []*IndexInfo `json:"indexes"` -} - -type getStatusResponse struct { - State string `json:"state"` - Nodes []*Node `json:"nodes"` - LocalID string `json:"localID"` -} - -// handlePostQuery handles /query requests. -func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - // Parse incoming request. - req, err := h.readQueryRequest(r) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - // TODO: Remove - req.Index = mux.Vars(r)["index"] - - resp, err := h.API.Query(r.Context(), req) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Set appropriate status code, if there is an error. - if resp.Err != nil { - switch resp.Err { - case ErrTooManyWrites: - w.WriteHeader(http.StatusRequestEntityTooLarge) - default: - w.WriteHeader(http.StatusInternalServerError) - } - } - - // Write response back to client. - if err := h.writeQueryResponse(w, r, &resp); err != nil { - h.Logger.Printf("write query response error: %s", err) - } -} - -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) - } -} - -type getSlicesMaxResponse struct { - Standard map[string]uint64 `json:"standard"` -} - -// handleGetIndexes handles GET /index request. -func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { - h.handleGetSchema(w, r) -} - -// handleGetIndex handles GET /index/ requests. -func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - index, err := h.API.Index(r.Context(), indexName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - if err := json.NewEncoder(w).Encode(getIndexResponse{ - map[string]string{"name": index.Name()}, - }); err != nil { - h.Logger.Printf("write response error: %s", err) - } -} - -type getIndexResponse struct { - Index map[string]string `json:"index"` -} - -type postIndexRequest struct { - Options IndexOptions `json:"options"` -} - -//_postIndexRequest is necessary to avoid recursion while decoding. -type _postIndexRequest postIndexRequest - -// Custom Unmarshal JSON to validate request body when creating a new index. -func (p *postIndexRequest) UnmarshalJSON(b []byte) error { - - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshalling unexpected values") - } - - validIndexOptions := getValidOptions(IndexOptions{}) - err := validateOptions(m, validIndexOptions) - if err != nil { - return err - } - // Unmarshal expected values. - var _p _postIndexRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected values") - } - - p.Options = _p.Options - - return nil -} - -// Raise errors for any unknown key -func validateOptions(data map[string]interface{}, validIndexOptions []string) error { - for k, v := range data { - switch k { - case "options": - options, ok := v.(map[string]interface{}) - if !ok { - return errors.New("options is not map[string]interface{}") - } - for kk, vv := range options { - if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) - } - } - default: - return fmt.Errorf("Unknown key: %v:%v", k, v) - } - } - return nil -} - -func foundItem(items []string, item string) bool { - for _, i := range items { - if item == i { - return true - } - } - return false -} - -type postIndexResponse struct{} - -// handleDeleteIndex handles DELETE /index request. -func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - err := h.API.DeleteIndex(r.Context(), indexName) - if err != nil { - h.Logger.Printf("problem deleting index: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteIndexResponse struct{} - -// handlePostIndex handles POST /index request. -func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the index - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) - if errors.Cause(err) == ErrIndexExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -// handlePostIndexAttrDiff handles POST /index/attr/diff requests. -func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postIndexAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postIndexAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// handlePostFrame handles POST /frame request. -func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req postFrameRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the frame - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrFrameExists: - http.Error(w, err.Error(), http.StatusConflict) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type _postFrameRequest postFrameRequest - -// Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions, -// adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed -func (p *postFrameRequest) UnmarshalJSON(b []byte) error { - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshaling unexpected keys") - } - - validFrameOptions := getValidOptions(FrameOptions{}) - err := validateOptions(m, validFrameOptions) - if err != nil { - return err - } - - // Unmarshal expected values. - var _p _postFrameRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected keys") - } - - p.Options = _p.Options - return nil - -} - -func getValidOptions(option interface{}) []string { - validOptions := []string{} - val := reflect.ValueOf(option) - for i := 0; i < val.Type().NumField(); i++ { - jsonTag := val.Type().Field(i).Tag.Get("json") - s := strings.Split(jsonTag, ",") - validOptions = append(validOptions, s[0]) - } - return validOptions -} - -type postFrameRequest struct { - Options FrameOptions `json:"options"` -} - -type postFrameResponse struct{} - -// handleDeleteFrame handles DELETE /frame request. -func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - err := h.API.DeleteFrame(r.Context(), indexName, frameName) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } - return - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteFrameResponse struct{} - -// handlePostFrameField handles POST /frame/field request. -func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFrameFieldRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - field := &Field{ - Name: fieldName, - Type: req.Type, - Min: req.Min, - Max: req.Max, - } - - if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFrameFieldRequest struct { - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -type postFrameFieldResponse struct{} - -// handleDeleteFrameField handles DELETE /frame/field request. -func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - fields, err := h.API.Fields(r.Context(), indexName, frameName) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type getFrameFieldsResponse struct { - Fields []*Field `json:"fields,omitempty"` -} - -type deleteFrameFieldResponse struct{} - -// handlePostFrameAttrDiff handles POST /frame/attr/diff requests. -func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req postFrameAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) - if err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFrameAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postFrameAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { - switch r.Header.Get("Content-Type") { - case "application/x-protobuf": - return h.readProtobufQueryRequest(r) - default: - return h.readURLQueryRequest(r) - } -} - -// readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { - // Slurp the body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - - return decodeQueryRequest(&req), nil -} - -// readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { - q := r.URL.Query() - - // Parse query string. - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - query := string(buf) - - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) - if err != nil { - return nil, errors.New("invalid slice argument") - } - - return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", - ExcludeColumns: q.Get("excludeColumns") == "true", - }, nil -} - -// writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - return h.writeProtobufQueryResponse(w, resp) - } - return h.writeJSONQueryResponse(w, resp) -} - -// writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { - return errors.Wrap(err, "marshalling") - } else if _, err := w.Write(buf); err != nil { - return errors.Wrap(err, "writing") - } - return nil -} - -// writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - return json.NewEncoder(w).Encode(resp) -} - -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.Import(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handlePostImportValue handles /import-value requests. -func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ImportValue(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handleGetExport handles /export requests. -func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { - switch r.Header.Get("Accept") { - case "text/csv": - h.handleGetExportCSV(w, r) - default: - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - } -} - -func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { - // Parse query parameters. - q := r.URL.Query() - index, frame := q.Get("index"), q.Get("frame") - - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) - return - } - - if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - break - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } -} - -// handleGetFragmentNodes handles /fragment/nodes requests. -func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - index := q.Get("index") - - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) - return - } - - // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Write to response. - if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.Logger.Printf("json write error: %s", err) - } -} - -// handleGetFragmentBlockData handles GET /fragment/block/data requests. -func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - buf, err := h.API.FragmentBlockData(r.Context(), r.Body) - if err != nil { - if _, ok := err.(BadRequestError); ok { - http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Write response. - w.Header().Set("Content-Type", "application/protobuf") - w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) -} - -// handleGetFragmentBlocks handles GET /fragment/blocks requests. -func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice) - if err != nil { - if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ - Blocks: blocks, - }); err != nil { - h.Logger.Printf("block response encoding error: %s", err) - } -} - -type getFragmentBlocksResponse struct { - Blocks []FragmentBlock `json:"blocks"` -} - -// handleGetVersion handles /version requests. -func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - err := json.NewEncoder(w).Encode(struct { - Version string `json:"version"` - }{ - Version: h.API.Version(), - }) - if err != nil { - h.Logger.Printf("write version response error: %s", err) - } -} - -// QueryResult types. -const ( - QueryResultTypeNil uint32 = iota - QueryResultTypeRow - QueryResultTypePairs - QueryResultTypeValCount - QueryResultTypeUint64 - QueryResultTypeBool ) // QueryRequest represent a request to process a query. @@ -1063,9 +12,9 @@ type QueryRequest struct { // The query string to parse and execute. Query string - // The slices to include in the query execution. - // If empty, all slices are included. - Slices []uint64 + // The shards to include in the query execution. + // If empty, all shards are included. + Shards []uint64 // Return column attributes, if true. ColumnAttrs bool @@ -1081,19 +30,6 @@ type QueryRequest struct { Remote bool } -func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - // QueryResponse represent a response from a processed query. type QueryResponse struct { // Result for each top-level query call. @@ -1123,234 +59,56 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { return json.Marshal(output) } -func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = encodeRow(result) - case []Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = encodePairs(result) - case ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = encodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb +type Handler interface { + Serve() error + Close() error } -// parseUint64Slice returns a slice of uint64s from a comma-delimited string. -func parseUint64Slice(s string) ([]uint64, error) { - var a []uint64 - for _, str := range strings.Split(s, ",") { - // Ignore blanks. - if str == "" { - continue - } +type nopHandler struct{} - // Parse number. - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return nil, errors.Wrap(err, "parsing int") - } - a = append(a, num) - } - return a, nil -} - -// errorString returns the string representation of err. -func errorString(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req setCoordinatorRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) - return - } - - oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: oldNode, - New: newNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type setCoordinatorRequest struct { - ID string `json:"id"` -} - -type setCoordinatorResponse struct { - Old *Node `json:"old"` - New *Node `json:"new"` -} - -// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. -func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req removeNodeRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - removeNode, err := h.API.RemoveNode(req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(removeNodeResponse{ - Remove: removeNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type removeNodeRequest struct { - ID string `json:"id"` -} - -type removeNodeResponse struct { - Remove *Node `json:"remove"` -} - -// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. -func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { - err := h.API.ResizeAbort() - var msg string - if err != nil { - switch errors.Cause(err) { - case ErrNodeNotCoordinator: - http.Error(w, err.Error(), http.StatusBadRequest) - return - case ErrResizeNotRunning: - msg = err.Error() - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - // Encode response. - if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ - Info: msg, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type clusterResizeAbortResponse struct { - Info string `json:"info"` -} - -func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.API.RecalculateCaches(r.Context()) - if err != nil { - http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} - -func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - err := h.API.ClusterMessage(r.Context(), r.Body) - if err != nil { - // TODO this was the previous behavior, but perhaps not everything is a bad request - http.Error(w, err.Error(), http.StatusBadRequest) - } - - if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type defaultClusterMessageResponse struct{} - -type queryValidationSpec struct { - required []string - args map[string]struct{} -} - -func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { - args := map[string]struct{}{} - for _, arg := range requiredArgs { - args[arg] = struct{}{} - } - - return &queryValidationSpec{ - required: requiredArgs, - args: args, - } -} - -func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { - for _, arg := range args { - s.args[arg] = struct{}{} - } - return s -} - -func (s queryValidationSpec) validate(query url.Values) error { - for _, req := range s.required { - if query.Get(req) == "" { - return errors.Errorf("%s is required", req) - } - } - for k := range query { - if _, ok := s.args[k]; !ok { - return errors.Errorf("%s is not a valid argument", k) - } - } +func (n nopHandler) Serve() error { return nil } + +func (n nopHandler) Close() error { + return nil +} + +var NopHandler Handler = nopHandler{} + +type ImportValueRequest struct { + Index string + Field string + Shard uint64 + ColumnIDs []uint64 + ColumnKeys []string + Values []int64 +} + +type ImportRequest struct { + Index string + Field string + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +type ImportResponse struct { + Err string +} + +type BlockDataRequest struct { + Index string + Field string + View string + Shard uint64 + Block uint64 +} + +type BlockDataResponse struct { + RowIDs []uint64 + ColumnIDs []uint64 +} diff --git a/handler_internal_test.go b/handler_internal_test.go deleted file mode 100644 index ceaeb707c..000000000 --- a/handler_internal_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "encoding/json" - "reflect" - "testing" -) - -// Test custom UnmarshalJSON for postIndexRequest object -func TestPostIndexRequestUnmarshalJSON(t *testing.T) { - tests := []struct { - json string - expected postIndexRequest - err string - }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, - {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, - {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, - } - for _, test := range tests { - actual := &postIndexRequest{} - err := json.Unmarshal([]byte(test.json), actual) - - if err != nil { - if test.err == "" || test.err != err.Error() { - t.Errorf("expected error: %v, but got result: %v", test.err, err) - } - } else { - if test.err != "" { - t.Errorf("expected error: %v, but got no error", test.err) - } - } - - if test.err == "" { - if !reflect.DeepEqual(*actual, test.expected) { - t.Errorf("expected: %v, but got: %v", test.expected, *actual) - } - } - - } -} - -// Test custom UnmarshalJSON for postFrameRequest object -func TestPostFrameRequestUnmarshalJSON(t *testing.T) { - tests := []struct { - json string - expected postFrameRequest - err string - }{ - {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, - {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, - {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, - {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}}, - {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, - } - for _, test := range tests { - actual := &postFrameRequest{} - err := json.Unmarshal([]byte(test.json), actual) - if err != nil { - if test.err == "" || test.err != err.Error() { - t.Errorf("expected error: %v, but got result: %v", test.err, err) - } - } - - if test.err == "" { - if !reflect.DeepEqual(*actual, test.expected) { - t.Errorf("expected: %v, but got: %v", test.expected, *actual) - } - } - - } -} diff --git a/handler_test.go b/handler_test.go deleted file mode 100644 index a7ba8fa85..000000000 --- a/handler_test.go +++ /dev/null @@ -1,1222 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "reflect" - "strings" - "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/statik" - "github.com/pilosa/pilosa/test" -) - -func TestHandlerPanics(t *testing.T) { - h := test.MustNewHandler() - bufLogger := test.NewBufferLogger() - h.Handler.Logger = bufLogger - - w := httptest.NewRecorder() - // will panic since Handler has no Holder set up - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil)) - bufbytes, err := bufLogger.ReadAll() - if err != nil { - t.Fatalf("reading all logoutput: %v", err) - } - if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { - t.Fatalf("expected panic in log, but got: %s", bufbytes) - } - if w.Code != http.StatusInternalServerError { - t.Fatalf("expected internal server error, but got: %v", w.Code) - } - bodyBytes := w.Body.Bytes() - if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { - t.Fatalf("response to client should have panic, but got %s", bodyBytes) - } -} - -// Ensure the handler returns "not found" for invalid paths. -func TestHandler_NotFound(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != http.StatusNotFound { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler can return the schema. -func TestHandler_Schema(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return the status. -func TestHandler_Status(t *testing.T) { - s := test.NewServer() - hldr := test.MustOpenHolder() - defer s.Close() - defer hldr.Close() - - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { - t.Fatal(err) - } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { - t.Fatal(err) - } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateNormal) - h.API.StatusHandler = s - s.Handler = h - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -func TestHandler_Info(t *testing.T) { - s := test.NewServer() - defer s.Close() - h := test.MustNewHandler() - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can abort a cluster resize. -func TestHandler_ClusterResizeAbort(t *testing.T) { - - t.Run("No resize job", func(t *testing.T) { - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Cluster.SetState(pilosa.ClusterStateResizing) - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) - if w.Code != http.StatusOK { - bod, err := ioutil.ReadAll(w.Body) - t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) - } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } - }) - -} - -// Ensure the handler can return the maxslice map. -func TestHandler_MaxSlices(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can accept URL arguments. -func TestHandler_Query_Args_URL(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can accept arguments via protobufs. -func TestHandler_Query_Args_Protobuf(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "idx0" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Count(Bitmap(id=100))` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { - t.Fatalf("unexpected slices: %+v", slices) - } - return []interface{}{uint64(100)}, nil - } - - // Generate request body. - reqBody, err := proto.Marshal(&internal.QueryRequest{ - Query: "Count(Bitmap(id=100))", - Slices: []uint64{0, 1}, - }) - if err != nil { - t.Fatal(err) - } - - // Generate protobuf request. - req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) - req.Header.Set("Content-Type", "application/x-protobuf") - - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -// Ensure the handler returns an error when parsing bad arguments. -func TestHandler_Query_Args_Err(t *testing.T) { - w := httptest.NewRecorder() - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} -func TestHandler_Query_Params_Err(t *testing.T) { - w := httptest.NewRecorder() - test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } - -} - -// Ensure the handler can execute a query with a uint64 response as JSON. -func TestHandler_Query_Uint64_JSON(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can execute a query with a uint64 response as protobufs. -func TestHandler_Query_Uint64_Protobuf(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(100)}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if n := resp.Results[0].N; n != 100 { - t.Fatalf("unexpected n: %d", n) - } -} - -// Ensure the handler can execute a query that returns a bitmap as JSON. -func TestHandler_Query_Bitmap_JSON(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can execute a query that returns a row with column attributes as JSON. -func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can execute a query that returns a row as protobuf. -func TestHandler_Query_Row_Protobuf(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } -} - -// Ensure the handler can execute a query that returns a row with column attributes as protobuf. -func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { - hldr := test.NewHolder() - defer hldr.Close() - - // Create index and set column attributes. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - r := pilosa.NewRow(1, pilosa.SliceWidth+1) - r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} - return []interface{}{r}, nil - } - - // Encode request body. - buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", - ColumnAttrs: true, - }) - if err != nil { - t.Fatal(err) - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) - r.Header.Set("Content-Type", "application/x-protobuf") - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { - t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { - t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v { - t.Fatalf("unexpected attr[2]: %s=%v", k, v) - } - - if a := resp.ColumnAttrSets; len(a) != 1 { - t.Fatalf("unexpected column attributes length: %d", len(a)) - } else if a[0].ID != 1 { - t.Fatalf("unexpected id: %d", a[0].ID) - } else if len(a[0].Attrs) != 1 { - t.Fatalf("unexpected column attr length: %d", len(a)) - } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { - t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } -} - -// Ensure the handler can execute a query that returns pairs as JSON. -func TestHandler_Query_Pairs_JSON(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can execute a query that returns pairs as protobuf. -func TestHandler_Query_Pairs_Protobuf(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 2}, - {ID: 3, Count: 4}, - }}, nil - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { - t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if a := resp.Results[0].GetPairs(); len(a) != 2 { - t.Fatalf("unexpected pair length: %d", len(a)) - } -} - -// Ensure the handler can return an error as JSON. -func TestHandler_Query_Err_JSON(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { - t.Fatalf("unexpected body: %q", body) - } -} - -// Ensure the handler can return an error as protobuf. -func TestHandler_Query_Err_Protobuf(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return nil, errors.New("marker") - } - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) - r.Header.Set("Accept", "application/x-protobuf") - h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var resp internal.QueryResponse - if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatal(err) - } else if s := resp.Err; s != `executing: marker` { - t.Fatalf("unexpected error: %s", s) - } -} - -// Ensure the handler returns "method not allowed" for non-POST queries. -func TestHandler_Query_MethodNotAllowed(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != http.StatusMethodNotAllowed { - t.Fatalf("invalid status: %d", w.Code) - } -} - -// Ensure the handler returns an error if there is a parsing error.. -func TestHandler_Query_ErrParse(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can delete an index. -func TestHandler_Index_Delete(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - - // Send request to delete index. - resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Verify body response. - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %d", resp.StatusCode) - } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if string(buf) != "{}\n" { - t.Fatalf("unexpected response body: %s", buf) - } - - // Verify index is gone. - if hldr.Index("i") != nil { - t.Fatal("expected nil index") - } -} - -// Ensure handler can delete a frame. -func TestHandler_DeleteFrame(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i0").Frame("f1"); f != nil { - t.Fatal("expected nil frame") - } -} - -// Ensure the handler can return data in differing blocks for an index. -func TestHandler_Index_AttrStore_Diff(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set attributes on the index. - index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { - t.Fatal(err) - } - - // Retrieve block checksums. - blks, err := index.ColumnAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } - - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - - // Send block checksums to determine diff. - resp, err := http.Post( - s.URL+"/index/i/attr/diff", - "application/json", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return data in differing blocks for a frame. -func TestHandler_Frame_AttrStore_Diff(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set attributes on the index. - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } - if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { - t.Fatal(err) - } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { - t.Fatal(err) - } - - // Retrieve block checksums. - blks, err := f.RowAttrStore().Blocks() - if err != nil { - t.Fatal(err) - } - - // Remove block #0 and alter block 2's checksum. - blks = blks[1:] - blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") - - // Send block checksums to determine diff. - resp, err := http.Post( - s.URL+"/index/i/frame/meta/attr/diff", - "application/json", - strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), - ) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Read and validate body. - if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can create a new field on an existing frame. -func TestHandler_Frame_AddField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); !reflect.DeepEqual(field, &pilosa.Field{Name: "x", Type: "int", Min: 100, Max: 200}) { - t.Fatalf("unexpected field: %#v", field) - } - }) - - t.Run("ErrInvalidFieldType", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"bad_type","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field type`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":200,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field range`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrFieldAlreadyExists", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, - }); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":0,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: field already exists`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -// Ensure the handler can delete existing fields. -func TestHandler_Frame_DeleteField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/x", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); field != nil { - t.Fatalf("expected nil field, got: %#v", field) - } - }) - - t.Run("ErrFieldNotFound", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/y", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `deleting field: field not found` { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -func TestHandler_Frame_GetFields(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { - t.Fatal(err) - } - resp, err := http.Get(s.URL + "/index/i/frame/f/fields") - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - var fields FrameFields - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - if err = json.Unmarshal([]byte(body), &fields); err != nil { - t.Fatal(err) - } - field := fields.Fields[0] - if field.Name != "x" { - t.Fatalf("expected field's name: x, actuall name: %v", field.Name) - } else if field.Min != 1 { - t.Fatalf("expected field's min: x, actuall min: %v", field.Min) - } else if field.Max != 100 { - t.Fatalf("expected field's max: x, actuall max: %v", field.Max) - } - - }) - - t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) - if err != nil { - t.Fatalf("creating frame: %v", err) - } - - resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") - if err != nil { - t.Fatal(err) - } - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { - t.Fatalf("shouldn't get frame fields not allowed error: %q", body) - } - }) - -} - -type FrameFields struct { - Fields []pilosa.Field -} - -// Ensure the handler can retrieve the version. -func TestHandler_Version(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/version", nil) - h.ServeHTTP(w, r) - version := pilosa.Version - if strings.HasPrefix(version, "v") { - version = version[1:] - } - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) - } -} - -// Ensure the handler can return a list of nodes for a fragment. -func TestHandler_Fragment_Nodes(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(3) - h.API.Cluster.ReplicaN = 2 - - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { - t.Fatalf("unexpected body: %q", body) - } - - // invalid argument should return BadRequest - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } - - // index is required - w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -// Ensure the handler can return expvars without panicking. -func TestHandler_Expvars(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Cluster = test.NewCluster(1) - h.API.Holder = hldr.Holder - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } -} - -func MustReadAll(r io.Reader) []byte { - buf, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return buf -} - -func TestHandler_RecalculateCaches(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != http.StatusNoContent { - t.Fatalf("unexpected status code: %d", w.Code) - } - -} - -func TestHandler_WebUI(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.FileSystem = &statik.FileSystem{} - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - if !strings.Contains(w.Body.String(), "Pilosa WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } - - // If curl is the client, the response should be different - w = httptest.NewRecorder() - req := test.MustNewHTTPRequest("GET", "/", nil) - req.Header.Add("User-Agent", "curl/7.54.0") - h.ServeHTTP(w, req) - if !strings.Contains(w.Body.String(), "try the WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } -} - -func TestHandler_CORS(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // No CORS config present, so should fail - handler := test.MustNewHandler() - - req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) - req.Header.Add("Origin", "http://test/") - req.Header.Add("Access-Control-Request-Method", "POST") - - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - result := w.Result() - - // This handler does not support CORS, return Method Not Allowed (405) - if result.StatusCode != 405 { - t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) - } - - // CORS config should allow preflight response - handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"})) - w = httptest.NewRecorder() - handler.ServeHTTP(w, req) - result = w.Result() - - if result.StatusCode != 200 { - t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) - } - if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { - t.Fatal("CORS header not present") - } -} diff --git a/holder.go b/holder.go index 6d13159b9..a8caf8664 100644 --- a/holder.go +++ b/holder.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "os" "path" "path/filepath" @@ -28,17 +27,16 @@ import ( "syscall" "time" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) const ( - // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. - DefaultCacheFlushInterval = 1 * time.Minute + // defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. + defaultCacheFlushInterval = 1 * time.Minute - // FileLimit is the maximum open file limit (ulimit -n) to automatically set. - FileLimit = 262144 // (512^2) + // fileLimit is the maximum open file limit (ulimit -n) to automatically set. + fileLimit = 262144 // (512^2) ) // Holder represents a container for indexes. @@ -51,7 +49,7 @@ type Holder struct { // opened channel is closed once Open() completes. opened chan struct{} - Broadcaster Broadcaster + broadcaster broadcaster NewAttrStore func(string) AttrStore @@ -66,7 +64,7 @@ type Holder struct { Path string // The interval at which the cached row ids are persisted to disk. - CacheFlushInterval time.Duration + cacheFlushInterval time.Duration Logger Logger } @@ -79,12 +77,12 @@ func NewHolder() *Holder { opened: make(chan struct{}), - Broadcaster: NopBroadcaster, + broadcaster: NopBroadcaster, Stats: NopStatsClient, - NewAttrStore: NewNopAttrStore, + NewAttrStore: newNopAttrStore, - CacheFlushInterval: DefaultCacheFlushInterval, + cacheFlushInterval: defaultCacheFlushInterval, Logger: NopLogger, } @@ -112,7 +110,8 @@ func (h *Holder) Open() error { } for _, fi := range fis { - if !fi.IsDir() { + // Skip files or hidden directories. + if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { continue } @@ -200,37 +199,53 @@ func (h *Holder) HasData() (bool, error) { return false, nil } -// MaxSlices returns MaxSlice map for all indexes. -func (h *Holder) MaxSlices() map[string]uint64 { +// maxShards returns MaxShard map for all indexes. +func (h *Holder) maxShards() map[string]uint64 { a := make(map[string]uint64) for _, index := range h.Indexes() { - a[index.Name()] = index.MaxSlice() + a[index.Name()] = index.maxShard() } return a } -// Schema returns schema information for all indexes, frames, and views. +// Schema returns schema information for all indexes, fields, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} - for _, frame := range index.Frames() { - fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()} - for _, view := range frame.Views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + for _, view := range field.views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) + di.Fields = append(di.Fields, fi) } - sort.Sort(frameInfoSlice(di.Frames)) + sort.Sort(fieldInfoSlice(di.Fields)) a = append(a, di) } sort.Sort(indexInfoSlice(a)) return a } -// ApplySchema applies an internal Schema to Holder. -func (h *Holder) ApplySchema(schema *internal.Schema) error { +// limitedSchema returns schema information for all indexes and fields. +func (h *Holder) limitedSchema() []*IndexInfo { + var a []*IndexInfo + for _, index := range h.Indexes() { + di := &IndexInfo{Name: index.Name()} + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + di.Fields = append(di.Fields, fi) + } + sort.Sort(fieldInfoSlice(di.Fields)) + a = append(a, di) + } + sort.Sort(indexInfoSlice(a)) + return a +} + +// applySchema applies an internal Schema to Holder. +func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. for _, index := range schema.Indexes { opt := IndexOptions{} @@ -238,16 +253,15 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { if err != nil { return errors.Wrap(err, "creating index") } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) - frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + // Create fields that don't exist. + for _, f := range index.Fields { + field, err := idx.createFieldIfNotExists(f.Name, f.Options) if err != nil { - return errors.Wrap(err, "creating frame") + return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := frame.CreateViewIfNotExists(v) + _, err := field.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -257,20 +271,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { return nil } -// EncodeMaxSlices creates and internal representation of max slices. -func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { - return &internal.MaxSlices{ - Standard: h.MaxSlices(), - } -} - -// EncodeSchema creates an internal representation of schema. -func (h *Holder) EncodeSchema() *internal.Schema { - return &internal.Schema{ - Indexes: EncodeIndexes(h.Indexes()), - } -} - // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } @@ -304,7 +304,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { // Ensure index doesn't already exist. if h.indexes[name] != nil { - return nil, ErrIndexExists + return nil, newConflictError(ErrIndexExists) } return h.createIndex(name, opt) } @@ -339,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { return nil, errors.Wrap(err, "creating") } + index.keys = opt.Keys + if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") + } else if err := index.saveMeta(); err != nil { + return nil, errors.Wrap(err, "meta") } // Update options. - h.indexes[index.Name()] = index return index, nil @@ -355,11 +358,11 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.Logger = h.Logger + index.logger = h.Logger index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) - index.broadcaster = h.Broadcaster - index.NewAttrStore = h.NewAttrStore - index.columnAttrStore = h.NewAttrStore(filepath.Join(index.path, ".data")) + index.broadcaster = h.broadcaster + index.newAttrStore = h.NewAttrStore + index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data")) return index, nil } @@ -368,10 +371,10 @@ func (h *Holder) DeleteIndex(name string) error { h.mu.Lock() defer h.mu.Unlock() - // Ignore if index doesn't exist. + // Confirm index exists. index := h.index(name) if index == nil { - return nil + return newNotFoundError(ErrIndexNotFound) } // Close index. @@ -390,37 +393,37 @@ func (h *Holder) DeleteIndex(name string) error { return nil } -// Frame returns the frame for an index and name. -func (h *Holder) Frame(index, name string) *Frame { +// Field returns the field for an index and name. +func (h *Holder) Field(index, name string) *Field { idx := h.Index(index) if idx == nil { return nil } - return idx.Frame(name) + return idx.Field(name) } -// View returns the view for an index, frame, and name. -func (h *Holder) View(index, frame, name string) *View { - f := h.Frame(index, frame) +// view returns the view for an index, field, and name. +func (h *Holder) view(index, field, name string) *view { + f := h.Field(index, field) if f == nil { return nil } - return f.View(name) + return f.view(name) } -// Fragment returns the fragment for an index, frame & slice. -func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment { - v := h.View(index, frame, view) +// fragment returns the fragment for an index, field & shard. +func (h *Holder) fragment(index, field, view string, shard uint64) *fragment { + v := h.view(index, field, view) if v == nil { return nil } - return v.Fragment(slice) + return v.Fragment(shard) } // monitorCacheFlush periodically flushes all fragment caches sequentially. // This is run in a goroutine. func (h *Holder) monitorCacheFlush() { - ticker := time.NewTicker(h.CacheFlushInterval) + ticker := time.NewTicker(h.cacheFlushInterval) defer ticker.Stop() for { @@ -435,9 +438,9 @@ func (h *Holder) monitorCacheFlush() { func (h *Holder) flushCaches() { for _, index := range h.Indexes() { - for _, frame := range index.Frames() { - for _, view := range frame.Views() { - for _, fragment := range view.Fragments() { + for _, field := range index.Fields() { + for _, view := range field.views() { + for _, fragment := range view.allFragments() { select { case <-h.closing: return @@ -445,7 +448,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath()) } } } @@ -453,13 +456,13 @@ func (h *Holder) flushCaches() { } } -// RecalculateCaches recalculates caches on every index in the holder. This is +// recalculateCaches recalculates caches on every index in the holder. This is // probably not practical to call in real-world workloads, but makes writing // integration tests much eaiser, since one doesn't have to wait 10 seconds // after setting bits to get expected response. -func (h *Holder) RecalculateCaches() { +func (h *Holder) recalculateCaches() { for _, index := range h.Indexes() { - index.RecalculateCaches() + index.recalculateCaches() } } @@ -473,11 +476,11 @@ func (h *Holder) setFileLimit() { return } // If the soft limit is lower than the FileLimit constant, we will try to change it. - if oldLimit.Cur < FileLimit { - newLimit.Cur = FileLimit + if oldLimit.Cur < fileLimit { + newLimit.Cur = fileLimit // If the hard limit is not high enough, we will try to change it too. - if oldLimit.Max < FileLimit { - newLimit.Max = FileLimit + if oldLimit.Max < fileLimit { + newLimit.Max = fileLimit } else { newLimit.Max = oldLimit.Max } @@ -505,8 +508,8 @@ func (h *Holder) setFileLimit() { if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { h.Logger.Printf("ERROR checking open file limit: %s", err) } else { - if oldLimit.Cur < FileLimit { - h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) + if oldLimit.Cur < fileLimit { + h.Logger.Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", fileLimit, oldLimit.Cur, fileLimit) } } } @@ -558,14 +561,15 @@ func (h *Holder) logStartup() error { return nil } -// HolderSyncer is an active anti-entropy tool that compares the local holder +// holderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. -type HolderSyncer struct { +type holderSyncer struct { + mu sync.Mutex + Holder *Holder - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *cluster // Stats Stats StatsClient @@ -575,7 +579,7 @@ type HolderSyncer struct { } // IsClosing returns true if the syncer has been marked to close. -func (s *HolderSyncer) IsClosing() bool { +func (s *holderSyncer) IsClosing() bool { select { case <-s.Closing: return true @@ -585,7 +589,9 @@ func (s *HolderSyncer) IsClosing() bool { } // SyncHolder compares the holder on host with the local holder and resolves differences. -func (s *HolderSyncer) SyncHolder() error { +func (s *holderSyncer) SyncHolder() error { + s.mu.Lock() // only allow one instance of SyncHolder to be running at a time + defer s.mu.Unlock() ti := time.Now() // Iterate over schema in sorted order. for _, di := range s.Holder.Schema() { @@ -600,15 +606,15 @@ func (s *HolderSyncer) SyncHolder() error { } tf := time.Now() - for _, fi := range di.Frames { + for _, fi := range di.Fields { // Verify syncer has not closed. if s.IsClosing() { return nil } - // Sync frame row attributes. - if err := s.syncFrame(di.Name, fi.Name); err != nil { - return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err) + // Sync field row attributes. + if err := s.syncField(di.Name, fi.Name); err != nil { + return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err) } for _, vi := range fi.Views { @@ -617,9 +623,9 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { - // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { + for shard := uint64(0); shard <= s.Holder.Index(di.Name).maxShard(); shard++ { + // Ignore shards that this host doesn't own. + if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) { continue } @@ -629,12 +635,12 @@ func (s *HolderSyncer) SyncHolder() error { } // Sync fragment if own it. - if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + if err := s.syncFragment(di.Name, fi.Name, vi.Name, shard); err != nil { + return fmt.Errorf("fragment sync error: index=%s, field=%s, shard=%d, err=%s", di.Name, fi.Name, shard, err) } } } - s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0) + s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0) tf = time.Now() // reset tf } s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0) @@ -645,7 +651,7 @@ func (s *HolderSyncer) SyncHolder() error { } // syncIndex synchronizes index attributes with the rest of the cluster. -func (s *HolderSyncer) syncIndex(index string) error { +func (s *holderSyncer) syncIndex(index string) error { // Retrieve index reference. idx := s.Holder.Index(index) if idx == nil { @@ -662,11 +668,9 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ColumnAttrDiff(context.Background(), index, blks) + m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks) if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { @@ -689,38 +693,36 @@ func (s *HolderSyncer) syncIndex(index string) error { return nil } -// syncFrame synchronizes frame attributes with the rest of the cluster. -func (s *HolderSyncer) syncFrame(index, name string) error { - // Retrieve frame reference. - f := s.Holder.Frame(index, name) +// syncField synchronizes field attributes with the rest of the cluster. +func (s *holderSyncer) syncField(index, name string) error { + // Retrieve field reference. + f := s.Holder.Field(index, name) if f == nil { return nil } indexTag := fmt.Sprintf("index:%s", index) - frameTag := fmt.Sprintf("frame:%s", name) + fieldTag := fmt.Sprintf("field:%s", name) // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { return errors.Wrap(err, "getting blocks") } - s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) + s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag}) // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.RowAttrDiff(context.Background(), index, name, blks) - if err == ErrFrameNotFound { - continue // frame not created remotely yet, skip + m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks) + if err == ErrFieldNotFound { + continue // field not created remotely yet, skip } else if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID}) + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { @@ -738,53 +740,52 @@ func (s *HolderSyncer) syncFrame(index, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error { - // Retrieve local frame. - f := s.Holder.Frame(index, frame) +func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) error { + // Retrieve local field. + f := s.Holder.Field(index, field) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Ensure view exists locally. - v, err := f.CreateViewIfNotExists(view) + v, err := f.createViewIfNotExists(view) if err != nil { return errors.Wrap(err, "creating view") } // Ensure fragment exists locally. - frag, err := v.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return errors.Wrap(err, "creating fragment") } // Sync fragments together. - fs := FragmentSyncer{ - Fragment: frag, - Node: s.Node, - Cluster: s.Cluster, - Closing: s.Closing, - RemoteClient: s.RemoteClient, + fs := fragmentSyncer{ + Fragment: frag, + Node: s.Node, + Cluster: s.Cluster, + Closing: s.Closing, } - if err := fs.SyncFragment(); err != nil { + if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") } return nil } -// HolderCleaner removes fragments and data files that are no longer used. -type HolderCleaner struct { +// holderCleaner removes fragments and data files that are no longer used. +type holderCleaner struct { Node *Node Holder *Holder - Cluster *Cluster + Cluster *cluster // Signals that the sync should stop. Closing <-chan struct{} } // IsClosing returns true if the cleaner has been marked to close. -func (c *HolderCleaner) IsClosing() bool { +func (c *holderCleaner) IsClosing() bool { select { case <-c.Closing: return true @@ -795,7 +796,7 @@ func (c *HolderCleaner) IsClosing() bool { // CleanHolder compares the holder with the cluster state and removes // any unnecessary fragments and files. -func (c *HolderCleaner) CleanHolder() error { +func (c *holderCleaner) CleanHolder() error { for _, index := range c.Holder.Indexes() { // Verify cleaner has not closed. if c.IsClosing() { @@ -803,19 +804,19 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) + containedShards := c.Cluster.containsShards(index.Name(), index.maxShard(), c.Node) // Get the fragments registered in memory. - for _, frame := range index.Frames() { - for _, view := range frame.Views() { - for _, fragment := range view.Fragments() { - fragSlice := fragment.Slice() + for _, field := range index.Fields() { + for _, view := range field.views() { + for _, fragment := range view.allFragments() { + fragShard := fragment.shard // Ignore fragments that should be present. - if uint64InSlice(fragSlice, containedSlices) { + if uint64InSlice(fragShard, containedShards) { continue } // Delete fragment. - if err := view.DeleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragShard); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_internal_test.go b/holder_internal_test.go new file mode 100644 index 000000000..75a3508fd --- /dev/null +++ b/holder_internal_test.go @@ -0,0 +1,291 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +type tHolder struct { + *Holder +} + +// Close closes the holder and removes all underlying data. +func (h *tHolder) Close() error { + defer os.RemoveAll(h.Path) + return h.Holder.Close() +} + +// Reopen instantiates and opens a new holder. +// Note that the holder must be Closed first. +func (h *tHolder) Reopen() error { + path, logger := h.Path, h.Holder.Logger + h.Holder = NewHolder() + h.Holder.Path = path + h.Holder.Logger = logger + if err := h.Holder.Open(); err != nil { + return err + } + + return nil +} + +func newHolder() *tHolder { + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } + + h := &tHolder{Holder: NewHolder()} + h.Path = path + return h +} + +// MustCreateFieldIfNotExists returns a given field. Panic on error. +func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field { + f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault()) + if err != nil { + panic(err) + } + return f +} + +// MustCreateIndexIfNotExists returns a given index. Panic on error. +func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index { + idx, err := h.Holder.CreateIndexIfNotExists(index, opt) + if err != nil { + panic(err) + } + return idx +} + +// SetBit clears a bit on the given field. +func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) { + f := h.MustCreateFieldIfNotExists(index, field) + _, err := f.SetBit(rowID, columnID, nil) + if err != nil { + panic(err) + } +} + +// Row returns a Row for a given field. +func (h *tHolder) Row(index, field string, rowID uint64) *Row { + f := h.MustCreateFieldIfNotExists(index, field) + row, err := f.Row(rowID) + if err != nil { + panic(err) + } + return row +} + +func TestHolder_Optn(t *testing.T) { + t.Run("ErrViewPermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFragmentCachePermission", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("Skipping permissions test since user is root.") + } + h := newHolder() + defer h.Close() + + if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { + t.Fatal(err) + } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if view, err := field.createViewIfNotExists(viewStandard); err != nil { + t.Fatal(err) + } else if _, err := field.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } else if err := view.Fragment(0).FlushCache(); err != nil { + t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) + } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) + + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("unexpected error: %s", err) + } + }) + +} + +// Ensure holder can clean up orphaned fragments. +func TestHolderCleaner_CleanHolder(t *testing.T) { + cluster := NewTestCluster(2) + + // Create a local holder. + hldr0 := newHolder() + defer hldr0.Close() + + // Mock 2-node, fully replicated cluster. + cluster.ReplicaN = 2 + + cluster.Nodes[0].URI = NewTestURIFromHostPort("localhost", 0) + + // Create fields on nodes. + for _, hldr := range []*tHolder{hldr0} { + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") + } + + // Set data on the local holder. + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 0, 4000) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 3, 10) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) + + hldr0.SetBit("i", "f0", 9, ShardWidth+5) + + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) + + // Set highest shard. + hldr0.Index("i").setRemoteMaxShard(1) + hldr0.Index("y").setRemoteMaxShard(2) + + // Keep replication the same and ensure we get the expected results. + cluster.ReplicaN = 2 + + // Set up cleaner for replication 2. + cleaner2 := holderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner2.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*tHolder{hldr0} { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) + } + + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) + } + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + } + } + + // Change replication factor to ensure we have fragments to remove. + cluster.ReplicaN = 1 + + // Set up cleaner for replication 1. + cleaner1 := holderCleaner{ + Node: cluster.Nodes[0], + Holder: hldr0.Holder, + Cluster: cluster, + } + + if err := cleaner1.CleanHolder(); err != nil { + t.Fatal(err) + } + + // Verify data is the same on both nodes. + for i, hldr := range []*tHolder{hldr0} { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) + } + + f := hldr.fragment("i", "f0", viewStandard, 1) + if f != nil { + t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) + } + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + } + } +} diff --git a/holder_test.go b/holder_test.go index c7877e3ba..d4886ff30 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,6 @@ import ( "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -91,7 +89,7 @@ func TestHolder_Open(t *testing.T) { } }) - t.Run("ErrFramePermission", func(t *testing.T) { + t.Run("ErrFieldPermission", func(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } @@ -100,7 +98,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -113,13 +111,13 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -127,17 +125,17 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -145,56 +143,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrViewPermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0777) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0777) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") { t.Fatalf("unexpected error: %s", err) } }) @@ -208,11 +157,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -231,11 +178,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -243,39 +188,11 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=opening storage: unmarshal storage") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFragmentCachePermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := test.MustOpenHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { - t.Fatal(err) - } else if err := view.Fragment(0).FlushCache(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { - t.Fatal(err) - } - defer os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0666) - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) } func TestHolder_HasData(t *testing.T) { @@ -331,14 +248,8 @@ func TestHolder_DeleteIndex(t *testing.T) { defer hldr.Close() // Write bits to separate indexes. - f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) - if _, err := f0.SetBit(100, 200); err != nil { - t.Fatal(err) - } - f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0) - if _, err := f1.SetBit(100, 200); err != nil { - t.Fatal(err) - } + hldr.SetBit("i0", "f", 100, 200) + hldr.SetBit("i1", "f", 100, 200) // Ensure i0 exists. if _, err := os.Stat(hldr.IndexPath("i0")); err != nil { @@ -360,263 +271,94 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { - cluster := test.NewCluster(2) - client := server.GetHTTPClient(nil) - // Create a local holder. - hldr0 := test.MustOpenHolder() - defer hldr0.Close() - - // Create a remote holder wrapped by an HTTP - hldr1 := test.MustOpenHolder() - defer hldr1.Close() - s := test.NewServer() - defer s.Close() - s.Handler.API.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(client) - e.Holder = hldr1.Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) - } - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - uri, err := pilosa.NewURIFromAddress(s.URL) + c := test.MustNewCluster(t, 2) + c[0].Config.Cluster.ReplicaN = 2 + c[0].Config.AntiEntropy.Interval = 0 + c[1].Config.Cluster.ReplicaN = 2 + c[1].Config.AntiEntropy.Interval = 0 + err := c.Start() if err != nil { - t.Fatal(err) + t.Fatalf("starting cluster: %v", err) + } + defer c.Close() + + _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index i: %v", err) + } + _, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index y: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f0: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field z in y: %v", err) } - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) - cluster.Nodes[1].URI = *uri - - // Create frames on nodes. - for _, hldr := range []*test.Holder{hldr0, hldr1} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") - } + hldr0 := &test.Holder{Holder: c[0].Server.Holder()} + hldr1 := &test.Holder{Holder: c[1].Server.Holder()} // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f0", 9, ShardWidth+5) - hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) + // Set a bit to create the fragment. + hldr0.SetBit("y", "z", 0, 0) // Set data on the remote holder. - f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) + hldr1.SetBit("i", "f", 0, 4000) + hldr1.SetBit("i", "f", 3, 10) + hldr1.SetBit("i", "f", 120, 10) + + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7) + + err = c[0].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 0: %v", err) } - - f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { - t.Fatal(err) - } - - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(3) - - // Set up syncer. - syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - RemoteClient: server.GetHTTPClient(nil), - Stats: pilosa.NopStatsClient, - } - - if err := syncer.SyncHolder(); err != nil { - t.Fatal(err) + err = c[1].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 1: %v", err) } // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Errorf("unexpected columns(%d/0): %+v", i, a) + } + if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Errorf("unexpected columns(%d/2): %+v", i, a) + } + if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/3): %+v", i, a) + } + if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/120): %+v", i, a) + } + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Errorf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { + t.Errorf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) - } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } -} - -// Ensure holder can clean up orphaned fragments. -func TestHolderCleaner_CleanHolder(t *testing.T) { - cluster := test.NewCluster(2) - - // Create a local holder. - hldr0 := test.MustOpenHolder() - defer hldr0.Close() - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) - - // Create frames on nodes. - for _, hldr := range []*test.Holder{hldr0} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") - } - - // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } - - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } - - f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2) - if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil { - t.Fatal(err) - } - - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(2) - - // Keep replication the same and ensure we get the expected results. - cluster.ReplicaN = 2 - - // Set up cleaner for replication 2. - cleaner2 := pilosa.HolderCleaner{ - Node: cluster.Nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner2.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) - } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } - - // Change replication factor to ensure we have fragments to remove. - cluster.ReplicaN = 1 - - // Set up cleaner for replication 1. - cleaner1 := pilosa.HolderCleaner{ - Node: cluster.Nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner1.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - if f != nil { - t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) - } - - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) { + t.Errorf("unexpected columns(%d/y/z): %+v", i, a) } } } diff --git a/http/client.go b/http/client.go new file mode 100644 index 000000000..bf5c9f5dc --- /dev/null +++ b/http/client.go @@ -0,0 +1,998 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "sort" + "strconv" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/encoding/proto" + "github.com/pkg/errors" +) + +// InternalClient represents a client to the Pilosa cluster. +type InternalClient struct { + defaultURI *pilosa.URI + serializer pilosa.Serializer + + // The client to use for HTTP communication. + httpClient *http.Client +} + +// NewInternalClient returns a new instance of InternalClient to connect to host. +func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) { + if host == "" { + return nil, pilosa.ErrHostRequired + } + + uri, err := pilosa.NewURIFromAddress(host) + if err != nil { + return nil, errors.Wrap(err, "getting URI") + } + + client := NewInternalClientFromURI(uri, remoteClient) + return client, nil +} + +func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { + return &InternalClient{ + defaultURI: defaultURI, + serializer: proto.Serializer{}, + httpClient: remoteClient, + } +} + +// MaxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxShardByIndex(ctx) +} + +// maxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/internal/shards/max") + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getShardsMaxResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return rsp.Standard, nil +} + +// Schema returns all index and field schema information. +func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + // Execute request against the host. + u := c.defaultURI.Path("/schema") + + // Build request. + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSchemaResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + return rsp.Indexes, nil +} + +// CreateIndex creates a new index on the server. +func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { + // Encode query request. + buf, err := json.Marshal(&postIndexRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "encoding request") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrIndexExists + default: + return errors.New(string(body)) + } +} + +// FragmentNodes returns a list of nodes that own a shard. +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes") + u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var a []*pilosa.Node + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return a, nil +} + +// Query executes query against the index. +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + if index == "" { + return nil, pilosa.ErrIndexRequired + } else if queryRequest.Query == "" { + return nil, pilosa.ErrQueryRequired + } + + buf, err := c.serializer.Marshal(queryRequest) + if err != nil { + return nil, errors.Wrap(err, "marshaling queryRequest") + } + + // Create HTTP request. + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return nil, errors.New(string(body)) + } + + qresp := &pilosa.QueryResponse{} + if err := c.serializer.Unmarshal(body, qresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } else if qresp.Err != nil { + return nil, qresp.Err + } + + return qresp, nil +} + +// Import bulk imports bits for a single shard to a host. +func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := c.marshalImportPayload(index, field, shard, bits) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) + if err != nil { + return fmt.Errorf("shard nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importNode(ctx, node, index, field, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// ImportK bulk imports bits specified by string keys to a host. +func (c *InternalClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := c.marshalImportPayloadK(index, field, columns) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + node := &pilosa.Node{ + URI: *c.defaultURI, + } + + // Import to node. + if err := c.importNode(ctx, node, index, field, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + + return nil +} + +func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { + err := c.CreateIndex(ctx, name, options) + if err == nil || errors.Cause(err) == pilosa.ErrIndexExists { + return nil + } + return err +} + +func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { + err := c.CreateField(ctx, indexName, fieldName) + if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { + return nil + } + return err +} + +// marshalImportPayload marshalls the import parameters into a protobuf byte slice. +func (c *InternalClient) marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ + Index: index, + Field: field, + Shard: shard, + RowIDs: rowIDs, + ColumnIDs: columnIDs, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. +func (c *InternalClient) marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowKeys := Bits(bits).RowKeys() + columnKeys := Bits(bits).ColumnKeys() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := c.serializer.Marshal(&pilosa.ImportRequest{ + Index: index, + Field: field, + RowKeys: rowKeys, + ColumnKeys: columnKeys, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importNode sends a pre-marshaled import request to a node. +func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte) error { + // Create URL & HTTP request. + path := fmt.Sprintf("/index/%s/field/%s/import", index, field) + u := nodePathToURL(node, path) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp pilosa.ImportResponse + if err := c.serializer.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ImportValue bulk imports field values for a single shard to a host. +func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := c.marshalImportValuePayload(index, field, shard, vals) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) + if err != nil { + return fmt.Errorf("shard nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importNode(ctx, node, index, field, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. +func (c *InternalClient) marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + columnIDs := FieldValues(vals).ColumnIDs() + values := FieldValues(vals).Values() + + // Marshal data to protobuf. + buf, err := c.serializer.Marshal(&pilosa.ImportValueRequest{ + Index: index, + Field: field, + Shard: shard, + ColumnIDs: columnIDs, + Values: values, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// ExportCSV bulk exports data for a single shard from a host to CSV format. +func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) + if err != nil { + return fmt.Errorf("shard nodes: %s", err) + } + + // Attempt nodes in random order. + var e error + for _, i := range rand.Perm(len(nodes)) { + node := nodes[i] + + if err := c.exportNodeCSV(ctx, node, index, field, shard, w); err != nil { + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) + continue + } else { + return nil + } + } + + return e +} + +// exportNode copies a CSV export from a node to w. +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error { + // Create URL. + u := nodePathToURL(node, "/export") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "shard": {strconv.FormatUint(shard, 10)}, + }.Encode() + + // Generate HTTP request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Accept", "text/csv") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Validate status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: %d", resp.StatusCode) + } + + // Copy body to writer. + if _, err := io.Copy(w, resp.Body); err != nil { + return errors.Wrap(err, "copying") + } + + return nil +} + +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { + node := &pilosa.Node{ + URI: uri, + } + return c.backupShardNode(ctx, index, field, shard, node) +} + +func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { + u := nodePathToURL(node, "/fragment/data") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "shard": {strconv.FormatUint(shard, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + + // Return error if status is not OK. + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil, pilosa.ErrFragmentNotFound + } else if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) + } + + return resp.Body, nil +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + if index == "" { + return pilosa.ErrIndexRequired + } + + // TODO: remove buf completely? (depends on whether importer needs to create specific field types) + // Encode query request. + buf, err := json.Marshal(&postFieldRequest{ + //Options: opt, + }) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrFieldExists + default: + return errors.New(string(body)) + } +} + +// FragmentBlocks returns a list of block checksums for a fragment on a host. +// Only returns blocks which contain data. +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64) ([]pilosa.FragmentBlock, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, "/internal/fragment/blocks") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "shard": {strconv.FormatUint(shard, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFragmentNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp getFragmentBlocksResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Blocks, nil +} + +// BlockData returns row/column id pairs for a block. +func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { + if uri == nil { + panic("need to pass a URI to BlockData") + } + buf, err := c.serializer.Marshal(&pilosa.BlockDataRequest{ + Index: index, + Field: field, + Shard: shard, + Block: uint64(block), + }) + if err != nil { + return nil, nil, errors.Wrap(err, "marshaling") + } + + u := uriPathToURL(uri, "/internal/fragment/block/data") + req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/protobuf") + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Accept", "application/protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // fallthrough + case http.StatusNotFound: + return nil, nil, nil + default: + return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp pilosa.BlockDataResponse + if body, err := ioutil.ReadAll(resp.Body); err != nil { + return nil, nil, errors.Wrap(err, "reading") + } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { + return nil, nil, errors.Wrap(err, "unmarshalling") + } + return rsp.RowIDs, rsp.ColumnIDs, nil +} + +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/attr/diff", index)) + + // Encode request. + buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postIndexAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/field/%s/attr/diff", index, field)) + + // Encode request. + buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFieldNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postFieldAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// SendMessage posts a message synchronously. +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error { + u := uriPathToURL(uri, "/internal/cluster/message") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) + if err != nil { + return errors.Wrap(err, "making new request") + } + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/json") + + // Execute request. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("executing http request: %v", err) + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %v", err) + } + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) + } + + return nil +} + +// Bits is a slice of Bit. +type Bits []pilosa.Bit + +func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Bits) Len() int { return len(p) } + +func (p Bits) Less(i, j int) bool { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { + return p[i].Timestamp < p[j].Timestamp + } + return p[i].ColumnID < p[j].ColumnID + } + return p[i].RowID < p[j].RowID +} + +// RowIDs returns a slice of all the row IDs. +func (p Bits) RowIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].RowID + } + return other +} + +// ColumnIDs returns a slice of all the column IDs. +func (p Bits) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// RowKeys returns a slice of all the row keys. +func (p Bits) RowKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].RowKey + } + return other +} + +// ColumnKeys returns a slice of all the column keys. +func (p Bits) ColumnKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].ColumnKey + } + return other +} + +// Timestamps returns a slice of all the timestamps. +func (p Bits) Timestamps() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Timestamp + } + return other +} + +// GroupByShard returns a map of bits by shard. +func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { + m := make(map[uint64][]pilosa.Bit) + for _, bit := range p { + shard := bit.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], bit) + } + + for shard, bits := range m { + sort.Sort(Bits(bits)) + m[shard] = bits + } + + return m +} + +// FieldValues represents a slice of field values. +type FieldValues []pilosa.FieldValue + +func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p FieldValues) Len() int { return len(p) } + +func (p FieldValues) Less(i, j int) bool { + return p[i].ColumnID < p[j].ColumnID +} + +// ColumnIDs returns a slice of all the column IDs. +func (p FieldValues) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// Values returns a slice of all the values. +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Value + } + return other +} + +// GroupByShard returns a map of field values by shard. +func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { + m := make(map[uint64][]pilosa.FieldValue) + for _, val := range p { + shard := val.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], val) + } + + for shard, vals := range m { + sort.Sort(FieldValues(vals)) + m[shard] = vals + } + + return m +} + +// BitsByPos is a slice of bits sorted row then column. +type BitsByPos []pilosa.Bit + +func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitsByPos) Len() int { return len(p) } +func (p BitsByPos) Less(i, j int) bool { + p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 +} + +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { + return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) +} + +func uriPathToURL(uri *pilosa.URI, path string) url.URL { + return url.URL{ + Scheme: uri.Scheme, + Host: uri.HostPort(), + Path: path, + } +} + +func nodePathToURL(node *pilosa.Node, path string) url.URL { + return url.URL{ + Scheme: node.URI.Scheme, + Host: node.URI.HostPort(), + Path: path, + } +} diff --git a/http/client_test.go b/http/client_test.go new file mode 100644 index 000000000..2944a7586 --- /dev/null +++ b/http/client_test.go @@ -0,0 +1,322 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http_test + +import ( + "context" + "fmt" + gohttp "net/http" + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/test" +) + +var defaultClient *gohttp.Client + +func init() { + defaultClient = http.GetHTTPClient(nil) + +} + +// Test distributed TopN Row count across 3 nodes. +func TestClient_MultiNode(t *testing.T) { + c := test.MustRunCluster(t, 3, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + ) + defer c.Close() + + hldr := []test.Holder{} + for _, command := range c { + hldr = append(hldr, test.Holder{Holder: command.Server.Holder()}) + } + + // Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN. + shardNums := []uint64{1, 2, 6} + + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())` + owns := [][]uint64{ + {1, 3, 4, 8, 10, 13, 17, 19}, + {2, 5, 7, 11, 12, 14, 18}, + {0, 6, 9, 15, 16, 20}, + } + + for i, num := range shardNums { + ownsNum := false + for _, ownNum := range owns[i] { + if ownNum == num { + ownsNum = true + break + } + } + if !ownsNum { + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns) + } + } + + baseBit0 := pilosa.ShardWidth * shardNums[0] + baseBit1 := pilosa.ShardWidth * shardNums[1] + baseBit2 := pilosa.ShardWidth * shardNums[2] + + maxShard := uint64(0) + for _, x := range shardNums { + if x > maxShard { + maxShard = x + } + } + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + hldr[0].MustSetBits("i", "f", 100, baseBit0+10) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2) + + hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustSetBits("i", "f", 1, baseBit1+4) + hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + + hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustSetBits("i", "f", 21, baseBit2+10) + hldr[2].MustSetBits("i", "f", 100, baseBit2+10) + hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11) + hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12) + + // Rebuild the RankCache. + // We have to do this to avoid the 10-second cache invalidation delay + // built into cache.Invalidate() + c[0].RecalculateCaches() + c[1].RecalculateCaches() + c[2].RecalculateCaches() + + // Connect to each node to compare results. + client := make([]*Client, 3) + client[0] = MustNewClient(c[0].URL(), defaultClient) + client[1] = MustNewClient(c[1].URL(), defaultClient) + client[2] = MustNewClient(c[2].URL(), defaultClient) + + topN := 4 + queryRequest := &pilosa.QueryRequest{ + Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), + Remote: false, + } + + result, err := client[0].Query(context.Background(), "i", queryRequest) + if err != nil { + t.Fatal(err) + } + + result, err = client[0].Query(context.Background(), "i", queryRequest) + if err != nil { + t.Fatal(err) + } + + // Test must return exactly N results. + if len(result.Results[0].([]pilosa.Pair)) != topN { + t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) + } + p := []pilosa.Pair{ + {ID: 100, Count: 12}, + {ID: 22, Count: 10}, + {ID: 98, Count: 8}, + {ID: 99, Count: 7}} + + // Valdidate the Top 4 result counts. + if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { + t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) + } + + result1, err := client[1].Query(context.Background(), "i", queryRequest) + if err != nil { + t.Fatal(err) + } + result2, err := client[2].Query(context.Background(), "i", queryRequest) + if err != nil { + t.Fatal(err) + } + + // Compare TopN results across all nodes in the cluster. + if !reflect.DeepEqual(result, result1) { + t.Fatalf("TopN result should be the same on node0 and node1: %s", spew.Sdump(result1)) + } + + if !reflect.DeepEqual(result, result2) { + t.Fatalf("TopN result should be the same on node0 and node2: %s", spew.Sdump(result2)) + } +} + +// Ensure client can bulk import data. +func TestClient_Import(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + host := cmd.URL() + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + + // Load bitmap into cache to ensure cache gets updated. + hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. + hldr.Row("i", "f", 0) + + // Send import request. + c := MustNewClient(host, defaultClient) + if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, + }); err != nil { + t.Fatal(err) + } + + // Verify data. + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + t.Fatalf("unexpected columns: %+v", a) + } + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + t.Fatalf("unexpected columns: %+v", a) + } +} + +// Ensure client can bulk import value data. +func TestClient_ImportValue(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + host := cmd.URL() + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + + fldName := "f" + + // Load bitmap into cache to ensure cache gets updated. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) + if err != nil { + t.Fatal(err) + } + + // Send import request. + c := MustNewClient(host, defaultClient) + if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ + {ColumnID: 1, Value: -10}, + {ColumnID: 2, Value: 20}, + {ColumnID: 3, Value: 40}, + }); err != nil { + t.Fatal(err) + } + + // Verify Sum. + sum, cnt, err := field.Sum(nil, fldName) + if err != nil { + t.Fatal(err) + } + if sum != 50 || cnt != 3 { + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify Min. + min, cnt, err := field.Min(nil, fldName) + if err != nil { + t.Fatal(err) + } + if min != -10 || cnt != 1 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) + } + + // Verify Min with Filter. + filter, err := field.Range(fldName, pql.GT, 40) + if err != nil { + t.Fatal(err) + } + min, cnt, err = field.Min(filter, fldName) + if err != nil { + t.Fatal(err) + } + if min != -100 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + } + + // Verify Max. + max, cnt, err := field.Max(nil, fldName) + if err != nil { + t.Fatal(err) + } + if max != 40 || cnt != 1 { + t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) + } +} + +// Ensure client can retrieve a list of all checksums for blocks in a fragment. +func TestClient_FragmentBlocks(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) + + // Set a bit on a different shard. + hldr.SetBit("i", "f", 0, 1) + c := MustNewClient(cmd.URL(), defaultClient) + blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) + if err != nil { + t.Fatal(err) + } else if len(blocks) != 2 { + t.Fatalf("unexpected blocks: %s", spew.Sdump(blocks)) + } else if blocks[0].ID != 0 { + t.Fatalf("unexpected block id(0): %d", blocks[0].ID) + } else if blocks[1].ID != 3 { + t.Fatalf("unexpected block id(1): %d", blocks[1].ID) + } + + // Verify data matches local blocks. + if a, err := cmd.API.FragmentBlocks(context.Background(), "i", "f", 0); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(a, blocks) { + t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) + } +} + +// Client represents a test wrapper for pilosa.Client. +type Client struct { + *http.InternalClient +} + +// MustNewClient returns a new instance of Client. Panic on error. +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalClient(host, h) + if err != nil { + panic(err) + } + return &Client{InternalClient: c} +} diff --git a/test/test.go b/http/error.go similarity index 63% rename from test/test.go rename to http/error.go index 01f980ad2..1a3f07c04 100644 --- a/test/test.go +++ b/http/error.go @@ -12,4 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package test +package http + +// Error defines a standard application error. +type Error struct { + // Machine-readable error code. + code string `json:"code,omitempty"` + + // Human-readable message. + Message string `json:"message"` +} + +// Error returns the string representation of the error message. +func (e *Error) Error() string { + return e.Message +} diff --git a/http/handler.go b/http/handler.go new file mode 100644 index 000000000..4946f69e0 --- /dev/null +++ b/http/handler.go @@ -0,0 +1,1370 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "context" + "crypto/tls" + "encoding/json" + "expvar" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" + "reflect" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa" + + "github.com/pkg/errors" +) + +// Handler represents an HTTP handler. +type Handler struct { + Handler http.Handler + + logger pilosa.Logger + + // Keeps the query argument validators for each handler + validators map[string]*queryValidationSpec + + api *pilosa.API + + allowedOrigins []string + + ln net.Listener + + server *http.Server +} + +// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. +// This is used for stats tagging. +var externalPrefixFlag = map[string]bool{ + "schema": true, + "query": true, + "import": true, + "export": true, + "index": true, + "field": true, + "nodes": true, + "version": true, +} + +type errorResponse struct { + Error string `json:"error"` +} + +// handlerOption is a functional option type for pilosa.Handler +type handlerOption func(s *Handler) error + +func OptHandlerAllowedOrigins(origins []string) handlerOption { + return func(h *Handler) error { + h.Handler = handlers.CORS( + handlers.AllowedOrigins(origins), + handlers.AllowedHeaders([]string{"Content-Type"}), + )(h.Handler) + return nil + } +} + +func OptHandlerAPI(api *pilosa.API) handlerOption { + return func(h *Handler) error { + h.api = api + return nil + } +} + +func OptHandlerLogger(logger pilosa.Logger) handlerOption { + return func(h *Handler) error { + h.logger = logger + return nil + } +} + +func OptHandlerListener(ln net.Listener) handlerOption { + return func(h *Handler) error { + h.ln = ln + return nil + } +} + +// NewHandler returns a new instance of Handler with a default logger. +func NewHandler(opts ...handlerOption) (*Handler, error) { + handler := &Handler{ + logger: pilosa.NopLogger, + } + handler.Handler = newRouter(handler) + handler.populateValidators() + + for _, opt := range opts { + err := opt(handler) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + if handler.api == nil { + return nil, errors.New("must pass OptHandlerAPI") + } + + if handler.ln == nil { + return nil, errors.New("must pass OptHandlerListener") + } + + handler.server = &http.Server{Handler: handler} + + return handler, nil +} + +func (h *Handler) Serve() error { + err := h.server.Serve(h.ln) + if err != nil && err.Error() != "http: Server closed" { + h.logger.Printf("HTTP handler terminated with error: %s\n", err) + return errors.Wrap(err, "serve http") + } + return nil +} + +func (h *Handler) Close() error { + // TODO: timeout? + err := h.server.Shutdown(context.Background()) + return errors.Wrap(err, "shutdown http server") +} + +func (h *Handler) populateValidators() { + h.validators = map[string]*queryValidationSpec{} + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") + h.validators["GetShardMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "shard") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { + if err := validator.validate(r.URL.Query()); err != nil { + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) +} + +// newRouter creates a new mux http router. +func newRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() + router.HandleFunc("/", handler.handleHome).Methods("GET") + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + // /internal endpoints are for internal use only; they may change at any time. + // DO NOT rely on these for external applications! + router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") + router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET") + + // TODO: Apply MethodNotAllowed statuses to all endpoints. + // Ideally this would be automatic, as described in this (wontfix) ticket: + // https://github.com/gorilla/mux/issues/6 + // For now we just do it for the most commonly used handler, /query + router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + + router.Use(handler.queryArgValidator) + return router +} + +func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} + +// ServeHTTP handles an HTTP request. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + h.logger.Printf(msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + + t := time.Now() + h.Handler.ServeHTTP(w, r) + dif := time.Since(t) + + // Calculate per request StatsD metrics when the handler is fully configured. + statsTags := make([]string, 0, 3) + + longQueryTime := h.api.LongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") + } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + stats := h.api.StatsWithTags(statsTags) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } +} + +// successResponse is a general success/error struct for http responses. +type successResponse struct { + Success bool `json:"success"` + Error *Error `json:"error,omitempty"` +} + +// check determines success or failure based on the error. +// It also returns the corresponding http status code. +func (r *successResponse) check(err error) (statusCode int) { + if err == nil { + r.Success = true + return + } + + cause := errors.Cause(err) + + // Determine HTTP status code based on the error type. + switch cause.(type) { + case pilosa.BadRequestError: + statusCode = http.StatusBadRequest + case pilosa.ConflictError: + statusCode = http.StatusConflict + case pilosa.NotFoundError: + statusCode = http.StatusNotFound + default: + statusCode = http.StatusInternalServerError + } + + r.Success = false + r.Error = &Error{Message: cause.Error()} + + return +} + +// write sends a response to the http.ResponseWriter based on the success +// status and the error. +func (r *successResponse) write(w http.ResponseWriter, err error) { + // Apply the error and get the status code. + statusCode := r.check(err) + + // Marshal the json response. + msg, err := json.Marshal(r) + if err != nil { + http.Error(w, string(msg), http.StatusInternalServerError) + return + } + + // Write the response. + if statusCode == 0 { + w.Write(msg) + w.Write([]byte("\n")) + } else { + http.Error(w, string(msg), statusCode) + } +} + +func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) +} + +// validHeaderAcceptJSON returns false if one or more Accept +// headers are present, but none of them are "application/json" +// (or any matching wildcard). Otherwise returns true. +func validHeaderAcceptJSON(header http.Header) bool { + if v, found := header["Accept"]; found { + for _, v := range v { + if v == "application/json" || v == "*/*" || v == "*/json" || v == "application/*" { + return true + } + } + return false + } + return true +} + +// handleGetSchema handles GET /schema requests. +func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + schema := h.api.Schema(r.Context()) + if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { + h.logger.Printf("write schema response error: %s", err) + } +} + +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + status := getStatusResponse{ + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + LocalID: h.api.Node().ID, + } + if err := json.NewEncoder(w).Encode(status); err != nil { + h.logger.Printf("write status response error: %s", err) + } +} + +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + info := h.api.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.logger.Printf("write info response error: %s", err) + } +} + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +type getStatusResponse struct { + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` +} + +// handlePostQuery handles /query requests. +func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { + // Parse incoming request. + req, err := h.readQueryRequest(r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + // TODO: Remove + req.Index = mux.Vars(r)["index"] + + resp, err := h.api.Query(r.Context(), req) + if err != nil { + switch errors.Cause(resp.Err) { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusBadRequest) + } + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + + // Set appropriate status code, if there is an error. It doesn't appear that + // resp.Err could ever be set in API.Query, so this code block is probably + // doing nothing right now. + if resp.Err != nil { + switch errors.Cause(resp.Err) { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusBadRequest) + } + } + + // Write response back to client. + if err := h.writeQueryResponse(w, r, &resp); err != nil { + h.logger.Printf("write query response error: %s", err) + } +} + +// handleGetShardsMax handles GET /internal/shards/max requests. +func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + if err := json.NewEncoder(w).Encode(getShardsMaxResponse{ + Standard: h.api.MaxShards(r.Context()), + }); err != nil { + h.logger.Printf("write shards-max response error: %s", err) + } +} + +type getShardsMaxResponse struct { + Standard map[string]uint64 `json:"standard"` +} + +// handleGetIndexes handles GET /index request. +func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { + h.handleGetSchema(w, r) +} + +// handleGetIndex handles GET /index/ requests. +func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + for _, idx := range h.api.Schema(r.Context()) { + if idx.Name == indexName { + if err := json.NewEncoder(w).Encode(idx); err != nil { + h.logger.Printf("write response error: %s", err) + } + return + } + } + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) +} + +type postIndexRequest struct { + Options pilosa.IndexOptions `json:"options"` +} + +//_postIndexRequest is necessary to avoid recursion while decoding. +type _postIndexRequest postIndexRequest + +// Custom Unmarshal JSON to validate request body when creating a new index. +func (p *postIndexRequest) UnmarshalJSON(b []byte) error { + + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshalling unexpected values") + } + + validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + err := validateOptions(m, validIndexOptions) + if err != nil { + return err + } + // Unmarshal expected values. + var _p _postIndexRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected values") + } + + p.Options = _p.Options + + return nil +} + +func getValidOptions(option interface{}) []string { + validOptions := []string{} + val := reflect.ValueOf(option) + for i := 0; i < val.Type().NumField(); i++ { + jsonTag := val.Type().Field(i).Tag.Get("json") + s := strings.Split(jsonTag, ",") + validOptions = append(validOptions, s[0]) + } + return validOptions +} + +// Raise errors for any unknown key +func validateOptions(data map[string]interface{}, validIndexOptions []string) error { + for k, v := range data { + switch k { + case "options": + options, ok := v.(map[string]interface{}) + if !ok { + return errors.New("options is not map[string]interface{}") + } + for kk, vv := range options { + if !foundItem(validIndexOptions, kk) { + return fmt.Errorf("Unknown key: %v:%v", kk, vv) + } + } + default: + return fmt.Errorf("Unknown key: %v:%v", k, v) + } + } + return nil +} + +func foundItem(items []string, item string) bool { + for _, i := range items { + if item == i { + return true + } + } + return false +} + +// handleDeleteIndex handles DELETE /index request. +func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + + resp := successResponse{} + err := h.api.DeleteIndex(r.Context(), indexName) + resp.write(w, err) +} + +// handlePostIndex handles POST /index request. +func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + + resp := successResponse{} + + // Decode request. + var req postIndexRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil && err != io.EOF { + resp.write(w, err) + return + } + _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) + + resp.write(w, err) +} + +// handlePostIndexAttrDiff handles POST /internal/index/attr/diff requests. +func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.api.IndexAttrDiff(r.Context(), indexName, req.Blocks) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type postIndexAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postIndexAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// handlePostField handles POST /field request. +func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + resp := successResponse{} + + // Decode request. + var req postFieldRequest + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + err := dec.Decode(&req) + if err != nil && err != io.EOF { + resp.write(w, err) + return + } + + // Validate field options. + if err := req.Options.validate(); err != nil { + resp.write(w, err) + return + } + + // Convert json options into functional options. + var fos []pilosa.FieldOption + switch req.Options.Type { + case pilosa.FieldTypeSet: + fos = append(fos, pilosa.OptFieldTypeSet(*req.Options.CacheType, *req.Options.CacheSize)) + case pilosa.FieldTypeInt: + fos = append(fos, pilosa.OptFieldTypeInt(*req.Options.Min, *req.Options.Max)) + case pilosa.FieldTypeTime: + fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) + } + if req.Options.Keys != nil { + if *req.Options.Keys { + fos = append(fos, pilosa.OptFieldKeys()) + } + } + + _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + resp.write(w, err) +} + +type postFieldRequest struct { + Options fieldOptions `json:"options"` +} + +// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values, +// and used for input validation. +type fieldOptions struct { + Type string `json:"type,omitempty"` + CacheType *string `json:"cacheType,omitempty"` + CacheSize *uint32 `json:"cacheSize,omitempty"` + Min *int64 `json:"min,omitempty"` + Max *int64 `json:"max,omitempty"` + TimeQuantum *pilosa.TimeQuantum `json:"timeQuantum,omitempty"` + Keys *bool `json:"keys,omitempty"` +} + +func (o *fieldOptions) validate() error { + // Pointers to default values. + defaultCacheType := pilosa.DefaultCacheType + defaultCacheSize := uint32(pilosa.DefaultCacheSize) + + switch o.Type { + case pilosa.FieldTypeSet, "": + // Because FieldTypeSet is the default, its arguments are + // not required. Instead, the defaults are applied whenever + // a value does not exist. + if o.Type == "" { + o.Type = pilosa.FieldTypeSet + } + if o.CacheType == nil { + o.CacheType = &defaultCacheType + } + if o.CacheSize == nil { + o.CacheSize = &defaultCacheSize + } + if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type set")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type set")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } + case pilosa.FieldTypeInt: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int")) + } else if o.Min == nil { + return pilosa.NewBadRequestError(errors.New("min is required for field type int")) + } else if o.Max == nil { + return pilosa.NewBadRequestError(errors.New("max is required for field type int")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } + case pilosa.FieldTypeTime: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type time")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type time")) + } else if o.TimeQuantum == nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time")) + } + default: + return errors.Errorf("invalid field type: %s", o.Type) + } + return nil +} + +// handleDeleteField handles DELETE /field request. +func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + resp := successResponse{} + err := h.api.DeleteField(r.Context(), indexName, fieldName) + resp.write(w, err) +} + +// handlePostFieldAttrDiff handles POST /internal/field/attr/diff requests. +func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.api.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type postFieldAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postFieldAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// readQueryRequest parses an query parameters from r. +func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + switch r.Header.Get("Content-Type") { + case "application/x-protobuf": + return h.readProtobufQueryRequest(r) + default: + return h.readURLQueryRequest(r) + } +} + +// readProtobufQueryRequest parses query parameters in protobuf from r. +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + // Slurp the body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + qreq := &pilosa.QueryRequest{} + err = h.api.Serializer.Unmarshal(body, qreq) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling query request") + } + return qreq, nil +} + +// readURLQueryRequest parses query parameters from URL parameters from r. +func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + q := r.URL.Query() + + // Parse query string. + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + query := string(buf) + + // Parse list of shards. + shards, err := parseUint64Slice(q.Get("shards")) + if err != nil { + return nil, errors.New("invalid shard argument") + } + + return &pilosa.QueryRequest{ + Query: query, + Shards: shards, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", + }, nil +} + +// writeQueryResponse writes the response from the executor to w. +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { + if !validHeaderAcceptJSON(r.Header) { + return h.writeProtobufQueryResponse(w, resp) + } + return h.writeJSONQueryResponse(w, resp) +} + +// writeProtobufQueryResponse writes the response from the executor to w as protobuf. +func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + if buf, err := h.api.Serializer.Marshal(resp); err != nil { + return errors.Wrap(err, "marshalling") + } else if _, err := w.Write(buf); err != nil { + return errors.Wrap(err, "writing") + } + return nil +} + +// writeJSONQueryResponse writes the response from the executor to w as JSON. +func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + return json.NewEncoder(w).Encode(resp) +} + +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Get index and field type to determine how to handle the + // import data. + field, err := h.api.Field(r.Context(), indexName, fieldName) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal request based on field type. + if field.Type() == pilosa.FieldTypeInt { + // Field type: Int + // Marshal into request object. + req := &pilosa.ImportValueRequest{} + if err := h.api.Serializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportValue(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } else { + // Field type: Set, Time + // Marshal into request object. + req := &pilosa.ImportRequest{} + if err := h.api.Serializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.Import(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + // Marshal response object. + buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: ""}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) + return + } + + // Write response. + w.Write(buf) +} + +// handleGetExport handles /export requests. +func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { + switch r.Header.Get("Accept") { + case "text/csv": + h.handleGetExportCSV(w, r) + default: + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + } +} + +func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { + // Parse query parameters. + q := r.URL.Query() + index, field := q.Get("index"), q.Get("field") + + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) + if err != nil { + http.Error(w, "invalid shard", http.StatusBadRequest) + return + } + + if err = h.api.ExportCSV(r.Context(), index, field, shard, w); err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + break + case pilosa.ErrClusterDoesNotOwnShard: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } +} + +// handleGetFragmentNodes handles /internal/fragment/nodes requests. +func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + q := r.URL.Query() + index := q.Get("index") + + // Read shard parameter. + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) + if err != nil { + http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) + return + } + + // Retrieve fragment owner nodes. + nodes, err := h.api.ShardNodes(r.Context(), index, shard) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Write to response. + if err := json.NewEncoder(w).Encode(nodes); err != nil { + h.logger.Printf("json write error: %s", err) + } +} + +// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests. +func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { + buf, err := h.api.FragmentBlockData(r.Context(), r.Body) + if err != nil { + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Write response. + w.Header().Set("Content-Type", "application/protobuf") + w.Header().Set("Content-Length", strconv.Itoa(len(buf))) + w.Write(buf) +} + +// handleGetFragmentBlocks handles GET /internal/fragment/blocks requests. +func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + // Read shard parameter. + q := r.URL.Query() + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) + if err != nil { + http.Error(w, "shard required", http.StatusBadRequest) + return + } + + blocks, err := h.api.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard) + if err != nil { + if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ + Blocks: blocks, + }); err != nil { + h.logger.Printf("block response encoding error: %s", err) + } +} + +type getFragmentBlocksResponse struct { + Blocks []pilosa.FragmentBlock `json:"blocks"` +} + +// handleGetVersion handles /version requests. +func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + err := json.NewEncoder(w).Encode(struct { + Version string `json:"version"` + }{ + Version: h.api.Version(), + }) + if err != nil { + h.logger.Printf("write version response error: %s", err) + } +} + +// QueryResult types. +const ( + queryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + queryResultTypeValCount + QueryResultTypeUint64 + queryResultTypeBool +) + +// parseUint64Slice returns a slice of uint64s from a comma-delimited string. +func parseUint64Slice(s string) ([]uint64, error) { + var a []uint64 + for _, str := range strings.Split(s, ",") { + // Ignore blanks. + if str == "" { + continue + } + + // Parse number. + num, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "parsing int") + } + a = append(a, num) + } + return a, nil +} + +// errorString returns the string representation of err. +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) + return + } + + oldNode, newNode, err := h.api.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, + New: newNode, + }); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + ID string `json:"id"` +} + +type setCoordinatorResponse struct { + Old *pilosa.Node `json:"old"` + New *pilosa.Node `json:"new"` +} + +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + removeNode, err := h.api.RemoveNode(req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeNode, + }); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + ID string `json:"id"` +} + +type removeNodeResponse struct { + Remove *pilosa.Node `json:"remove"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + err := h.api.ResizeAbort() + var msg string + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case pilosa.ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` +} + +func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + err := h.api.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + err := h.api.ClusterMessage(r.Context(), r.Body) + if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request + http.Error(w, err.Error(), http.StatusBadRequest) + } + + if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { + h.logger.Printf("response encoding error: %s", err) + } +} + +type defaultClusterMessageResponse struct{} + +func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) + + pipeR, pipeW := io.Pipe() + + err := h.api.GetTranslateData(r.Context(), pipeW, offset) + + if err != nil { + if errors.Cause(err) == pilosa.ErrNotImplemented { + http.Error(w, err.Error(), http.StatusNotImplemented) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Flush header so client can continue. + w.WriteHeader(http.StatusOK) + if w, ok := w.(http.Flusher); ok { + w.Flush() + } + + io.Copy(w, pipeR) +} + +type queryValidationSpec struct { + required []string + args map[string]struct{} +} + +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]struct{}{} + for _, arg := range requiredArgs { + args[arg] = struct{}{} + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, + } +} + +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { + for _, arg := range args { + s.args[arg] = struct{}{} + } + return s +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.Errorf("%s is required", req) + } + } + for k := range query { + if _, ok := s.args[k]; !ok { + return errors.Errorf("%s is not a valid argument", k) + } + } + return nil +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go new file mode 100644 index 000000000..071ac1926 --- /dev/null +++ b/http/handler_internal_test.go @@ -0,0 +1,180 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "bytes" + "encoding/json" + "reflect" + "testing" + + "github.com/pilosa/pilosa" +) + +// Test custom UnmarshalJSON for postIndexRequest object +func TestPostIndexRequestUnmarshalJSON(t *testing.T) { + tests := []struct { + json string + expected postIndexRequest + err string + }{ + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, + {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, + {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, + {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, + } + for _, test := range tests { + actual := &postIndexRequest{} + err := json.Unmarshal([]byte(test.json), actual) + + if err != nil { + if test.err == "" || test.err != err.Error() { + t.Errorf("expected error: %v, but got result: %v", test.err, err) + } + } else { + if test.err != "" { + t.Errorf("expected error: %v, but got no error", test.err) + } + } + + if test.err == "" { + if !reflect.DeepEqual(*actual, test.expected) { + t.Errorf("expected: %v, but got: %v", test.expected, *actual) + } + } + + } +} + +// Test custom UnmarshalJSON for postFieldRequest object +func TestPostFieldRequestUnmarshalJSON(t *testing.T) { + foo := "foo" + tests := []struct { + json string + expected postFieldRequest + err string + }{ + {json: `{"options": {}}`, expected: postFieldRequest{}}, + {json: `{"options": 4}`, err: "json: cannot unmarshal number into Go struct field postFieldRequest.options of type http.fieldOptions"}, + {json: `{"option": {}}`, err: `json: unknown field "option"`}, + {json: `{"options": {"badKey": "test"}}`, err: `json: unknown field "badKey"`}, + {json: `{"options": {"inverseEnabled": true}}`, err: `json: unknown field "inverseEnabled"`}, + {json: `{"options": {"cacheType": "foo"}}`, expected: postFieldRequest{Options: fieldOptions{CacheType: &foo}}}, + {json: `{"options": {"inverse": true, "cacheType": "foo"}}`, err: `json: unknown field "inverse"`}, + } + for i, test := range tests { + actual := &postFieldRequest{} + dec := json.NewDecoder(bytes.NewReader([]byte(test.json))) + dec.DisallowUnknownFields() + err := dec.Decode(actual) + if err != nil { + if test.err == "" || test.err != err.Error() { + t.Errorf("test %d: expected error: %v, but got result: %v", i, test.err, err) + } + } + + if test.err == "" { + if !reflect.DeepEqual(*actual, test.expected) { + t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) + } + } + + } +} + +func stringPtr(s string) *string { + return &s +} + +func int64Ptr(i int64) *int64 { + return &i +} + +// Test fieldOption validation. +func TestFieldOptionValidation(t *testing.T) { + timeQuantum := pilosa.TimeQuantum("YMD") + defaultCacheSize := uint32(pilosa.DefaultCacheSize) + tests := []struct { + json string + expected postFieldRequest + err string + }{ + // FieldType: Set + {json: `{"options": {}}`, expected: postFieldRequest{Options: fieldOptions{ + Type: pilosa.FieldTypeSet, + CacheType: stringPtr(pilosa.DefaultCacheType), + CacheSize: &defaultCacheSize, + }}}, + {json: `{"options": {"type": "set"}}`, expected: postFieldRequest{Options: fieldOptions{ + Type: pilosa.FieldTypeSet, + CacheType: stringPtr(pilosa.DefaultCacheType), + CacheSize: &defaultCacheSize, + }}}, + {json: `{"options": {"type": "set", "cacheType": "lru"}}`, expected: postFieldRequest{Options: fieldOptions{ + Type: pilosa.FieldTypeSet, + CacheType: stringPtr("lru"), + CacheSize: &defaultCacheSize, + }}}, + {json: `{"options": {"type": "set", "min": 0}}`, err: "min does not apply to field type set"}, + {json: `{"options": {"type": "set", "max": 100}}`, err: "max does not apply to field type set"}, + {json: `{"options": {"type": "set", "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type set"}, + + // FieldType: Int + {json: `{"options": {"type": "int"}}`, err: "min is required for field type int"}, + {json: `{"options": {"type": "int", "min": 0}}`, err: "max is required for field type int"}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000}}`, expected: postFieldRequest{Options: fieldOptions{ + Type: pilosa.FieldTypeInt, + Min: int64Ptr(0), + Max: int64Ptr(1000), + }}}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000, "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type int"}, + + // FieldType: Time + {json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"}, + {json: `{"options": {"type": "time", "timeQuantum": "YMD"}}`, expected: postFieldRequest{Options: fieldOptions{ + Type: pilosa.FieldTypeTime, + TimeQuantum: &timeQuantum, + }}}, + {json: `{"options": {"type": "time", "timeQuantum": "YMD", "min": 0}}`, err: "min does not apply to field type time"}, + {json: `{"options": {"type": "time", "timeQuantum": "YMD", "max": 1000}}`, err: "max does not apply to field type time"}, + {json: `{"options": {"type": "time", "timeQuantum": "YMD", "cacheType": "ranked"}}`, err: "cacheType does not apply to field type time"}, + {json: `{"options": {"type": "time", "timeQuantum": "YMD", "cacheSize": 1000}}`, err: "cacheSize does not apply to field type time"}, + } + for i, test := range tests { + actual := &postFieldRequest{} + dec := json.NewDecoder(bytes.NewReader([]byte(test.json))) + dec.DisallowUnknownFields() + err := dec.Decode(actual) + if err != nil { + t.Errorf("test %d: %v", i, err) + } + + // Validate field options. + if err := actual.Options.validate(); err != nil { + if test.err == "" || test.err != err.Error() { + t.Errorf("test %d: expected error: %v, but got result: %v", i, test.err, err) + } + } + + if test.err == "" { + if !reflect.DeepEqual(*actual, test.expected) { + t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) + } + } + + } +} diff --git a/statik/filesystem.go b/http/handler_test.go similarity index 51% rename from statik/filesystem.go rename to http/handler_test.go index e3bf95cb1..49d24ffec 100644 --- a/statik/filesystem.go +++ b/http/handler_test.go @@ -11,27 +11,32 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -//go:generate statik -src=../webui -dest=.. -// -// Package statik contains static assets for the Web UI. `go generate` or -// `make generate-statik` will produce statik.go, which is ignored by git. -package statik + +package http_test import ( - "net/http" + "net" + "testing" "github.com/pilosa/pilosa" - "github.com/rakyll/statik/fs" + "github.com/pilosa/pilosa/http" ) -// Ensure nopFileSystem implements interface. -var _ pilosa.FileSystem = &FileSystem{} - -// FileSystem represents a static FileSystem. -type FileSystem struct{} - -// New is a statik implementation of FileSystem New method. -func (s *FileSystem) New() (http.FileSystem, error) { - return fs.New() +func TestHandlerOptions(t *testing.T) { + _, err := http.NewHandler() + if err == nil { + t.Fatalf("expected error making handler without options, got nil") + } + _, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{})) + if err == nil { + t.Fatalf("expected error making handler without options, got nil") + } + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + _, err = http.NewHandler(http.OptHandlerListener(ln)) + if err == nil { + t.Fatalf("expected error making handler without options, got nil") + } } diff --git a/http/translator.go b/http/translator.go new file mode 100644 index 000000000..d094a8c28 --- /dev/null +++ b/http/translator.go @@ -0,0 +1,87 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/pilosa/pilosa" +) + +// Ensure implementation implements inteface. +var _ pilosa.TranslateStore = (*translateStore)(nil) + +// translateStore represents an implementation of translateStore that +// communicates over HTTP. This is used with the TranslateHandler. +type translateStore struct { + URL string +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore(rawurl string) *translateStore { + return &translateStore{URL: rawurl} +} + +// TranslateColumnsToUint64 is not currently implemented. +func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateColumnToString is not currently implemented. +func (s *translateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// TranslateRowsToUint64 is not currently implemented. +func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateRowToString is not currently implemented. +func (s *translateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// Reader returns a reader that can stream data from a remote store. +func (s *translateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + // Generate remote URL. + u, err := url.Parse(s.URL) + if err != nil { + return nil, err + } + u.Path = "/internal/translate/data" + u.RawQuery = (url.Values{ + "offset": {strconv.FormatInt(off, 10)}, + }).Encode() + + // Connect a stream to the remote server. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + // Connect a stream to the remote server. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err) + } + + // Handle error codes or return body as stream. + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusNotImplemented: + resp.Body.Close() + return nil, pilosa.ErrNotImplemented + default: + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body)) + } +} diff --git a/http/translator_test.go b/http/translator_test.go new file mode 100644 index 000000000..04531dab5 --- /dev/null +++ b/http/translator_test.go @@ -0,0 +1,163 @@ +package http_test + +import ( + "context" + "io" + "io/ioutil" + gohttp "net/http" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/mock" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/test" +) + +func newMockReadCloser() *mock.ReadCloser { + return &mock.ReadCloser{ + ReadFunc: func(p []byte) (int, error) { + return 0, io.EOF + }, + CloseFunc: func() error { + return nil + }, + } +} + +func TestTranslateStore_Reader(t *testing.T) { + // Ensure client can connect and stream the translate store data. + t.Run("OK", func(t *testing.T) { + t.Run("ServerDisconnect", func(t *testing.T) { + var mrc mock.ReadCloser + var readN int + mrc.ReadFunc = func(p []byte) (int, error) { + readN++ + switch readN { + case 1: + copy(p, []byte("foo")) + return 3, nil + case 2: + copy(p, []byte("barbaz")) + return 6, nil + case 3: + return 0, io.EOF + default: + t.Fatal("unexpected read") + return 0, nil + } + } + closeInvoked := make(chan struct{}) + mrc.CloseFunc = func() error { + close(closeInvoked) + return nil + } + + // Setup handler on test server. + var translateStore mock.TranslateStore + + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + // Check context to make sure this is the call we are looking for. + // (Something else calls ReaderFunc on server startup) + if ctx.Value(gohttp.ServerContextKey) != nil { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil + } + return newMockReadCloser(), nil + } + + opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + + defer main.Close() + + // Connect to server and stream all available data. + store := http.NewTranslateStore(main.URL()) + + rc, err := store.Reader(context.Background(), 100) + if err != nil { + t.Fatal(err) + } else if data, err := ioutil.ReadAll(rc); err != nil { + t.Fatal(err) + } else if string(data) != `foobarbaz` { + t.Fatalf("unexpected data: %q", data) + } else if err := rc.Close(); err != nil { + t.Fatal(err) + } + + select { + case <-time.NewTimer(time.Millisecond * 100).C: + t.Fatal("expected server close") + case <-closeInvoked: + return + } + }) + + // Ensure server closes store reader if client disconnects. + t.Run("ClientDisconnect", func(t *testing.T) { + // Setup mock so that Read() hangs. + done := make(chan struct{}) + + var mrc mock.ReadCloser + mrc.ReadFunc = func(p []byte) (int, error) { + <-done + return 0, io.EOF + } + + closeInvoked := make(chan struct{}) + + mrc.CloseFunc = func() error { + close(closeInvoked) + return nil + } + + var translateStore mock.TranslateStore + + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return &mrc, nil + } + + opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + + defer main.Close() + defer close(done) + + // Connect to server and begin streaming. + ctx, cancel := context.WithCancel(context.Background()) + store := http.NewTranslateStore(main.URL()) + if _, err := store.Reader(ctx, 0); err != nil { + t.Fatal(err) + } + + // Cancel the context and check if server is closed. + cancel() + select { + case <-time.NewTimer(time.Millisecond * 100).C: + t.Fatal("expected server close") + case <-closeInvoked: + return + } + }) + }) + + // Ensure client is notified if the server doesn't support streaming replication. + t.Run("ErrNotImplemented", func(t *testing.T) { + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return nil, pilosa.ErrNotImplemented + } + + opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + defer main.Close() + + _, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0) + if err != pilosa.ErrNotImplemented { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/index.go b/index.go index 534930cd5..1cda5e7c9 100644 --- a/index.go +++ b/index.go @@ -28,32 +28,33 @@ import ( "github.com/pkg/errors" ) -// Index represents a container for frames. +// Index represents a container for fields. type Index struct { mu sync.RWMutex path string name string + keys bool // use string keys - // Frames by name. - frames map[string]*Frame + // Fields by name. + fields map[string]*Field - // Max Slice on any node in the cluster, according to this node. - remoteMaxSlice uint64 + // Max shard on any node in the cluster, according to this node. + remoteMaxShard uint64 - NewAttrStore func(string) AttrStore + newAttrStore func(string) AttrStore // Column attribute storage and cache. - columnAttrStore AttrStore + columnAttrs AttrStore - broadcaster Broadcaster + broadcaster broadcaster Stats StatsClient - Logger Logger + logger Logger } // NewIndex returns a new instance of Index. func NewIndex(path, name string) (*Index, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } @@ -61,16 +62,16 @@ func NewIndex(path, name string) (*Index, error) { return &Index{ path: path, name: name, - frames: make(map[string]*Frame), + fields: make(map[string]*Field), - remoteMaxSlice: 0, + remoteMaxShard: 0, - NewAttrStore: NewNopAttrStore, - columnAttrStore: NopAttrStore, + newAttrStore: newNopAttrStore, + columnAttrs: nopStore, broadcaster: NopBroadcaster, Stats: NopStatsClient, - Logger: NopLogger, + logger: NopLogger, }, nil } @@ -80,8 +81,11 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// Keys returns true if the index uses string keys. +func (i *Index) Keys() bool { return i.keys } + // ColumnAttrStore returns the storage for column attributes. -func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } +func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs } // Options returns all options for this index. func (i *Index) Options() IndexOptions { @@ -106,19 +110,19 @@ func (i *Index) Open() error { return errors.Wrap(err, "loading meta file") } - if err := i.openFrames(); err != nil { - return errors.Wrap(err, "opening frames") + if err := i.openFields(); err != nil { + return errors.Wrap(err, "opening fields") } - if err := i.columnAttrStore.Open(); err != nil { + if err := i.columnAttrs.Open(); err != nil { return errors.Wrap(err, "opening attrstore") } return nil } -// openFrames opens and initializes the frames inside the index. -func (i *Index) openFrames() error { +// openFields opens and initializes the fields inside the index. +func (i *Index) openFields() error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -135,14 +139,14 @@ func (i *Index) openFrames() error { continue } - fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { return ErrName } - if err := fr.Open(); err != nil { - return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) + if err := fld.Open(); err != nil { + return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } - i.frames[fr.Name()] = fr + i.fields[fld.Name()] = fld } return nil } @@ -164,18 +168,17 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. + i.keys = pb.Keys return nil } -// NOTE: Until we introduce new attributes to store in the index .meta file, -// we don't need to actually write the file. The code related to index.options -// and the index meta file are left in place for future use. -/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{}) + buf, err := proto.Marshal(&internal.IndexMeta{ + Keys: i.keys, + }) if err != nil { return errors.Wrap(err, "marshalling") } @@ -187,201 +190,209 @@ func (i *Index) saveMeta() error { return nil } -*/ -// Close closes the index and its frames. +// Close closes the index and its fields. func (i *Index) Close() error { i.mu.Lock() defer i.mu.Unlock() // Close the attribute store. - i.columnAttrStore.Close() + i.columnAttrs.Close() - // Close all frames. - for _, f := range i.frames { + // Close all fields. + for _, f := range i.fields { if err := f.Close(); err != nil { - return errors.Wrap(err, "closing frame") + return errors.Wrap(err, "closing field") } } - i.frames = make(map[string]*Frame) + i.fields = make(map[string]*Field) return nil } -// MaxSlice returns the max slice in the index according to this node. -func (i *Index) MaxSlice() uint64 { +// maxShard returns the max shard in the index according to this node. +func (i *Index) maxShard() uint64 { if i == nil { return 0 } i.mu.RLock() defer i.mu.RUnlock() - max := i.remoteMaxSlice - for _, f := range i.frames { - if slice := f.MaxSlice(); slice > max { - max = slice + max := i.remoteMaxShard + for _, f := range i.fields { + if shard := f.maxShard(); shard > max { + max = shard } } - i.Stats.Gauge("maxSlice", float64(max), 1.0) + i.Stats.Gauge("maxShard", float64(max), 1.0) return max } -// SetRemoteMaxSlice sets the remote max slice value received from another node. -func (i *Index) SetRemoteMaxSlice(newmax uint64) { +// setRemoteMaxShard sets the remote max shard value received from another node. +func (i *Index) setRemoteMaxShard(newmax uint64) { i.mu.Lock() defer i.mu.Unlock() - i.remoteMaxSlice = newmax + i.remoteMaxShard = newmax } -// FramePath returns the path to a frame in the index. -func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } +// fieldPath returns the path to a field in the index. +func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) } -// Frame returns a frame in the index by name. -func (i *Index) Frame(name string) *Frame { +// Field returns a field in the index by name. +func (i *Index) Field(name string) *Field { i.mu.RLock() defer i.mu.RUnlock() - return i.frame(name) + return i.field(name) } -func (i *Index) frame(name string) *Frame { return i.frames[name] } +func (i *Index) field(name string) *Field { return i.fields[name] } -// Frames returns a list of all frames in the index. -func (i *Index) Frames() []*Frame { +// Fields returns a list of all fields in the index. +func (i *Index) Fields() []*Field { i.mu.RLock() defer i.mu.RUnlock() - a := make([]*Frame, 0, len(i.frames)) - for _, f := range i.frames { + a := make([]*Field, 0, len(i.fields)) + for _, f := range i.fields { a = append(a, f) } - sort.Sort(frameSlice(a)) + sort.Sort(fieldSlice(a)) return a } -// RecalculateCaches recalculates caches on every frame in the index. -func (i *Index) RecalculateCaches() { - for _, frame := range i.Frames() { - frame.RecalculateCaches() +// recalculateCaches recalculates caches on every field in the index. +func (i *Index) recalculateCaches() { + for _, field := range i.Fields() { + field.recalculateCaches() } } -// CreateFrame creates a frame. -func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { +// CreateField creates a field. +func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() - // Ensure frame doesn't already exist. - if i.frames[name] != nil { - return nil, ErrFrameExists - } - return i.createFrame(name, opt) -} - -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { - i.mu.Lock() - defer i.mu.Unlock() - - // Find frame in cache first. - if f := i.frames[name]; f != nil { - return f, nil + // Ensure field doesn't already exist. + if i.fields[name] != nil { + return nil, newConflictError(ErrFieldExists) } - return i.createFrame(name, opt) -} - -func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { - if name == "" { - return nil, errors.New("frame name required") - } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { - return nil, ErrInvalidCacheType - } - - // Validate fields. - for _, field := range opt.Fields { - if err := ValidateField(field); err != nil { - return nil, err + // Apply functional options. + fo := FieldOptions{} + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") } } - // Initialize frame. - f, err := i.newFrame(i.FramePath(name), name) + return i.createField(name, fo) +} + +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Find field in cache first. + if f := i.fields[name]; f != nil { + return f, nil + } + + // Apply functional option. + fo := FieldOptions{} + err := opts(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + + return i.createField(name, fo) +} + +func (i *Index) createFieldIfNotExists(name string, opt FieldOptions) (*Field, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Find field in cache first. + if f := i.fields[name]; f != nil { + return f, nil + } + + return i.createField(name, opt) +} + +func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { + if name == "" { + return nil, errors.New("field name required") + } else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) { + return nil, ErrInvalidCacheType + } + + // Initialize field. + f, err := i.newField(i.fieldPath(name), name) if err != nil { return nil, errors.Wrap(err, "initializing") } - // Open frame. + // Open field. if err := f.Open(); err != nil { return nil, errors.Wrap(err, "opening") } - // Set the time quantum. - if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + // Apply field options. + if err := f.applyOptions(opt); err != nil { f.Close() - return nil, errors.Wrap(err, "setting time quantum") + return nil, errors.Wrap(err, "applying options") } - // Set cache type. - if opt.CacheType == "" { - opt.CacheType = DefaultCacheType - } - f.cacheType = opt.CacheType - - if opt.CacheSize != 0 { - f.cacheSize = opt.CacheSize - } - - // Set fields. - f.fields = opt.Fields - if err := f.saveMeta(); err != nil { f.Close() return nil, errors.Wrap(err, "saving meta") } - // Add to index's frame lookup. - i.frames[name] = f + // Add to index's field lookup. + i.fields[name] = f return f, nil } -func (i *Index) newFrame(path, name string) (*Frame, error) { - f, err := NewFrame(path, i.name, name) +func (i *Index) newField(path, name string) (*Field, error) { + f, err := NewField(path, i.name, name, OptFieldTypeDefault()) if err != nil { return nil, err } - f.Logger = i.Logger - f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.logger = i.logger + f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster - f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) + f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) return f, nil } -// DeleteFrame removes a frame from the index. -func (i *Index) DeleteFrame(name string) error { +// DeleteField removes a field from the index. +func (i *Index) DeleteField(name string) error { i.mu.Lock() defer i.mu.Unlock() - // Ignore if frame doesn't exist. - f := i.frame(name) + // Confirm field exists. + f := i.field(name) if f == nil { - return nil + return newNotFoundError(ErrFieldNotFound) } - // Close frame. + // Close field. if err := f.Close(); err != nil { return errors.Wrap(err, "closing") } - // Delete frame directory. - if err := os.RemoveAll(i.FramePath(name)); err != nil { + // Delete field directory. + if err := os.RemoveAll(i.fieldPath(name)); err != nil { return errors.Wrap(err, "removing directory") } // Remove reference. - delete(i.frames, name) + delete(i.fields, name) return nil } @@ -394,8 +405,9 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { - Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` + Name string `json:"name"` + options IndexOptions `json:"options"` + Fields []*FieldInfo `json:"fields"` } type indexInfoSlice []*IndexInfo @@ -404,29 +416,9 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// EncodeIndexes converts a into its internal representation. -func EncodeIndexes(a []*Index) []*internal.Index { - other := make([]*internal.Index, len(a)) - for i := range a { - other[i] = encodeIndex(a[i]) - } - return other -} - -// encodeIndex converts d into its internal representation. -func encodeIndex(d *Index) *internal.Index { - return &internal.Index{ - Name: d.name, - Frames: encodeFrames(d.Frames()), - } -} - // IndexOptions represents options to set when initializing an index. -type IndexOptions struct{} - -// Encode converts i into its internal representation. -func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{} +type IndexOptions struct { + Keys bool `json:"keys"` } // hasTime returns true if a contains a non-nil time. @@ -441,7 +433,7 @@ func hasTime(a []*time.Time) bool { type importKey struct { View string - Slice uint64 + Shard uint64 } type importData struct { diff --git a/filesystem.go b/index_internal_test.go similarity index 51% rename from filesystem.go rename to index_internal_test.go index 5664f0987..1e6d592ab 100644 --- a/filesystem.go +++ b/index_internal_test.go @@ -15,28 +15,32 @@ package pilosa import ( - "fmt" - "net/http" + "io/ioutil" ) -// Ensure nopFileSystem implements interface. -var _ FileSystem = &nopFileSystem{} - -// FileSystem represents an interface for a WebUI file system. -type FileSystem interface { - New() (http.FileSystem, error) +// mustOpenIndex returns a new, opened index at a temporary path. Panic on error. +func mustOpenIndex() *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := NewIndex(path, "i") + if err != nil { + panic(err) + } + if err := index.Open(); err != nil { + panic(err) + } + return index } -func init() { - NopFileSystem = &nopFileSystem{} -} - -// NopFileSystem represents a FileSystem that returns an error if called. -var NopFileSystem FileSystem - -type nopFileSystem struct{} - -// New is a no-op implementation of FileSystem New method. -func (n *nopFileSystem) New() (http.FileSystem, error) { - return nil, fmt.Errorf("file system not implemented") +// reopen closes the index and reopens it. +func (i *Index) reopen() error { + if err := i.Close(); err != nil { + return err + } + if err := i.Open(); err != nil { + return err + } + return nil } diff --git a/index_test.go b/index_test.go index 0a42804e2..bc412646a 100644 --- a/index_test.go +++ b/index_test.go @@ -21,183 +21,181 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" + "github.com/pkg/errors" ) -// Ensure index can open and retrieve a frame. -func TestIndex_CreateFrameIfNotExists(t *testing.T) { +// ShardWidth is a helper reference to use when testing. +const ShardWidth = pilosa.ShardWidth + +// Ensure index can open and retrieve a field. +func TestIndex_CreateFieldIfNotExists(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + // Create field. + f, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } else if f == nil { - t.Fatal("expected frame") + t.Fatal("expected field") } - // Retrieve existing frame. - other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + // Retrieve existing field. + other, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) - } else if f.Frame != other.Frame { - t.Fatal("frame mismatch") + } else if f.Field != other.Field { + t.Fatal("field mismatch") } - if f.Frame != index.Frame("f") { - t.Fatal("frame mismatch") + if f.Field != index.Field("f") { + t.Fatal("field mismatch") } } -func TestIndex_CreateFrame(t *testing.T) { - // Ensure time quantum can be set appropriately on a new frame. +func TestIndex_CreateField(t *testing.T) { + // Ensure time quantum can be set appropriately on a new field. t.Run("TimeQuantum", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with explicit quantum. - f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) + // Create field with explicit quantum. + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected frame time quantum: %s", q) + t.Fatalf("unexpected field time quantum: %s", q) } }) }) - // Ensure frame can include range columns. + // Ensure field can include range columns. t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with schema and verify it exists. - if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, - }); err != nil { + // Create field with schema and verify it exists. + if f, err := index.CreateField("f", pilosa.OptFieldTypeInt(10, 20)); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields: %#v", f.Fields()) + } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { + t.Fatalf("unexpected type: %#v", f.Type()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields after reopen: %#v", f.Fields()) + } else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { + t.Fatalf("unexpected type after reopen: %#v", f.Type()) } }) - t.Run("ErrRangeCacheAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + // TODO: These errors don't apply here. Instead, we need these tests + // on field creation FieldOptions validation. + /* + t.Run("ErrRangeCacheAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeRanked, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + CacheType: pilosa.CacheTypeRanked, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), - }); err != nil { - t.Fatal(err) - } - }) + t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + if _, err := index.CreateField("f", pilosa.FieldOptions{ + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFieldFieldsAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt}, - }, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFieldNameRequired", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "", Type: pilosa.FieldTypeInt}, - }, - }); err != pilosa.ErrFieldNameRequired { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldType", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: "bad_type"}, - }, - }); err != pilosa.ErrInvalidFieldType { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, - }, - }); err != pilosa.ErrInvalidFieldRange { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidBSIGroupRange { + t.Fatal(err) + } + }) + */ }) } -// Ensure index can delete a frame. -func TestIndex_DeleteFrame(t *testing.T) { +// Ensure index can delete a field. +func TestIndex_DeleteField(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - // Delete frame & verify it's gone. - if err := index.DeleteFrame("f"); err != nil { + // Delete field & verify it's gone. + if err := index.DeleteField("f"); err != nil { t.Fatal(err) - } else if index.Frame("f") != nil { - t.Fatal("expected nil frame") + } else if index.Field("f") != nil { + t.Fatal("expected nil field") } - // Delete again to make sure it doesn't error. - if err := index.DeleteFrame("f"); err != nil { - t.Fatal(err) + // Delete again to make sure it errors. + err := index.DeleteField("f") + if !isNotFoundError(err) { + t.Fatalf("expected 'field not found' error, got: %#v", err) } } -// Ensure index can delete a frame. +// Ensure index can validate its name. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { @@ -211,3 +209,9 @@ func TestIndex_InvalidName(t *testing.T) { t.Fatalf("unexpected index name %v", index) } } + +func isNotFoundError(err error) bool { + root := errors.Cause(err) + _, ok := root.(pilosa.NotFoundError) + return ok +} diff --git a/inmem/translator.go b/inmem/translator.go new file mode 100644 index 000000000..4e1e96b55 --- /dev/null +++ b/inmem/translator.go @@ -0,0 +1,215 @@ +package inmem + +import ( + "context" + "io" + "sync" + + "github.com/pilosa/pilosa" +) + +// Ensure type implements interface. +var _ pilosa.TranslateStore = &translateStore{} + +// translateStore is an in-memory storage engine for translating string-to-uint64 values. +type translateStore struct { + mu sync.RWMutex + + cols map[string]*translateIndex + rows map[frameKey]*translateIndex +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore() *translateStore { + return &translateStore{ + cols: make(map[string]*translateIndex), + rows: make(map[frameKey]*translateIndex), + } +} + +// Reader returns an error because it is not supported by the inmem store. +func (s *translateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + return nil, pilosa.ErrReplicationNotSupported +} + +// TranslateColumnsToUint64 converts value to a uint64 id. +// If value does not have an associated id then one is created. +func (s *translateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.cols[index] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *translateStore) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *translateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.rows[key] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +func (s *translateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +type frameKey struct { + index string + frame string +} + +type translateIndex struct { + seq uint64 + lookup map[string]uint64 + reverse map[uint64]string +} + +func newTranslateIndex() *translateIndex { + return &translateIndex{ + lookup: make(map[string]uint64), + reverse: make(map[uint64]string), + } +} diff --git a/inmem/translator_test.go b/inmem/translator_test.go new file mode 100644 index 000000000..d4d232566 --- /dev/null +++ b/inmem/translator_test.go @@ -0,0 +1,132 @@ +package inmem_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/pilosa/pilosa/inmem" +) + +func TestTranslateStore_TranslateColumn(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func TestTranslateStore_TranslateRow(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/1000) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index 35b452dce..8fdcfa7db 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: private.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -9,20 +10,18 @@ It has these top-level messages: IndexMeta - FrameMeta + FieldOptions ImportResponse BlockDataRequest BlockDataResponse Cache - MaxSlices - CreateSliceMessage + MaxShards + CreateShardMessage DeleteIndexMessage CreateIndexMessage - CreateFrameMessage - DeleteFrameMessage CreateFieldMessage DeleteFieldMessage - Frame + Field Schema Index URI @@ -31,7 +30,7 @@ NodeEventMessage NodeStatus ClusterStatus - Field + BSIGroup CreateViewMessage DeleteViewMessage ResizeInstruction @@ -62,6 +61,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -69,44 +69,75 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -type FrameMeta struct { - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` +func (m *IndexMeta) GetKeys() bool { + if m != nil { + return m.Keys + } + return false } -func (m *FrameMeta) Reset() { *m = FrameMeta{} } -func (m *FrameMeta) String() string { return proto.CompactTextString(m) } -func (*FrameMeta) ProtoMessage() {} -func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +type FieldOptions struct { + Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` +} -func (m *FrameMeta) GetCacheType() string { +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } + +func (m *FieldOptions) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *FieldOptions) GetCacheType() string { if m != nil { return m.CacheType } return "" } -func (m *FrameMeta) GetCacheSize() uint32 { +func (m *FieldOptions) GetCacheSize() uint32 { if m != nil { return m.CacheSize } return 0 } -func (m *FrameMeta) GetTimeQuantum() string { +func (m *FieldOptions) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *FieldOptions) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + +func (m *FieldOptions) GetTimeQuantum() string { if m != nil { return m.TimeQuantum } return "" } -func (m *FrameMeta) GetFields() []*Field { +func (m *FieldOptions) GetKeys() bool { if m != nil { - return m.Fields + return m.Keys } - return nil + return false } type ImportResponse struct { @@ -127,9 +158,9 @@ func (m *ImportResponse) GetErr() string { type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } @@ -145,9 +176,9 @@ func (m *BlockDataRequest) GetIndex() string { return "" } -func (m *BlockDataRequest) GetFrame() string { +func (m *BlockDataRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -159,9 +190,9 @@ func (m *BlockDataRequest) GetView() string { return "" } -func (m *BlockDataRequest) GetSlice() uint64 { +func (m *BlockDataRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -213,42 +244,42 @@ func (m *Cache) GetIDs() []uint64 { return nil } -type MaxSlices struct { +type MaxShards struct { Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxSlices) Reset() { *m = MaxSlices{} } -func (m *MaxSlices) String() string { return proto.CompactTextString(m) } -func (*MaxSlices) ProtoMessage() {} -func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } +func (m *MaxShards) Reset() { *m = MaxShards{} } +func (m *MaxShards) String() string { return proto.CompactTextString(m) } +func (*MaxShards) ProtoMessage() {} +func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } -func (m *MaxSlices) GetStandard() map[string]uint64 { +func (m *MaxShards) GetStandard() map[string]uint64 { if m != nil { return m.Standard } return nil } -type CreateSliceMessage struct { +type CreateShardMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } -func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) } -func (*CreateSliceMessage) ProtoMessage() {} -func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } +func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } +func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } +func (*CreateShardMessage) ProtoMessage() {} +func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } -func (m *CreateSliceMessage) GetIndex() string { +func (m *CreateShardMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *CreateSliceMessage) GetSlice() uint64 { +func (m *CreateShardMessage) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -293,72 +324,16 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } -type CreateFrameMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` -} - -func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } -func (m *CreateFrameMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFrameMessage) ProtoMessage() {} -func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } - -func (m *CreateFrameMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *CreateFrameMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - -func (m *CreateFrameMessage) GetMeta() *FrameMeta { - if m != nil { - return m.Meta - } - return nil -} - -type DeleteFrameMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` -} - -func (m *DeleteFrameMessage) Reset() { *m = DeleteFrameMessage{} } -func (m *DeleteFrameMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFrameMessage) ProtoMessage() {} -func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } - -func (m *DeleteFrameMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *DeleteFrameMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - type CreateFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field *Field `protobuf:"bytes,3,opt,name=Field" json:"Field,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} -func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } func (m *CreateFieldMessage) GetIndex() string { if m != nil { @@ -367,30 +342,29 @@ func (m *CreateFieldMessage) GetIndex() string { return "" } -func (m *CreateFieldMessage) GetFrame() string { +func (m *CreateFieldMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } -func (m *CreateFieldMessage) GetField() *Field { +func (m *CreateFieldMessage) GetMeta() *FieldOptions { if m != nil { - return m.Field + return m.Meta } return nil } type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` } func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} -func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } func (m *DeleteFieldMessage) GetIndex() string { if m != nil { @@ -399,13 +373,6 @@ func (m *DeleteFieldMessage) GetIndex() string { return "" } -func (m *DeleteFieldMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - func (m *DeleteFieldMessage) GetField() string { if m != nil { return m.Field @@ -413,32 +380,32 @@ func (m *DeleteFieldMessage) GetField() string { return "" } -type Frame struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` +type Field struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } -func (m *Frame) Reset() { *m = Frame{} } -func (m *Frame) String() string { return proto.CompactTextString(m) } -func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } -func (m *Frame) GetName() string { +func (m *Field) GetName() string { if m != nil { return m.Name } return "" } -func (m *Frame) GetMeta() *FrameMeta { +func (m *Field) GetMeta() *FieldOptions { if m != nil { return m.Meta } return nil } -func (m *Frame) GetViews() []string { +func (m *Field) GetViews() []string { if m != nil { return m.Views } @@ -452,7 +419,7 @@ type Schema struct { func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} -func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } func (m *Schema) GetIndexes() []*Index { if m != nil { @@ -463,13 +430,13 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` } func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Index) GetName() string { if m != nil { @@ -478,9 +445,9 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetFrames() []*Frame { +func (m *Index) GetFields() []*Field { if m != nil { - return m.Frames + return m.Fields } return nil } @@ -494,7 +461,7 @@ type URI struct { func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *URI) GetScheme() string { if m != nil { @@ -526,7 +493,7 @@ type Node struct { func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *Node) GetID() string { if m != nil { @@ -557,7 +524,7 @@ type NodeStateMessage struct { func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *NodeStateMessage) GetNodeID() string { if m != nil { @@ -581,7 +548,7 @@ type NodeEventMessage struct { func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} -func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeEventMessage) GetEvent() uint32 { if m != nil { @@ -599,14 +566,14 @@ func (m *NodeEventMessage) GetNode() *Node { type NodeStatus struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` + MaxShards *MaxShards `protobuf:"bytes,2,opt,name=MaxShards" json:"MaxShards,omitempty"` Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -615,9 +582,9 @@ func (m *NodeStatus) GetNode() *Node { return nil } -func (m *NodeStatus) GetMaxSlices() *MaxSlices { +func (m *NodeStatus) GetMaxShards() *MaxShards { if m != nil { - return m.MaxSlices + return m.MaxShards } return nil } @@ -638,7 +605,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -661,40 +628,40 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } -type Field struct { +type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *Field) Reset() { *m = Field{} } -func (m *Field) String() string { return proto.CompactTextString(m) } -func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (m *BSIGroup) Reset() { *m = BSIGroup{} } +func (m *BSIGroup) String() string { return proto.CompactTextString(m) } +func (*BSIGroup) ProtoMessage() {} +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } -func (m *Field) GetName() string { +func (m *BSIGroup) GetName() string { if m != nil { return m.Name } return "" } -func (m *Field) GetType() string { +func (m *BSIGroup) GetType() string { if m != nil { return m.Type } return "" } -func (m *Field) GetMin() int64 { +func (m *BSIGroup) GetMin() int64 { if m != nil { return m.Min } return 0 } -func (m *Field) GetMax() int64 { +func (m *BSIGroup) GetMax() int64 { if m != nil { return m.Max } @@ -703,14 +670,14 @@ func (m *Field) GetMax() int64 { type CreateViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -719,9 +686,9 @@ func (m *CreateViewMessage) GetIndex() string { return "" } -func (m *CreateViewMessage) GetFrame() string { +func (m *CreateViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -735,14 +702,14 @@ func (m *CreateViewMessage) GetView() string { type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -751,9 +718,9 @@ func (m *DeleteViewMessage) GetIndex() string { return "" } -func (m *DeleteViewMessage) GetFrame() string { +func (m *DeleteViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -777,7 +744,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -824,15 +791,15 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -848,9 +815,9 @@ func (m *ResizeSource) GetIndex() string { return "" } -func (m *ResizeSource) GetFrame() string { +func (m *ResizeSource) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -862,9 +829,9 @@ func (m *ResizeSource) GetView() string { return "" } -func (m *ResizeSource) GetSlice() uint64 { +func (m *ResizeSource) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -879,7 +846,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{28} + return fileDescriptorPrivate, []int{26} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -910,7 +877,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -926,7 +893,7 @@ type UpdateCoordinatorMessage struct { func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} -func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *UpdateCoordinatorMessage) GetNew() *Node { if m != nil { @@ -943,7 +910,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *Topology) GetClusterID() string { if m != nil { @@ -965,24 +932,22 @@ type RecalculateCaches struct { func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") - proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") + proto.RegisterType((*FieldOptions)(nil), "internal.FieldOptions") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices") - proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") + proto.RegisterType((*MaxShards)(nil), "internal.MaxShards") + proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") - proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") - proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") - proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*URI)(nil), "internal.URI") @@ -991,7 +956,7 @@ func init() { proto.RegisterType((*NodeEventMessage)(nil), "internal.NodeEventMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") - proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*BSIGroup)(nil), "internal.BSIGroup") proto.RegisterType((*CreateViewMessage)(nil), "internal.CreateViewMessage") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") @@ -1017,10 +982,20 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Keys { + dAtA[i] = 0x18 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } -func (m *FrameMeta) Marshal() (dAtA []byte, err error) { +func (m *FieldOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1030,7 +1005,7 @@ func (m *FrameMeta) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { +func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1052,17 +1027,31 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } - if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0x3a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n + if len(m.Type) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x48 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + if m.Keys { + dAtA[i] = 0x58 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } + i++ } return i, nil } @@ -1112,21 +1101,21 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Block != 0 { dAtA[i] = 0x18 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x20 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } if len(m.View) > 0 { dAtA[i] = 0x2a @@ -1224,7 +1213,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *MaxSlices) Marshal() (dAtA []byte, err error) { +func (m *MaxShards) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1234,7 +1223,7 @@ func (m *MaxSlices) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { +func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1258,7 +1247,7 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1268,7 +1257,7 @@ func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1279,10 +1268,10 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -1345,76 +1334,6 @@ func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - if m.Meta != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n8, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n8 - } - return i, nil -} - -func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - return i, nil -} - func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1436,21 +1355,21 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if m.Field != nil { + if m.Meta != nil { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Field.Size())) - n9, err := m.Field.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n8, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n8 } return i, nil } @@ -1476,14 +1395,8 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } if len(m.Field) > 0 { - dAtA[i] = 0x1a + dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) @@ -1491,7 +1404,7 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Frame) Marshal() (dAtA []byte, err error) { +func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1501,7 +1414,7 @@ func (m *Frame) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Frame) MarshalTo(dAtA []byte) (int, error) { +func (m *Field) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1516,11 +1429,11 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) + n9, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n9 } if len(m.Views) > 0 { for _, s := range m.Views { @@ -1591,8 +1504,8 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if len(m.Frames) > 0 { - for _, msg := range m.Frames { + if len(m.Fields) > 0 { + for _, msg := range m.Fields { dAtA[i] = 0x22 i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1666,11 +1579,11 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n11, err := m.URI.MarshalTo(dAtA[i:]) + n10, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n10 } if m.IsCoordinator { dAtA[i] = 0x18 @@ -1739,11 +1652,11 @@ func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n12, err := m.Node.MarshalTo(dAtA[i:]) + n11, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n11 } return i, nil } @@ -1767,31 +1680,31 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n13, err := m.Node.MarshalTo(dAtA[i:]) + n12, err := m.Node.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n12 + } + if m.MaxShards != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxShards.Size())) + n13, err := m.MaxShards.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n13 } - if m.MaxSlices != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n14, err := m.MaxSlices.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n14 - } if m.Schema != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n15, err := m.Schema.MarshalTo(dAtA[i:]) + n14, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n14 } return i, nil } @@ -1838,7 +1751,7 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Field) Marshal() (dAtA []byte, err error) { +func (m *BSIGroup) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1848,7 +1761,7 @@ func (m *Field) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Field) MarshalTo(dAtA []byte) (int, error) { +func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1899,11 +1812,11 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1935,11 +1848,11 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1974,21 +1887,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n16, err := m.Node.MarshalTo(dAtA[i:]) + n15, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n15 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n17, err := m.Coordinator.MarshalTo(dAtA[i:]) + n16, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n16 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2006,21 +1919,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + n17, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n17 } if m.ClusterStatus != nil { dAtA[i] = 0x32 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n18 } return i, nil } @@ -2044,11 +1957,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n20, err := m.Node.MarshalTo(dAtA[i:]) + n19, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n19 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2056,11 +1969,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x22 @@ -2068,10 +1981,10 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x28 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -2100,11 +2013,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n21, err := m.Node.MarshalTo(dAtA[i:]) + n20, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n20 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2134,11 +2047,11 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) + n21, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n21 } return i, nil } @@ -2162,11 +2075,11 @@ func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n23, err := m.New.MarshalTo(dAtA[i:]) + n22, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n22 } return i, nil } @@ -2228,6 +2141,24 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2240,10 +2171,13 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l + if m.Keys { + n += 2 + } return n } -func (m *FrameMeta) Size() (n int) { +func (m *FieldOptions) Size() (n int) { var l int _ = l l = len(m.CacheType) @@ -2257,11 +2191,18 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Fields) > 0 { - for _, e := range m.Fields { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) + } + if m.Keys { + n += 2 } return n } @@ -2283,15 +2224,15 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } if m.Block != 0 { n += 1 + sovPrivate(uint64(m.Block)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } l = len(m.View) if l > 0 { @@ -2333,7 +2274,7 @@ func (m *Cache) Size() (n int) { return n } -func (m *MaxSlices) Size() (n int) { +func (m *MaxShards) Size() (n int) { var l int _ = l if len(m.Standard) > 0 { @@ -2347,15 +2288,15 @@ func (m *MaxSlices) Size() (n int) { return n } -func (m *CreateSliceMessage) Size() (n int) { +func (m *CreateShardMessage) Size() (n int) { var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -2384,38 +2325,6 @@ func (m *CreateIndexMessage) Size() (n int) { return n } -func (m *CreateFrameMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - -func (m *DeleteFrameMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - func (m *CreateFieldMessage) Size() (n int) { var l int _ = l @@ -2423,12 +2332,12 @@ func (m *CreateFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Field != nil { - l = m.Field.Size() + if m.Meta != nil { + l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } return n @@ -2441,10 +2350,6 @@ func (m *DeleteFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -2452,7 +2357,7 @@ func (m *DeleteFieldMessage) Size() (n int) { return n } -func (m *Frame) Size() (n int) { +func (m *Field) Size() (n int) { var l int _ = l l = len(m.Name) @@ -2491,8 +2396,8 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Frames) > 0 { - for _, e := range m.Frames { + if len(m.Fields) > 0 { + for _, e := range m.Fields { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -2568,8 +2473,8 @@ func (m *NodeStatus) Size() (n int) { l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.MaxSlices != nil { - l = m.MaxSlices.Size() + if m.MaxShards != nil { + l = m.MaxShards.Size() n += 1 + l + sovPrivate(uint64(l)) } if m.Schema != nil { @@ -2599,7 +2504,7 @@ func (m *ClusterStatus) Size() (n int) { return n } -func (m *Field) Size() (n int) { +func (m *BSIGroup) Size() (n int) { var l int _ = l l = len(m.Name) @@ -2626,7 +2531,7 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2644,7 +2549,7 @@ func (m *DeleteViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2697,7 +2602,7 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2705,8 +2610,8 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -2812,6 +2717,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2833,7 +2758,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } return nil } -func (m *FrameMeta) Unmarshal(dAtA []byte) error { +func (m *FieldOptions) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2856,10 +2781,10 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: FrameMeta: wiretype end group for non-group") + return fmt.Errorf("proto: FieldOptions: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FieldOptions: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 3: @@ -2939,11 +2864,11 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 7: + case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -2953,23 +2878,79 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex > l { return io.ErrUnexpectedEOF } - m.Fields = append(m.Fields, &Field{}) - if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Type = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3130,7 +3111,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3155,7 +3136,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3178,9 +3159,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3190,7 +3171,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3531,7 +3512,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } return nil } -func (m *MaxSlices) Unmarshal(dAtA []byte) error { +func (m *MaxShards) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3554,10 +3535,10 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group") + return fmt.Errorf("proto: MaxShards: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: MaxShards: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3586,14 +3567,51 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 + if iNdEx < postIndex { + var valuekey uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3603,69 +3621,31 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + valuekey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy } + m.Standard[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.Standard[mapkey] = mapvalue } - m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3688,7 +3668,7 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { +func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3711,10 +3691,10 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateShardMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateShardMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3748,9 +3728,9 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3760,7 +3740,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3977,255 +3957,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CreateFrameMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CreateFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &FrameMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: DeleteFrameMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4286,7 +4017,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -4311,11 +4042,11 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4339,10 +4070,10 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Field == nil { - m.Field = &Field{} + if m.Meta == nil { + m.Meta = &FieldOptions{} } - if err := m.Field.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4426,35 +4157,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } @@ -4504,7 +4206,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *Frame) Unmarshal(dAtA []byte) error { +func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4527,10 +4229,10 @@ func (m *Frame) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Frame: wiretype end group for non-group") + return fmt.Errorf("proto: Field: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4589,7 +4291,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &FrameMeta{} + m.Meta = &FieldOptions{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -4786,7 +4488,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4810,8 +4512,8 @@ func (m *Index) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frames = append(m.Frames, &Frame{}) - if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -5369,7 +5071,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxShards", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5393,10 +5095,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.MaxSlices == nil { - m.MaxSlices = &MaxSlices{} + if m.MaxShards == nil { + m.MaxShards = &MaxShards{} } - if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.MaxShards.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -5593,7 +5295,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } -func (m *Field) Unmarshal(dAtA []byte) error { +func (m *BSIGroup) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -5616,10 +5318,10 @@ func (m *Field) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Field: wiretype end group for non-group") + return fmt.Errorf("proto: BSIGroup: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: BSIGroup: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -5799,7 +5501,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5824,7 +5526,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -5936,7 +5638,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5961,7 +5663,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -6338,7 +6040,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -6363,7 +6065,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -6396,9 +6098,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6408,7 +6110,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6997,70 +6699,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1035 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, - 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, - 0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9, - 0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85, - 0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, - 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce, - 0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40, - 0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1, - 0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b, - 0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90, - 0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c, - 0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c, - 0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64, - 0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c, - 0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68, - 0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e, - 0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc, - 0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d, - 0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f, - 0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa, - 0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9, - 0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12, - 0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2, - 0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1, - 0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1, - 0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4, - 0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce, - 0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09, - 0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b, - 0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99, - 0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f, - 0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb, - 0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88, - 0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39, - 0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56, - 0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b, - 0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4, - 0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07, - 0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6, - 0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3, - 0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a, - 0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1, - 0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8, - 0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd, - 0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf, - 0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3, - 0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c, - 0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d, - 0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8, - 0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69, - 0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08, - 0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33, - 0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac, - 0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d, - 0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b, - 0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34, - 0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19, - 0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53, - 0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f, - 0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96, - 0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31, - 0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58, - 0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f, - 0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00, - 0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00, + // 1027 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0x45, + 0x18, 0x66, 0xbd, 0x6b, 0xc7, 0x7e, 0x53, 0x87, 0x64, 0x0a, 0x61, 0x8b, 0x50, 0x6a, 0x46, 0x95, + 0x1a, 0x7a, 0x88, 0x4a, 0x7b, 0xe1, 0xab, 0x52, 0x14, 0x3b, 0xc0, 0x02, 0x09, 0x30, 0x9b, 0xf4, + 0xd6, 0xc3, 0xd4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0x96, 0xdd, 0xd9, 0x24, 0xee, 0x81, 0x2b, 0x5c, + 0xb8, 0x23, 0x7e, 0x09, 0x3f, 0x81, 0x23, 0x3f, 0x01, 0x85, 0x3f, 0x82, 0xe6, 0x9d, 0xd9, 0x8f, + 0xc4, 0x4e, 0x53, 0x85, 0xde, 0xe6, 0xfd, 0x7e, 0xe6, 0xfd, 0x9a, 0x81, 0x7e, 0x9a, 0x45, 0x27, + 0x5c, 0x89, 0xad, 0x34, 0x93, 0x4a, 0x92, 0x6e, 0x94, 0x28, 0x91, 0x25, 0x3c, 0xa6, 0x77, 0xa1, + 0x17, 0x24, 0x13, 0x71, 0xb6, 0x27, 0x14, 0x27, 0x04, 0xbc, 0x6f, 0xc5, 0x2c, 0xf7, 0xdd, 0x81, + 0xb3, 0xd9, 0x65, 0x78, 0xa6, 0x7f, 0x3a, 0x70, 0xeb, 0xcb, 0x48, 0xc4, 0x93, 0xef, 0x53, 0x15, + 0xc9, 0x24, 0x27, 0x1f, 0x40, 0x6f, 0xc8, 0xc7, 0x47, 0xe2, 0x60, 0x96, 0x0a, 0xd4, 0xec, 0xb1, + 0x9a, 0x51, 0x49, 0xc3, 0xe8, 0xa5, 0xf0, 0xbd, 0x81, 0xb3, 0xd9, 0x67, 0x35, 0x83, 0x0c, 0x60, + 0xf9, 0x20, 0x9a, 0x8a, 0x1f, 0x0b, 0x9e, 0xa8, 0x62, 0xea, 0xb7, 0xd1, 0xba, 0xc9, 0xd2, 0x10, + 0xd0, 0x71, 0x17, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xee, 0x45, 0x89, 0xdf, 0x1b, 0x38, 0x9b, 0x2e, + 0xd3, 0x47, 0xe4, 0xf0, 0x33, 0x1f, 0x2c, 0x87, 0x9f, 0x55, 0xd0, 0x97, 0x1b, 0xd0, 0x29, 0xac, + 0x04, 0xd3, 0x54, 0x66, 0x8a, 0x89, 0x3c, 0x95, 0x49, 0x8e, 0x9e, 0x76, 0xb3, 0xcc, 0x77, 0xd0, + 0xb9, 0x3e, 0xd2, 0x9f, 0x61, 0x75, 0x27, 0x96, 0xe3, 0xe3, 0x11, 0x57, 0x9c, 0x89, 0x9f, 0x0a, + 0x91, 0x2b, 0xf2, 0x0e, 0xb4, 0x31, 0x27, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x79, 0xf0, 0x5b, 0x86, + 0x8b, 0x84, 0xe6, 0xa2, 0x3d, 0x66, 0xc2, 0x63, 0x86, 0xd0, 0xdc, 0xf0, 0x88, 0x67, 0x13, 0xcc, + 0x80, 0xc7, 0x0c, 0xa1, 0x31, 0x3e, 0x8d, 0xc4, 0xa9, 0xbd, 0x36, 0x9e, 0x69, 0x00, 0x6b, 0x8d, + 0xf8, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, 0x59, + 0x0a, 0x93, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x51, 0x0b, 0x45, 0x35, 0x83, 0xde, 0x81, 0x36, 0x66, + 0x5a, 0xdf, 0xb2, 0xb6, 0xd5, 0x47, 0xfa, 0x8b, 0x03, 0xbd, 0x3d, 0x7e, 0x86, 0x30, 0x72, 0xf2, + 0x04, 0xba, 0xa1, 0xe2, 0xc9, 0x44, 0x03, 0xd4, 0x4a, 0xcb, 0x8f, 0x3e, 0xdc, 0x2a, 0x1b, 0x62, + 0xab, 0x52, 0xdb, 0x2a, 0x75, 0x76, 0x13, 0x95, 0xcd, 0x58, 0x65, 0xf2, 0xfe, 0xe7, 0xd0, 0xbf, + 0x20, 0xd2, 0xf1, 0x8e, 0xc5, 0xac, 0xcc, 0xea, 0xb1, 0x98, 0xe9, 0xfb, 0x9f, 0xf0, 0xb8, 0x10, + 0x98, 0x2b, 0x8f, 0x19, 0xe2, 0xb3, 0xd6, 0x27, 0x0e, 0xdd, 0x06, 0x32, 0xcc, 0x04, 0x57, 0x02, + 0x83, 0xec, 0x89, 0x3c, 0xe7, 0x2f, 0xc4, 0xd5, 0x19, 0x37, 0x59, 0x6c, 0x35, 0xb2, 0x48, 0x1f, + 0x00, 0x19, 0x89, 0x58, 0x28, 0x61, 0xfb, 0xf6, 0x15, 0x1e, 0x68, 0x58, 0x46, 0xbb, 0x5e, 0x97, + 0xdc, 0x07, 0x4f, 0x0f, 0x01, 0x06, 0x5b, 0x7e, 0x74, 0xbb, 0xce, 0x48, 0x35, 0x1f, 0x0c, 0x15, + 0x68, 0x5c, 0x3a, 0xc5, 0x0e, 0xb8, 0xf6, 0x0a, 0x0b, 0x9a, 0xe6, 0x81, 0x0d, 0xe5, 0x62, 0xa8, + 0xf5, 0x3a, 0x54, 0x73, 0xd0, 0x6c, 0xb4, 0xed, 0xf2, 0xba, 0x37, 0x8d, 0x46, 0x9f, 0x59, 0xae, + 0xee, 0xbf, 0x7d, 0x3e, 0x15, 0xd6, 0x06, 0xcf, 0x15, 0x94, 0xd6, 0xf5, 0x50, 0xb4, 0x7b, 0xdd, + 0xb3, 0x7a, 0x3f, 0xb8, 0xda, 0x3d, 0x12, 0xf4, 0x31, 0x74, 0xc2, 0xf1, 0x91, 0x98, 0x72, 0xf2, + 0x11, 0x2c, 0x21, 0x0e, 0x91, 0xdb, 0xb6, 0x7a, 0xfb, 0x52, 0x12, 0x59, 0x29, 0xa7, 0x23, 0x8b, + 0x7f, 0x21, 0xa6, 0xfb, 0xd0, 0xc1, 0xe8, 0xb9, 0xef, 0x5d, 0x76, 0x83, 0x7c, 0x66, 0xc5, 0x74, + 0x17, 0xdc, 0x43, 0x16, 0xe8, 0x71, 0x41, 0x04, 0xa5, 0x17, 0x4b, 0x69, 0xdf, 0x5f, 0xcb, 0x5c, + 0xd9, 0x6c, 0xe0, 0x59, 0xf3, 0x7e, 0x90, 0x99, 0xc2, 0xd4, 0xf7, 0x19, 0x9e, 0xe9, 0x33, 0xf0, + 0xf6, 0xe5, 0x44, 0x90, 0x15, 0x68, 0x05, 0x23, 0xeb, 0xa3, 0x15, 0x8c, 0xc8, 0x5d, 0x74, 0x6f, + 0x53, 0xd3, 0xaf, 0x41, 0x1c, 0xb2, 0x80, 0x61, 0xe0, 0x7b, 0xd0, 0x0f, 0xf2, 0xa1, 0x94, 0xd9, + 0x24, 0x4a, 0xb8, 0x92, 0x99, 0x5d, 0x9c, 0x17, 0x99, 0x74, 0x1b, 0x56, 0xb5, 0xfb, 0x50, 0x71, + 0x25, 0xca, 0xfa, 0xad, 0x43, 0x47, 0xf3, 0xaa, 0x70, 0x96, 0xc2, 0x96, 0xd7, 0x7a, 0x65, 0x05, + 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0xbb, 0x27, 0x22, 0x51, 0x8d, 0x0e, 0x40, 0x1a, 0x1d, 0xf4, 0x99, + 0x21, 0x08, 0x35, 0x57, 0xb1, 0x98, 0x57, 0x6a, 0xcc, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, + 0x4a, 0x40, 0x45, 0x5e, 0x99, 0x38, 0x57, 0x9b, 0x90, 0x8f, 0x1b, 0xeb, 0x63, 0x7e, 0x40, 0x2a, + 0x11, 0x6b, 0x2c, 0x99, 0xcd, 0xb2, 0x2d, 0x6c, 0x97, 0xaf, 0xd6, 0xfa, 0x86, 0x6f, 0xcb, 0xc4, + 0x69, 0x04, 0xfd, 0x61, 0x5c, 0xe4, 0x4a, 0x64, 0x16, 0x91, 0x5e, 0x73, 0x86, 0x51, 0xe5, 0xa7, + 0x66, 0x2c, 0x4e, 0x11, 0xb9, 0x07, 0x6d, 0x8d, 0xd4, 0xf4, 0xe6, 0xfc, 0x35, 0x8c, 0x90, 0x3e, + 0x85, 0xee, 0x4e, 0x18, 0x7c, 0x95, 0xc9, 0x22, 0x5d, 0xd8, 0x79, 0xe5, 0xeb, 0xd3, 0x9a, 0x7f, + 0x7d, 0xdc, 0xb9, 0xd7, 0xc7, 0xab, 0x5e, 0x1f, 0x1a, 0xc2, 0x9a, 0x59, 0x09, 0x7a, 0x24, 0x6e, + 0xb2, 0x11, 0xca, 0xa7, 0xc1, 0x6d, 0x3c, 0x0d, 0x21, 0xac, 0x99, 0xc9, 0x7f, 0x93, 0x4e, 0xff, + 0x68, 0xc1, 0x1a, 0x13, 0x79, 0xf4, 0x52, 0x04, 0x49, 0xae, 0xb2, 0x62, 0xac, 0x07, 0x5c, 0xdb, + 0x7f, 0x23, 0x9f, 0xdb, 0x6c, 0xbb, 0xcc, 0x10, 0xaf, 0xd3, 0x4c, 0xe4, 0x21, 0x2c, 0x5f, 0x1e, + 0x80, 0x79, 0xd5, 0xa6, 0x0a, 0x79, 0x08, 0x4b, 0xa1, 0x2c, 0xb2, 0xb1, 0x28, 0xc7, 0xbb, 0xb1, + 0x74, 0x0c, 0x32, 0x23, 0x66, 0xa5, 0x5a, 0xa3, 0x95, 0xda, 0xaf, 0x6e, 0x25, 0xf2, 0xe4, 0x52, + 0x2b, 0xf9, 0x1d, 0x34, 0x78, 0xaf, 0x36, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0xaf, 0x0e, 0xdc, + 0x6a, 0x42, 0x78, 0xad, 0xd9, 0xa8, 0x2a, 0xd2, 0x5a, 0x58, 0x11, 0x77, 0x51, 0x45, 0xbc, 0xba, + 0x22, 0xf5, 0x2b, 0xd7, 0x6e, 0xbe, 0x72, 0xc7, 0x70, 0x67, 0xae, 0x4c, 0x43, 0x39, 0x4d, 0x75, + 0x3f, 0xfc, 0x8f, 0x72, 0xe9, 0xad, 0x91, 0x65, 0xb6, 0x50, 0x3d, 0x66, 0x08, 0xfa, 0x29, 0xbc, + 0x1b, 0x0a, 0xd5, 0x28, 0x52, 0xd9, 0x6d, 0x03, 0x70, 0xf7, 0xc5, 0xe9, 0x15, 0xd7, 0xd7, 0x22, + 0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0x23, 0xeb, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6, + 0xf2, 0xc5, 0xec, 0x9a, 0xa9, 0xf7, 0x61, 0xc9, 0xac, 0x48, 0xf3, 0xf1, 0xe9, 0xb1, 0x92, 0xa4, + 0xb7, 0x75, 0x43, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, 0x43, 0xff, 0x80, 0xf2, 0x9d, 0xd5, 0xbf, + 0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0x7f, 0xce, 0x37, 0x9c, 0xdf, 0xff, 0xdd, 0x78, 0xeb, + 0x79, 0x07, 0x7f, 0xbe, 0x8f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x39, 0x2f, 0x93, 0x68, 0x0a, + 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 52e587f4b..408c065e5 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,13 +3,17 @@ syntax = "proto3"; package internal; message IndexMeta { + bool Keys = 3; } -message FrameMeta { +message FieldOptions { + string Type = 8; string CacheType = 3; uint32 CacheSize = 4; + int64 Min = 9; + int64 Max = 10; string TimeQuantum = 5; - repeated Field Fields = 7; + bool Keys = 11; } message ImportResponse { @@ -18,9 +22,9 @@ message ImportResponse { message BlockDataRequest { string Index = 1; - string Frame = 2; + string Field = 2; string View = 5; - uint64 Slice = 4; + uint64 Shard = 4; uint64 Block = 3; } @@ -33,13 +37,13 @@ message Cache { repeated uint64 IDs = 1; } -message MaxSlices { +message MaxShards { map Standard = 1; } -message CreateSliceMessage { +message CreateShardMessage { string Index = 1; - uint64 Slice = 2; + uint64 Shard = 2; } message DeleteIndexMessage { @@ -51,32 +55,20 @@ message CreateIndexMessage { IndexMeta Meta = 2; } -message CreateFrameMessage { - string Index = 1; - string Frame = 2; - FrameMeta Meta = 3; -} - -message DeleteFrameMessage { - string Index = 1; - string Frame = 2; -} - message CreateFieldMessage { string Index = 1; - string Frame = 2; - Field Field = 3; + string Field = 2; + FieldOptions Meta = 3; } message DeleteFieldMessage { string Index = 1; - string Frame = 2; - string Field = 3; + string Field = 2; } -message Frame { +message Field { string Name = 1; - FrameMeta Meta = 2; + FieldOptions Meta = 2; repeated string Views = 3; } @@ -86,7 +78,7 @@ message Schema { message Index { string Name = 1; - repeated Frame Frames = 4; + repeated Field Fields = 4; } message URI { @@ -113,7 +105,7 @@ message NodeEventMessage { message NodeStatus { Node Node = 1; - MaxSlices MaxSlices = 2; + MaxShards MaxShards = 2; Schema Schema = 3; } @@ -123,7 +115,7 @@ message ClusterStatus { repeated Node Nodes = 3; } -message Field { +message BSIGroup { string Name = 1; string Type = 2; int64 Min = 3; @@ -132,13 +124,13 @@ message Field { message CreateViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } message DeleteViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } @@ -154,9 +146,9 @@ message ResizeInstruction { message ResizeSource { Node Node = 1; string Index = 2; - string Frame = 3; + string Field = 3; string View = 4; - uint64 Slice = 5; + uint64 Shard = 5; } message ResizeInstructionComplete { diff --git a/internal/public.pb.go b/internal/public.pb.go index 0dab2c831..cd1927f06 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: public.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import binary "encoding/binary" - import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -268,7 +267,7 @@ func (m *AttrMap) GetAttrs() []*Attr { type QueryRequest struct { Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` @@ -287,9 +286,9 @@ func (m *QueryRequest) GetQuery() string { return "" } -func (m *QueryRequest) GetSlices() []uint64 { +func (m *QueryRequest) GetShards() []uint64 { if m != nil { - return m.Slices + return m.Shards } return nil } @@ -412,8 +411,8 @@ func (m *QueryResult) GetChanged() bool { type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` @@ -433,16 +432,16 @@ func (m *ImportRequest) GetIndex() string { return "" } -func (m *ImportRequest) GetFrame() string { +func (m *ImportRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } -func (m *ImportRequest) GetSlice() uint64 { +func (m *ImportRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -484,9 +483,8 @@ func (m *ImportRequest) GetTimestamps() []int64 { type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` @@ -504,20 +502,6 @@ func (m *ImportValueRequest) GetIndex() string { return "" } -func (m *ImportValueRequest) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - -func (m *ImportValueRequest) GetSlice() uint64 { - if m != nil { - return m.Slice - } - return 0 -} - func (m *ImportValueRequest) GetField() string { if m != nil { return m.Field @@ -525,6 +509,13 @@ func (m *ImportValueRequest) GetField() string { return "" } +func (m *ImportValueRequest) GetShard() uint64 { + if m != nil { + return m.Shard + } + return 0 +} + func (m *ImportValueRequest) GetColumnIDs() []uint64 { if m != nil { return m.ColumnIDs @@ -808,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i += 8 + i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) } return i, nil } @@ -865,10 +855,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) i += copy(dAtA[i:], m.Query) } - if len(m.Slices) > 0 { - dAtA4 := make([]byte, len(m.Slices)*10) + if len(m.Shards) > 0 { + dAtA4 := make([]byte, len(m.Shards)*10) var j3 int - for _, num := range m.Slices { + for _, num := range m.Shards { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1064,16 +1054,16 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { dAtA8 := make([]byte, len(m.RowIDs)*10) @@ -1181,23 +1171,17 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - if m.Slice != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) - } if len(m.Field) > 0 { - dAtA[i] = 0x22 + dAtA[i] = 0x12 i++ i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } + if m.Shard != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) + } if len(m.ColumnIDs) > 0 { dAtA14 := make([]byte, len(m.ColumnIDs)*10) var j13 int @@ -1251,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1392,9 +1394,9 @@ func (m *QueryRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if len(m.Slices) > 0 { + if len(m.Shards) > 0 { l = 0 - for _, e := range m.Slices { + for _, e := range m.Shards { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1472,12 +1474,12 @@ func (m *ImportRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) } if len(m.RowIDs) > 0 { l = 0 @@ -1522,17 +1524,13 @@ func (m *ImportValueRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPublic(uint64(l)) - } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) - } l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) + } if len(m.ColumnIDs) > 0 { l = 0 for _, e := range m.ColumnIDs { @@ -2335,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + v = uint64(dAtA[iNdEx-8]) + v |= uint64(dAtA[iNdEx-7]) << 8 + v |= uint64(dAtA[iNdEx-6]) << 16 + v |= uint64(dAtA[iNdEx-5]) << 24 + v |= uint64(dAtA[iNdEx-4]) << 32 + v |= uint64(dAtA[iNdEx-3]) << 40 + v |= uint64(dAtA[iNdEx-2]) << 48 + v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex @@ -2515,7 +2520,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -2555,10 +2560,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shards", wireType) } case 3: if wireType != 0 { @@ -3067,7 +3072,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3092,13 +3097,13 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3108,7 +3113,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3437,54 +3442,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPublic - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) - } - m.Slice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } @@ -3513,6 +3470,25 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) + } + m.Shard = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Shard |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } case 5: if wireType == 0 { var v uint64 @@ -3795,50 +3771,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 709 bytes of a gzipped FileDescriptorProto + // 701 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c, - 0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba, - 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4, - 0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd, - 0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64, - 0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8, - 0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82, - 0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69, - 0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f, - 0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, - 0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e, - 0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e, - 0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62, - 0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf, - 0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8, - 0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77, - 0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e, - 0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35, - 0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88, - 0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59, - 0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7, - 0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c, - 0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd, - 0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85, - 0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3, - 0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18, - 0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3, - 0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4, - 0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96, - 0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60, - 0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec, - 0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd, - 0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e, - 0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c, - 0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41, - 0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62, - 0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9, - 0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b, - 0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82, - 0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8, - 0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd, - 0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7, - 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7, - 0xa4, 0x5a, 0x06, 0x00, 0x00, + 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84, + 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b, + 0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0, + 0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6, + 0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5, + 0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8, + 0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41, + 0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69, + 0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b, + 0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93, + 0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1, + 0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3, + 0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c, + 0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7, + 0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22, + 0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb, + 0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49, + 0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75, + 0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88, + 0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c, + 0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d, + 0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97, + 0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd, + 0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44, + 0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58, + 0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd, + 0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1, + 0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85, + 0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2, + 0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6, + 0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b, + 0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85, + 0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84, + 0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64, + 0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66, + 0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, + 0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9, + 0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17, + 0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09, + 0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e, + 0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8, + 0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc, + 0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 9207d3a67..04c98d070 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -46,7 +46,7 @@ message AttrMap { message QueryRequest { string Query = 1; - repeated uint64 Slices = 2; + repeated uint64 Shards = 2; bool ColumnAttrs = 3; bool Remote = 5; bool ExcludeRowAttrs = 6; @@ -70,8 +70,8 @@ message QueryResult { message ImportRequest { string Index = 1; - string Frame = 2; - uint64 Slice = 3; + string Field = 2; + uint64 Shard = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; repeated string RowKeys = 7; @@ -81,9 +81,8 @@ message ImportRequest { message ImportValueRequest { string Index = 1; - string Frame = 2; - uint64 Slice = 3; - string Field = 4; + string Field = 2; + uint64 Shard = 3; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go new file mode 100644 index 000000000..32720cb68 --- /dev/null +++ b/internal/test/querygenerator.go @@ -0,0 +1,190 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} diff --git a/internal/test/querygenerator_test.go b/internal/test/querygenerator_test.go new file mode 100644 index 000000000..7af764602 --- /dev/null +++ b/internal/test/querygenerator_test.go @@ -0,0 +1,219 @@ +package test + +import ( + "testing" + "github.com/pilosa/pilosa/pql" +) + + +func TestPQL_Generator(t *testing.T) { + t.Run("pql.Query generator", func(t *testing.T) { + for _, u := range []struct { + pql string + calc *pql.Query + exp *pql.Query + }{ + { + pql: "Union(Row(aaa=10),Row(bbb=9))", + calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Row(aaa=10),Row(bbb=9))", + calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Difference(Row(aaa=10),Row(bbb=9))", + calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Difference", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Range(bbb > 20)", + calc: PQL(Gt("bbb", 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.GT, + "Value": 20, + }, + }, + }, + }, + }, + { + pql: "Range(10 < bbb < 20)", + calc: PQL(Between("bbb", 10, 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.BETWEEN, + "Value": []int{10, 20}, + }, + }, + }, + }, + }, + { + pql: "Set(10, aaa=9)", + calc: PQL(Set(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: "Clear(10, aaa=10)", + calc: PQL(Clear(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Clear", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: `Set(10, aaa=10, "2017-03-02T03:00")`, + calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + "timestamp": "2017-03-02T03:00", + }, + }, + }, + }, + }, + { + pql: `Count(Row(aaa=10))`, + calc: PQL(Count(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Count", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))", + calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 12}, + }, + }, + }, + }, + }, + }, + } { + + if !Compare(u.calc, u.exp) { + t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql) + } + } + }) + +} diff --git a/iterator.go b/iterator.go index 473e862dd..49c6aabff 100644 --- a/iterator.go +++ b/iterator.go @@ -20,37 +20,37 @@ import ( "github.com/pilosa/pilosa/roaring" ) -// Iterator is an interface for looping over row/column pairs. -type Iterator interface { +// iterator is an interface for looping over row/column pairs. +type iterator interface { Seek(rowID, columnID uint64) Next() (rowID, columnID uint64, eof bool) } -// BufIterator wraps an iterator to provide the ability to unread values. -type BufIterator struct { +// bufIterator wraps an iterator to provide the ability to unread values. +type bufIterator struct { buf struct { rowID uint64 columnID uint64 eof bool full bool } - itr Iterator + itr iterator } -// NewBufIterator returns a buffered iterator that wraps itr. -func NewBufIterator(itr Iterator) *BufIterator { - return &BufIterator{itr: itr} +// newBufIterator returns a buffered iterator that wraps itr. +func newBufIterator(itr iterator) *bufIterator { + return &bufIterator{itr: itr} } // Seek moves to the first pair equal to or greater than pseek/bseek. -func (itr *BufIterator) Seek(rowID, columnID uint64) { +func (itr *bufIterator) Seek(rowID, columnID uint64) { itr.buf.full = false itr.itr.Seek(rowID, columnID) } // Next returns the next pair in the row. // If a value has been buffered then it is returned and the buffer is cleared. -func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) { +func (itr *bufIterator) Next() (rowID, columnID uint64, eof bool) { if itr.buf.full { itr.buf.full = false return itr.buf.rowID, itr.buf.columnID, itr.buf.eof @@ -63,7 +63,7 @@ func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) { } // Peek reads the next value but leaves it on the buffer. -func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) { +func (itr *bufIterator) Peek() (rowID, columnID uint64, eof bool) { rowID, columnID, eof = itr.Next() itr.Unread() return @@ -71,25 +71,25 @@ func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) { // Unread pushes previous pair on to the buffer. // Panics if the buffer is already full. -func (itr *BufIterator) Unread() { +func (itr *bufIterator) Unread() { if itr.buf.full { panic("pilosa.BufIterator: buffer full") } itr.buf.full = true } -// LimitIterator wraps an Iterator and limits it to a max column/row pair. -type LimitIterator struct { - itr Iterator +// limitIterator wraps an Iterator and limits it to a max column/row pair. +type limitIterator struct { + itr iterator maxRowID uint64 maxColumnID uint64 eof bool } -// NewLimitIterator returns a new LimitIterator. -func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator { - return &LimitIterator{ +// newLimitIterator returns a new LimitIterator. +func newLimitIterator(itr iterator, maxRowID, maxColumnID uint64) *limitIterator { + return &limitIterator{ itr: itr, maxRowID: maxRowID, maxColumnID: maxColumnID, @@ -97,11 +97,11 @@ func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator } // Seek moves the underlying iterator to a column/row pair. -func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) } +func (itr *limitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) } // Next returns the next row/column ID pair. // If the underlying iterator returns a pair higher than the max then EOF is returned. -func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) { +func (itr *limitIterator) Next() (rowID, columnID uint64, eof bool) { // Always return EOF once it is reached by limit or the underlying iterator. if itr.eof { return 0, 0, true @@ -118,22 +118,22 @@ func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) { return rowID, columnID, false } -// SliceIterator iterates over a pair of row/column ID slices. -type SliceIterator struct { +// sliceIterator iterates over a pair of row/column ID slices. +type sliceIterator struct { rowIDs []uint64 columnIDs []uint64 i, n int } -// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs. +// newSliceIterator returns an iterator to iterate over a set of row/column ID pairs. // Both slices MUST have an equal length. Otherwise the function will panic. -func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator { +func newSliceIterator(rowIDs, columnIDs []uint64) *sliceIterator { if len(columnIDs) != len(rowIDs) { panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs))) } - return &SliceIterator{ + return &sliceIterator{ rowIDs: rowIDs, columnIDs: columnIDs, @@ -143,7 +143,7 @@ func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator { // Seek moves the cursor to a given pair. // If the pair is not found, the iterator seeks to the next pair. -func (itr *SliceIterator) Seek(bseek, pseek uint64) { +func (itr *sliceIterator) Seek(bseek, pseek uint64) { for i := 0; i < itr.n; i++ { rowID := itr.rowIDs[i] columnID := itr.columnIDs[i] @@ -159,7 +159,7 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) { } // Next returns the next row/column ID pair. -func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) { +func (itr *sliceIterator) Next() (rowID, columnID uint64, eof bool) { if itr.i >= itr.n { return 0, 0, true } @@ -171,24 +171,24 @@ func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) { return rowID, columnID, false } -// RoaringIterator converts a roaring.Iterator to output column/row pairs. -type RoaringIterator struct { +// roaringIterator converts a roaring.Iterator to output column/row pairs. +type roaringIterator struct { itr *roaring.Iterator } -// NewRoaringIterator returns a new iterator wrapping itr. -func NewRoaringIterator(itr *roaring.Iterator) *RoaringIterator { - return &RoaringIterator{itr: itr} +// newRoaringIterator returns a new iterator wrapping itr. +func newRoaringIterator(itr *roaring.Iterator) *roaringIterator { + return &roaringIterator{itr: itr} } // Seek moves the cursor to a pair matching bseek/pseek. // If the pair is not found then it moves to the next pair. -func (itr *RoaringIterator) Seek(bseek, pseek uint64) { - itr.itr.Seek((bseek * SliceWidth) + pseek) +func (itr *roaringIterator) Seek(bseek, pseek uint64) { + itr.itr.Seek((bseek * ShardWidth) + pseek) } // Next returns the next column/row ID pair. -func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) { +func (itr *roaringIterator) Next() (rowID, columnID uint64, eof bool) { v, eof := itr.itr.Next() - return v / SliceWidth, v % SliceWidth, eof + return v / ShardWidth, v % ShardWidth, eof } diff --git a/iterator_test.go b/iterator_internal_test.go similarity index 91% rename from iterator_test.go rename to iterator_internal_test.go index 025672f8a..71869afb8 100644 --- a/iterator_test.go +++ b/iterator_internal_test.go @@ -12,19 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "reflect" "testing" - - "github.com/pilosa/pilosa" ) // Ensure slice iterator and iterate over a set of pairs. func TestSliceIterator(t *testing.T) { // Initialize iterator. - itr := pilosa.NewSliceIterator( + itr := newSliceIterator( []uint64{0, 0, 2, 4}, []uint64{0, 1, 0, 10}, ) @@ -48,7 +46,7 @@ func TestSliceIterator(t *testing.T) { // Ensure buffered iterator can unread values on to the buffer. func TestBufIterator(t *testing.T) { - itr := pilosa.NewBufIterator(pilosa.NewSliceIterator( + itr := newBufIterator(newSliceIterator( []uint64{0, 0, 1, 2}, []uint64{1, 3, 0, 100}, )) @@ -77,7 +75,7 @@ func TestBufIterator_DoubleFillPanic(t *testing.T) { func() { defer func() { v = recover() }() - itr := pilosa.NewBufIterator(pilosa.NewSliceIterator(nil, nil)) + itr := newBufIterator(newSliceIterator(nil, nil)) itr.Unread() itr.Unread() }() diff --git a/logger.go b/logger.go index 93b12c3f8..28b35b999 100644 --- a/logger.go +++ b/logger.go @@ -43,46 +43,46 @@ func (n *nopLogger) Printf(format string, v ...interface{}) {} // Debugf is a no-op implementation of the Logger Debugf method. func (n *nopLogger) Debugf(format string, v ...interface{}) {} -// StandardLogger is a basic implementation of pilosa.Logger based on log.Logger. -type StandardLogger struct { +// standardLogger is a basic implementation of pilosa.Logger based on log.Logger. +type standardLogger struct { logger *log.Logger } -func NewStandardLogger(w io.Writer) *StandardLogger { - return &StandardLogger{ +func NewStandardLogger(w io.Writer) *standardLogger { + return &standardLogger{ logger: log.New(w, "", log.LstdFlags), } } -func (s *StandardLogger) Printf(format string, v ...interface{}) { +func (s *standardLogger) Printf(format string, v ...interface{}) { s.logger.Printf(format, v...) } -func (s *StandardLogger) Debugf(format string, v ...interface{}) {} +func (s *standardLogger) Debugf(format string, v ...interface{}) {} -func (s *StandardLogger) Logger() *log.Logger { +func (s *standardLogger) Logger() *log.Logger { return s.logger } -// VerboseLogger is an implementation of pilosa.Logger which includes debug messages. -type VerboseLogger struct { +// verboseLogger is an implementation of pilosa.Logger which includes debug messages. +type verboseLogger struct { logger *log.Logger } -func NewVerboseLogger(w io.Writer) *VerboseLogger { - return &VerboseLogger{ +func NewVerboseLogger(w io.Writer) *verboseLogger { + return &verboseLogger{ logger: log.New(w, "", log.LstdFlags), } } -func (vb *VerboseLogger) Printf(format string, v ...interface{}) { +func (vb *verboseLogger) Printf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Debugf(format string, v ...interface{}) { +func (vb *verboseLogger) Debugf(format string, v ...interface{}) { vb.logger.Printf(format, v...) } -func (vb *VerboseLogger) Logger() *log.Logger { +func (vb *verboseLogger) Logger() *log.Logger { return vb.logger } diff --git a/lru/lru.go b/lru/lru.go index 532cc45e6..59f896f7e 100644 --- a/lru/lru.go +++ b/lru/lru.go @@ -21,9 +21,9 @@ import "container/list" // Cache is an LRU cache. It is not safe for concurrent access. type Cache struct { - // MaxEntries is the maximum number of cache entries before + // maxEntries is the maximum number of cache entries before // an item is evicted. Zero means no limit. - MaxEntries int + maxEntries int // OnEvicted optionally specificies a callback function to be // executed when an entry is purged from the cache. @@ -46,7 +46,7 @@ type entry struct { // that eviction is done by the caller. func New(maxEntries int) *Cache { return &Cache{ - MaxEntries: maxEntries, + maxEntries: maxEntries, ll: list.New(), cache: make(map[interface{}]*list.Element), } @@ -65,8 +65,8 @@ func (c *Cache) Add(key Key, value interface{}) { } ele := c.ll.PushFront(&entry{key, value}) c.cache[key] = ele - if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { - c.RemoveOldest() + if c.maxEntries != 0 && c.ll.Len() > c.maxEntries { + c.removeOldest() } } @@ -82,8 +82,8 @@ func (c *Cache) Get(key Key) (value interface{}, ok bool) { return } -// Remove removes the provided key from the cache. -func (c *Cache) Remove(key Key) { +// remove removes the provided key from the cache. +func (c *Cache) remove(key Key) { if c.cache == nil { return } @@ -92,8 +92,8 @@ func (c *Cache) Remove(key Key) { } } -// RemoveOldest removes the oldest item from the cache. -func (c *Cache) RemoveOldest() { +// removeOldest removes the oldest item from the cache. +func (c *Cache) removeOldest() { if c.cache == nil { return } @@ -120,8 +120,8 @@ func (c *Cache) Len() int { return c.ll.Len() } -// Clear purges all stored items from the cache. -func (c *Cache) Clear() { +// clear purges all stored items from the cache. +func (c *Cache) clear() { if c.OnEvicted != nil { for _, e := range c.cache { kv := e.Value.(*entry) diff --git a/mock/mock.go b/mock/mock.go new file mode 100644 index 000000000..97ebf8641 --- /dev/null +++ b/mock/mock.go @@ -0,0 +1,21 @@ +package mock + +import "sync" + +type ReadCloser struct { + ReadFunc func(p []byte) (int, error) + CloseFunc func() error + once sync.Once +} + +func (rc *ReadCloser) Read(p []byte) (int, error) { + return rc.ReadFunc(p) +} + +func (rc *ReadCloser) Close() error { + var err error = nil + rc.once.Do(func() { + err = rc.CloseFunc() + }) + return err +} diff --git a/mock/translator.go b/mock/translator.go new file mode 100644 index 000000000..186c81894 --- /dev/null +++ b/mock/translator.go @@ -0,0 +1,38 @@ +package mock + +import ( + "context" + "io" + + "github.com/pilosa/pilosa" +) + +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +type TranslateStore struct { + TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) + TranslateColumnToStringFunc func(index string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) +} + +func (s TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return s.TranslateColumnsToUint64Func(index, values) +} + +func (s TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return s.TranslateColumnToStringFunc(index, values) +} + +func (s TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, frame, values) +} + +func (s TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, frame, value) +} + +func (s TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + return s.ReaderFunc(ctx, off) +} diff --git a/pilosa.go b/pilosa.go index 7ddea28eb..45d77a9f3 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,11 +16,7 @@ package pilosa import ( "errors" - "net" "regexp" - "strings" - - "github.com/pilosa/pilosa/internal" ) // System errors. @@ -31,26 +27,26 @@ var ( ErrIndexExists = errors.New("index already exists") ErrIndexNotFound = errors.New("index not found") - // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") + // ErrFieldRequired is returned when no field is specified. + ErrFieldRequired = errors.New("field required") + ErrFieldExists = errors.New("field already exists") + ErrFieldNotFound = errors.New("field not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrBSIGroupNotFound = errors.New("bsigroup not found") + ErrBSIGroupExists = errors.New("bsigroup already exists") + ErrBSIGroupNameRequired = errors.New("bsigroup name required") + ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") + ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") + ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") + ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") + ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. @@ -58,19 +54,26 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") + + ErrNotImplemented = errors.New("not implemented") ) -// ApiMethodNotAllowedError wraps an error value indicating that a particular +// apiMethodNotAllowedError wraps an error value indicating that a particular // API method is not allowed in the current cluster state. -type ApiMethodNotAllowedError struct { +type apiMethodNotAllowedError struct { error } +// newApiMethodNotAllowedError returns err wrapped in an ApiMethodNotAllowedError. +func newApiMethodNotAllowedError(err error) apiMethodNotAllowedError { + return apiMethodNotAllowedError{err} +} + // BadRequestError wraps an error value to signify that a request could not be // read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest // would be returned. @@ -78,56 +81,58 @@ type BadRequestError struct { error } -// Regular expression to validate index and frame names. +// NewBadRequestError returns err wrapped in a BadRequestError. +func NewBadRequestError(err error) BadRequestError { + return BadRequestError{err} +} + +// ConflictError wraps an error value to signify that a conflict with an +// existing resource occurred such that in an HTTP scenario, http.StatusConflict +// would be returned. +type ConflictError struct { + error +} + +// newConflictError returns err wrapped in a ConflictError. +func newConflictError(err error) ConflictError { + return ConflictError{err} +} + +// NotFoundError wraps an error value to signify that a resource was not found +// such that in an HTTP scenario, http.StatusNotFound would be returned. +type NotFoundError struct { + error +} + +// newNotFoundError returns err wrapped in a NotFoundError. +func newNotFoundError(err error) NotFoundError { + return NotFoundError{err} +} + +// Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeColumnAttrSets converts a into its internal representation. -func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { - other := make([]*internal.ColumnAttrSet, len(a)) - for i := range a { - other[i] = encodeColumnAttrSet(a[i]) - } - return other -} - -// encodeColumnAttrSet converts set into its internal representation. -func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { - return &internal.ColumnAttrSet{ - ID: set.ID, - Attrs: encodeAttrs(set.Attrs), - } -} - // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// ValidateName ensures that the name is a valid format. -func ValidateName(name string) error { +// validateName ensures that the name is a valid format. +func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { return ErrName } return nil } -// StringInSlice checks for substring a in the slice. -func StringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -// StringSlicesAreEqual determines if two string slices are equal. -func StringSlicesAreEqual(a, b []string) bool { +// stringSlicesAreEqual determines if two string slices are equal. +func stringSlicesAreEqual(a, b []string) bool { if a == nil && b == nil { return true @@ -150,59 +155,11 @@ func StringSlicesAreEqual(a, b []string) bool { return true } -// SliceDiff returns the difference between two uint64 slices. -func SliceDiff(a, b []uint64) []uint64 { - m := make(map[uint64]uint64) - - for _, y := range b { - m[y]++ - } - - var ret []uint64 - for _, x := range a { - if m[x] > 0 { - m[x]-- - continue - } - ret = append(ret, x) - } - - return ret -} - -// ContainsSubstring checks to see if substring a is contained in any string in the slice. -func ContainsSubstring(a string, list []string) bool { - for _, b := range list { - if strings.Contains(b, a) { - return true - } - } - return false -} - -// HostToIP converts host to an IP4 address based on net.LookupIP(). -func HostToIP(host string) string { - // if host is not an IP addr, check net.LookupIP() - if net.ParseIP(host) == nil { - hosts, err := net.LookupIP(host) - if err != nil { - return host - } - for _, h := range hosts { - // this restricts pilosa to IP4 - if h.To4() != nil { - return h.String() - } - } - } - return host -} - // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { if addr == "" { - return DefaultURI(), nil + return defaultURI(), nil } else { return NewURIFromAddress(addr) } diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go new file mode 100644 index 000000000..139d5c2b7 --- /dev/null +++ b/pilosa_internal_test.go @@ -0,0 +1,43 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "testing" +) + +func TestValidateName(t *testing.T) { + names := []string{ + "a", "ab", "ab1", "b-c", "d_e", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, name := range names { + if validateName(name) != nil { + t.Fatalf("Should be valid index name: %s", name) + } + } +} + +func TestValidateNameInvalid(t *testing.T) { + names := []string{ + "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, name := range names { + if validateName(name) == nil { + t.Fatalf("Should be invalid index name: %s", name) + } + } +} diff --git a/pilosa_test.go b/pilosa_test.go index 41b0d7098..1c98686e5 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -22,54 +22,6 @@ import ( _ "github.com/pilosa/pilosa/test" ) -func TestValidateName(t *testing.T) { - names := []string{ - "a", "ab", "ab1", "b-c", "d_e", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, name := range names { - if pilosa.ValidateName(name) != nil { - t.Fatalf("Should be valid index name: %s", name) - } - } -} - -func TestValidateNameInvalid(t *testing.T) { - names := []string{ - "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, name := range names { - if pilosa.ValidateName(name) == nil { - t.Fatalf("Should be invalid index name: %s", name) - } - } -} - -func TestStringInSlice(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "localhost:10101" - if !pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s in %v", substr, list) - } - substr = "10101" - if pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s not in %v", substr, list) - } -} - -func TestContainsSubstring(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "10101" - if !pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s contained in %v", substr, list) - } - substr = "4000" - if pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s in not contained in %v", substr, list) - } -} - func TestAddressWithDefaults(t *testing.T) { tests := []struct { addr string diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..88af892ad 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -26,6 +26,186 @@ import ( // Query represents a PQL query. type Query struct { Calls []*Call + + lastField string + lastCond Token + inList bool + callStack []*Call + + conditional []string +} + +func (q *Query) startCall(name string) { + newCall := &Call{Name: name} + q.callStack = append(q.callStack, newCall) + + if len(q.callStack) == 1 { + q.Calls = append(q.Calls, newCall) + } else { + calls := q.callStack[len(q.callStack)-2].Children + q.callStack[len(q.callStack)-2].Children = append(calls, newCall) + } +} + +func (q *Query) endCall() { + q.callStack = q.callStack[:len(q.callStack)-1] +} + +func (q *Query) addPosNum(key, value string) { + q.addField(key) + q.addNumVal(value) +} + +func (q *Query) addPosStr(key, value string) { + q.addField(key) + q.addVal(value) +} + +func (q *Query) startConditional() { + q.conditional = make([]string, 0) + call := q.callStack[len(q.callStack)-1] + if call.Args == nil { + call.Args = make(map[string]interface{}) + } +} + +func (q *Query) condAdd(val string) { + q.conditional = append(q.conditional, val) +} + +func (q *Query) endConditional() { + // do stuff + if len(q.conditional) != 5 { + panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional)) + } + low, _ := strconv.ParseInt(q.conditional[0], 10, 64) + field := q.conditional[2] + high, _ := strconv.ParseInt(q.conditional[4], 10, 64) + + if q.conditional[1] == "<" { + low++ + } + if q.conditional[3] == "<=" { + high++ + } + + call := q.callStack[len(q.callStack)-1] + call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} + + q.conditional = nil +} + +func (q *Query) addField(field string) { + if q.lastField != "" { + panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) + } + q.lastField = field + call := q.callStack[len(q.callStack)-1] + if call.Args == nil { + call.Args = make(map[string]interface{}) + } +} + +func (q *Query) addVal(val interface{}) { + if q.lastField == "" { + panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, val) + return + } + if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: val, + } + } else { + call.Args[q.lastField] = val + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addNumVal(val string) { + if q.lastField == "" { + panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val)) + } + var ival interface{} + var err error + if strings.Contains(val, ".") { + ival, err = strconv.ParseFloat(val, 64) + } else { + ival, err = strconv.ParseInt(val, 10, 64) + } + if err != nil { + panic(err) + } + call := q.callStack[len(q.callStack)-1] + if q.inList { + if q.lastCond != ILLEGAL { + list := call.Args[q.lastField].(*Condition).Value.([]interface{}) + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: append(list, ival), + } + } else { + list := call.Args[q.lastField].([]interface{}) + call.Args[q.lastField] = append(list, ival) + } + return + } else if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: ival, + } + } else { + call.Args[q.lastField] = ival + } + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) startList() { + call := q.callStack[len(q.callStack)-1] + if q.lastCond != ILLEGAL { + call.Args[q.lastField] = &Condition{ + Op: q.lastCond, + Value: make([]interface{}, 0), + } + } else { + call.Args[q.lastField] = make([]interface{}, 0) + } + q.inList = true +} + +func (q *Query) endList() { + q.inList = false + q.lastField = "" + q.lastCond = ILLEGAL +} + +func (q *Query) addGT() { + q.lastCond = GT +} +func (q *Query) addLT() { + q.lastCond = LT +} +func (q *Query) addGTE() { + q.lastCond = GTE +} +func (q *Query) addLTE() { + q.lastCond = LTE +} +func (q *Query) addEQ() { + q.lastCond = EQ +} +func (q *Query) addNEQ() { + q.lastCond = NEQ +} +func (q *Query) addBTWN() { + q.lastCond = BETWEEN } // WriteCallN returns the number of mutating calls. @@ -33,7 +213,7 @@ func (q *Query) WriteCallN() int { var n int for _, call := range q.Calls { switch call.Name { - case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs": + case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs": n++ } } @@ -56,6 +236,18 @@ type Call struct { Children []*Call } +// FieldArg determines which key-value pair contains the field and rowID, +// in the case of arguments like Set(colID, field=rowID). +// Returns the field as a string if present, or an error if not. +func (c *Call) FieldArg() (string, error) { + for arg := range c.Args { + if !strings.HasPrefix(arg, "_") { + return arg, nil + } + } + return "", fmt.Errorf("No field argument specified") +} + // UintArg is for reading the value at key from call.Args as a uint64. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a uint64 or an int64 and @@ -76,6 +268,26 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } } +// IntArg is for reading the value at key from call.Args as an int64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a unt64 or an int64 and +// then cast to an int64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) IntArg(key string) (int64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return tval, true, nil + case uint64: + return int64(tval), true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to int64 in Call.IntArg", tval, tval) + } +} + // UintSliceArg reads the value at key from call.Args as a slice of uint64. If // the key is not in Call.Args, the value of the returned bool will be false, // and the error will be nil. If the value is a slice of int64 it will convert @@ -100,8 +312,8 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } -// Keys returns a list of argument keys in sorted order. -func (c *Call) Keys() []string { +// keys returns a list of argument keys in sorted order. +func (c *Call) keys() []string { a := make([]string, 0, len(c.Args)) for k := range c.Args { a = append(a, k) @@ -157,7 +369,7 @@ func (c *Call) String() string { } // Write arguments in key order. - for i, key := range c.Keys() { + for i, key := range c.keys() { if i > 0 { buf.WriteString(", ") } @@ -167,7 +379,7 @@ func (c *Call) String() string { case *Condition: fmt.Fprintf(&buf, "%v %s", key, v.String()) default: - fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v)) + fmt.Fprintf(&buf, "%v=%s", key, formatValue(v)) } } @@ -196,7 +408,7 @@ type Condition struct { // String returns the string representation of the condition. func (cond *Condition) String() string { - return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value)) + return fmt.Sprintf("%s %s", cond.Op.String(), formatValue(cond.Value)) } // IntSliceValue reads cond.Value as a slice of uint64. @@ -224,7 +436,7 @@ func (cond *Condition) IntSliceValue() ([]int64, error) { } } -func FormatValue(v interface{}) string { +func formatValue(v interface{}) string { switch v := v.(type) { case string: return fmt.Sprintf("%q", v) @@ -233,7 +445,7 @@ func FormatValue(v interface{}) string { case []uint64: return fmt.Sprintf("%s", joinUint64Slice(v)) case time.Time: - return fmt.Sprintf("\"%s\"", v.Format(TimeFormat)) + return fmt.Sprintf("\"%s\"", v.Format(timeFormat)) case *Condition: return v.String() default: diff --git a/pql/ast_test.go b/pql/ast_test.go index f5d75e9de..4b130c610 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -33,11 +33,11 @@ func TestCall_String(t *testing.T) { c := &pql.Call{ Name: "Range", Args: map[string]interface{}{ - "frame": "f", + "other": "f", "field0": &pql.Condition{Op: pql.GTE, Value: 10}, }, } - if s := c.String(); s != `Range(field0 >= 10, frame="f")` { + if s := c.String(); s != `Range(field0 >= 10, other="f")` { t.Fatalf("unexpected string: %s", s) } }) diff --git a/pql/parser.go b/pql/parser.go index 3af0cbc9c..611294971 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,24 +15,28 @@ package pql import ( - "fmt" "io" - "strconv" + "io/ioutil" "strings" + + "github.com/pkg/errors" ) -// TimeFormat is the go-style time format used to parse string dates. -const TimeFormat = "2006-01-02T15:04" +// timeFormat is the go-style time format used to parse string dates. +const timeFormat = "2006-01-02T15:04" -// Parser represents a parser for the PQL language. -type Parser struct { - scanner *bufScanner +// parser represents a parser for the PQL language. +type parser struct { + r io.Reader + //scanner *bufScanner + PQL } // NewParser returns a new instance of Parser. -func NewParser(r io.Reader) *Parser { - return &Parser{ - scanner: newBufScanner(r), +func NewParser(r io.Reader) *parser { + return &parser{ + r: r, + // scanner: newBufScanner(r), } } @@ -42,288 +46,19 @@ func ParseString(s string) (*Query, error) { } // Parse parses the next node in the query. -func (p *Parser) Parse() (*Query, error) { - q := &Query{} - for { - call, err := p.parseCall() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - q.Calls = append(q.Calls, call) - } - - // Require at least one call. - if len(q.Calls) == 0 { - return nil, io.ErrUnexpectedEOF - } - - return q, nil -} - -// parseCall parses the next function call. -func (p *Parser) parseCall() (*Call, error) { - var c Call - - // Read call name. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == EOF { - return nil, io.EOF - } else if tok != IDENT { - return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} - } - c.Name = lit - - // Scan opening parenthesis. - if err := p.expect(LPAREN); err != nil { - return nil, err - } - - // Parse children first. - children, err := p.parseChildren() +func (p *parser) Parse() (*Query, error) { + buf, err := ioutil.ReadAll(p.r) if err != nil { - return nil, err + return nil, errors.Wrap(err, "reading buffer to parse") } - c.Children = children - - // If next token is a closing paren then exit. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - return &c, nil - } else if tok == IDENT { - p.unscan(1) - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + p.PQL = PQL{ + Buffer: string(buf), } - - // Parse key/value arguments. - args, err := p.parseArgs() + p.Init() + err = p.PQL.Parse() if err != nil { - return nil, err - } - c.Args = args - - // Scan closing parenthesis. - if err := p.expect(RPAREN); err != nil { - return nil, err - } - - return &c, nil -} - -// parseChildren parses call children. -func (p *Parser) parseChildren() ([]*Call, error) { - var offset int - var children []*Call - for { - // Ensure next two tokens are IDENT+LPAREN. - if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { - p.unscanIgnoreWhitespace(1 + offset) - return children, nil - } - if tok, _, _ := p.scan(); tok != LPAREN { - p.unscanIgnoreWhitespace(2 + offset) - return children, nil - } - - // Push tokens back on scanner and parse as a call. - p.unscan(2) - child, err := p.parseCall() - if err != nil { - return nil, err - } - children = append(children, child) - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return children, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - - // Make sure comma is unscanned. - offset = 1 - } -} - -// parseArgs parses key/value arguments. -func (p *Parser) parseArgs() (map[string]interface{}, error) { - args := make(map[string]interface{}) - for { - // Parse key. - tok, pos, lit := p.scanIgnoreWhitespace() - if tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != IDENT { - return nil, parseErrorf(pos, "expected argument key, found %q", lit) - } - key := lit - - // Expect '=' or a comparison next. - var op Token - switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { - case ASSIGN: - case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: - op = tok - default: - return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) - } - - // Parse value. - var value interface{} - tok, pos, lit = p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - value = true - } else if lit == "false" { - value = false - } else if lit == "null" { - value = nil - } else { - value = lit - } - case STRING: - value = lit - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - value = v - case FLOAT: - v, err := strconv.ParseFloat(lit, 64) - if err != nil { - return nil, err - } - value = v - case LBRACK: - v, err := p.parseList() - if err != nil { - return nil, err - } - value = v - default: - return nil, parseErrorf(pos, "invalid argument value: %q", lit) - } - - // Ensure key doesn't already exist. - if _, ok := args[key]; ok { - return nil, parseErrorf(pos, "argument key already used: %s", key) - } - - // If op is specified then create a condition. - if op != 0 { - value = &Condition{Op: op, Value: value} - } - - // Add key/value pair to arguments. - args[key] = value - - // Exit if closing paren. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { - p.unscan(1) - return args, nil - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) - } - } -} - -// parseList parses a list of primitives. This is used by the TopN() filters. -func (p *Parser) parseList() ([]interface{}, error) { - var values []interface{} - for { - // Read next value. - tok, pos, lit := p.scanIgnoreWhitespace() - switch tok { - case IDENT: - if lit == "true" { - values = append(values, true) - } else if lit == "false" { - values = append(values, false) - } else { - values = append(values, lit) - } - case STRING: - values = append(values, lit) - case INTEGER: - v, err := strconv.ParseInt(lit, 10, 64) - if err != nil { - return nil, err - } - values = append(values, v) - default: - return nil, parseErrorf(pos, "invalid list value: %q", lit) - } - - // Expect a comma or closing bracket next. - if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { - break - } else if tok != COMMA { - return nil, parseErrorf(pos, "expected comma, found %q", lit) - } - } - return values, nil -} - -// scan returns the next token from the scanner. -func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } - -// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. -func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { - tok, pos, lit = p.scan() - if tok == WS { - tok, pos, lit = p.scan() - } - return -} - -// unscan returns the last n tokens back to the scanner. -func (p *Parser) unscan(n int) { - for i := 0; i < n; i++ { - p.scanner.unscan() - } -} - -// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. -func (p *Parser) unscanIgnoreWhitespace(n int) { - for i := 0; i < n; { - p.scanner.unscan() - if tok, _, _ := p.scanner.curr(); tok != WS { - i++ - } - } -} - -// expect returns an error if the next token is not exp. -func (p *Parser) expect(exp Token) error { - if tok, pos, lit := p.scan(); tok != exp { - return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) - } - return nil -} - -// pos returns the current position. -func (p *Parser) pos() Pos { return p.scanner.pos() } - -// ParseError represents an error that occurred while parsing a PQL query. -type ParseError struct { - Message string - Pos Pos -} - -// Error returns a string representation of e. -func (e *ParseError) Error() string { - return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) -} - -// parseErrorf returns a formatted parse error. -func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { - return &ParseError{ - Message: fmt.Sprintf(format, args...), - Pos: pos, + return nil, errors.Wrap(err, "parsing") } + p.Execute() + return &p.Query, nil } diff --git a/pql/parser_test.go b/pql/parser_test.go index 0e2a5c17d..c7a260b92 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) { // Parse with both child calls and arguments. t.Run("ChildrenAndArguments", func(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`) + q, err := pql.ParseString(`TopN(f, Bitmap(id=100, field=other), n=3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -143,9 +143,9 @@ func TestParser_Parse(t *testing.T) { Name: "TopN", Children: []*pql.Call{{ Name: "Bitmap", - Args: map[string]interface{}{"id": int64(100), "frame": "other"}, + Args: map[string]interface{}{"id": int64(100), "field": "other"}, }}, - Args: map[string]interface{}{"n": int64(3), "frame": "f"}, + Args: map[string]interface{}{"n": int64(3), "_field": "f"}, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) @@ -154,15 +154,15 @@ func TestParser_Parse(t *testing.T) { // Parse a list argument. t.Run("ListArgument", func(t *testing.T) { - q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`) + q, err := pql.ParseString(`TopN(f, ids=[0,10,30])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ Name: "TopN", Args: map[string]interface{}{ - "frame": "f", - "ids": []interface{}{int64(0), int64(10), int64(30)}, + "_field": "f", + "ids": []interface{}{int64(0), int64(10), int64(30)}, }, }, ) { diff --git a/pql/pql.peg b/pql/pql.peg new file mode 100644 index 000000000..141d8f265 --- /dev/null +++ b/pql/pql.peg @@ -0,0 +1,74 @@ +package pql + +type PQL Peg { + Query +} + + +Calls <- sp (Call sp)* !. +Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()} + / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} + / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} + / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} + / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} + / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} + / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } +allargs <- Call (comma Call)* (comma args)? / args / sp +args <- arg (comma args)? sp +arg <- ( field sp '=' sp value + / field sp COND sp value + ) +COND <- ( '><' { p.addBTWN() } + / '<=' { p.addLTE() } + / '>=' { p.addGTE() } + / '==' { p.addEQ() } + / '!=' { p.addNEQ() } + / '<' { p.addLT() } + / '>' { p.addGT() } + ) +conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} +condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])} +condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} +condfield <- sp {p.condAdd(buffer[begin:end])} + +timerange <- field sp '=' sp value comma {p.addPosStr("_start", buffer[begin:end])} comma {p.addPosStr("_end", buffer[begin:end])} + +value <- ( item + / lbrack { p.startList() } list rbrack { p.endList() } + ) +list <- item (comma list)? +item <- ( 'null' &(comma / sp close) { p.addVal(nil) } + / 'true' &(comma / sp close) { p.addVal(true) } + / 'false' &(comma / sp close) { p.addVal(false) } + / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } + / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } + / '"' < doublequotedstring > '"' { p.addVal(buffer[begin:end]) } + / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } + ) + +doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* +singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )* + +fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' / '-' )* +field <- { p.addField(buffer[begin:end]) } +reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') +posfield <- { p.addPosStr("_field", buffer[begin:end]) } +uint <- [1-9] [0-9]* / '0' +uintrow <- {p.addPosNum("_row", buffer[begin:end])} +col <- ( {p.addPosNum("_col", buffer[begin:end])} + / '"' '"' {p.addPosStr("_col", buffer[begin:end])} + ) + +open <- '(' sp +close <- ')' sp +sp <- ( ' ' / '\t' / '\n' )* +comma <- sp ',' sp +lbrack <- '[' sp +rbrack <- sp ']' sp +IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* + + +timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] +timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt +timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} diff --git a/pql/pql.peg.go b/pql/pql.peg.go new file mode 100644 index 000000000..2516def6c --- /dev/null +++ b/pql/pql.peg.go @@ -0,0 +1,2818 @@ +package pql + +//go:generate peg -inline pql.peg + +import ( + "fmt" + "math" + "sort" + "strconv" +) + +const endSymbol rune = 1114112 + +/* The rule types inferred from the grammar are below. */ +type pegRule uint8 + +const ( + ruleUnknown pegRule = iota + ruleCalls + ruleCall + ruleallargs + ruleargs + rulearg + ruleCOND + ruleconditional + rulecondint + rulecondLT + rulecondfield + ruletimerange + rulevalue + rulelist + ruleitem + ruledoublequotedstring + rulesinglequotedstring + rulefieldExpr + rulefield + rulereserved + ruleposfield + ruleuint + ruleuintrow + rulecol + ruleopen + ruleclose + rulesp + rulecomma + rulelbrack + rulerbrack + ruleIDENT + ruletimestampbasicfmt + ruletimestampfmt + ruletimestamp + ruleAction0 + ruleAction1 + ruleAction2 + ruleAction3 + ruleAction4 + ruleAction5 + ruleAction6 + ruleAction7 + ruleAction8 + ruleAction9 + ruleAction10 + ruleAction11 + rulePegText + ruleAction12 + ruleAction13 + ruleAction14 + ruleAction15 + ruleAction16 + ruleAction17 + ruleAction18 + ruleAction19 + ruleAction20 + ruleAction21 + ruleAction22 + ruleAction23 + ruleAction24 + ruleAction25 + ruleAction26 + ruleAction27 + ruleAction28 + ruleAction29 + ruleAction30 + ruleAction31 + ruleAction32 + ruleAction33 + ruleAction34 + ruleAction35 + ruleAction36 + ruleAction37 + ruleAction38 + ruleAction39 + ruleAction40 + ruleAction41 + ruleAction42 + ruleAction43 +) + +var rul3s = [...]string{ + "Unknown", + "Calls", + "Call", + "allargs", + "args", + "arg", + "COND", + "conditional", + "condint", + "condLT", + "condfield", + "timerange", + "value", + "list", + "item", + "doublequotedstring", + "singlequotedstring", + "fieldExpr", + "field", + "reserved", + "posfield", + "uint", + "uintrow", + "col", + "open", + "close", + "sp", + "comma", + "lbrack", + "rbrack", + "IDENT", + "timestampbasicfmt", + "timestampfmt", + "timestamp", + "Action0", + "Action1", + "Action2", + "Action3", + "Action4", + "Action5", + "Action6", + "Action7", + "Action8", + "Action9", + "Action10", + "Action11", + "PegText", + "Action12", + "Action13", + "Action14", + "Action15", + "Action16", + "Action17", + "Action18", + "Action19", + "Action20", + "Action21", + "Action22", + "Action23", + "Action24", + "Action25", + "Action26", + "Action27", + "Action28", + "Action29", + "Action30", + "Action31", + "Action32", + "Action33", + "Action34", + "Action35", + "Action36", + "Action37", + "Action38", + "Action39", + "Action40", + "Action41", + "Action42", + "Action43", +} + +type token32 struct { + pegRule + begin, end uint32 +} + +func (t *token32) String() string { + return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) +} + +type node32 struct { + token32 + up, next *node32 +} + +func (node *node32) print(pretty bool, buffer string) { + var print func(node *node32, depth int) + print = func(node *node32, depth int) { + for node != nil { + for c := 0; c < depth; c++ { + fmt.Printf(" ") + } + rule := rul3s[node.pegRule] + quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) + if !pretty { + fmt.Printf("%v %v\n", rule, quote) + } else { + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) + } + if node.up != nil { + print(node.up, depth+1) + } + node = node.next + } + } + print(node, 0) +} + +func (node *node32) Print(buffer string) { + node.print(false, buffer) +} + +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) +} + +type tokens32 struct { + tree []token32 +} + +func (t *tokens32) Trim(length uint32) { + t.tree = t.tree[:length] +} + +func (t *tokens32) Print() { + for _, token := range t.tree { + fmt.Println(token.String()) + } +} + +func (t *tokens32) AST() *node32 { + type element struct { + node *node32 + down *element + } + tokens := t.Tokens() + var stack *element + for _, token := range tokens { + if token.begin == token.end { + continue + } + node := &node32{token32: token} + for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { + stack.node.next = node.up + node.up = stack.node + stack = stack.down + } + stack = &element{node: node, down: stack} + } + if stack != nil { + return stack.node + } + return nil +} + +func (t *tokens32) PrintSyntaxTree(buffer string) { + t.AST().Print(buffer) +} + +func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { + t.AST().PrettyPrint(buffer) +} + +func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, + } +} + +func (t *tokens32) Tokens() []token32 { + return t.tree +} + +type PQL struct { + Query + + Buffer string + buffer []rune + rules [79]func() bool + parse func(rule ...int) error + reset func() + Pretty bool + tokens32 +} + +func (p *PQL) Parse(rule ...int) error { + return p.parse(rule...) +} + +func (p *PQL) Reset() { + p.reset() +} + +type textPosition struct { + line, symbol int +} + +type textPositionMap map[int]textPosition + +func translatePositions(buffer []rune, positions []int) textPositionMap { + length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 + sort.Ints(positions) + +search: + for i, c := range buffer { + if c == '\n' { + line, symbol = line+1, 0 + } else { + symbol++ + } + if i == positions[j] { + translations[positions[j]] = textPosition{line, symbol} + for j++; j < length; j++ { + if i != positions[j] { + continue search + } + } + break search + } + } + + return translations +} + +type parseError struct { + p *PQL + max token32 +} + +func (e *parseError) Error() string { + tokens, error := []token32{e.max}, "\n" + positions, p := make([]int, 2*len(tokens)), 0 + for _, token := range tokens { + positions[p], p = int(token.begin), p+1 + positions[p], p = int(token.end), p+1 + } + translations := translatePositions(e.p.buffer, positions) + format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" + if e.p.Pretty { + format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" + } + for _, token := range tokens { + begin, end := int(token.begin), int(token.end) + error += fmt.Sprintf(format, + rul3s[token.pegRule], + translations[begin].line, translations[begin].symbol, + translations[end].line, translations[end].symbol, + strconv.Quote(string(e.p.buffer[begin:end]))) + } + + return error +} + +func (p *PQL) PrintSyntaxTree() { + if p.Pretty { + p.tokens32.PrettyPrintSyntaxTree(p.Buffer) + } else { + p.tokens32.PrintSyntaxTree(p.Buffer) + } +} + +func (p *PQL) Execute() { + buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 + for _, token := range p.Tokens() { + switch token.pegRule { + + case rulePegText: + begin, end = int(token.begin), int(token.end) + text = string(_buffer[begin:end]) + + case ruleAction0: + p.startCall("Set") + case ruleAction1: + p.endCall() + case ruleAction2: + p.startCall("SetRowAttrs") + case ruleAction3: + p.endCall() + case ruleAction4: + p.startCall("SetColumnAttrs") + case ruleAction5: + p.endCall() + case ruleAction6: + p.startCall("Clear") + case ruleAction7: + p.endCall() + case ruleAction8: + p.startCall("TopN") + case ruleAction9: + p.endCall() + case ruleAction10: + p.startCall("Range") + case ruleAction11: + p.endCall() + case ruleAction12: + p.startCall(buffer[begin:end]) + case ruleAction13: + p.endCall() + case ruleAction14: + p.addBTWN() + case ruleAction15: + p.addLTE() + case ruleAction16: + p.addGTE() + case ruleAction17: + p.addEQ() + case ruleAction18: + p.addNEQ() + case ruleAction19: + p.addLT() + case ruleAction20: + p.addGT() + case ruleAction21: + p.startConditional() + case ruleAction22: + p.endConditional() + case ruleAction23: + p.condAdd(buffer[begin:end]) + case ruleAction24: + p.condAdd(buffer[begin:end]) + case ruleAction25: + p.condAdd(buffer[begin:end]) + case ruleAction26: + p.addPosStr("_start", buffer[begin:end]) + case ruleAction27: + p.addPosStr("_end", buffer[begin:end]) + case ruleAction28: + p.startList() + case ruleAction29: + p.endList() + case ruleAction30: + p.addVal(nil) + case ruleAction31: + p.addVal(true) + case ruleAction32: + p.addVal(false) + case ruleAction33: + p.addNumVal(buffer[begin:end]) + case ruleAction34: + p.addNumVal(buffer[begin:end]) + case ruleAction35: + p.addVal(buffer[begin:end]) + case ruleAction36: + p.addVal(buffer[begin:end]) + case ruleAction37: + p.addVal(buffer[begin:end]) + case ruleAction38: + p.addField(buffer[begin:end]) + case ruleAction39: + p.addPosStr("_field", buffer[begin:end]) + case ruleAction40: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction41: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction42: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction43: + p.addPosStr("_timestamp", buffer[begin:end]) + + } + } + _, _, _, _, _ = buffer, _buffer, text, begin, end +} + +func (p *PQL) Init() { + var ( + max token32 + position, tokenIndex uint32 + buffer []rune + ) + p.reset = func() { + max = token32{} + position, tokenIndex = 0, 0 + + p.buffer = []rune(p.Buffer) + if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { + p.buffer = append(p.buffer, endSymbol) + } + buffer = p.buffer + } + p.reset() + + _rules := p.rules + tree := tokens32{tree: make([]token32, math.MaxInt16)} + p.parse = func(rule ...int) error { + r := 1 + if len(rule) > 0 { + r = rule[0] + } + matches := p.rules[r]() + p.tokens32 = tree + if matches { + p.Trim(tokenIndex) + return nil + } + return &parseError{p, max} + } + + add := func(rule pegRule, begin uint32) { + tree.Add(rule, begin, position, tokenIndex) + tokenIndex++ + if begin != position && position > max.end { + max = token32{rule, begin, position} + } + } + + matchDot := func() bool { + if buffer[position] != endSymbol { + position++ + return true + } + return false + } + + /*matchChar := func(c byte) bool { + if buffer[position] == c { + position++ + return true + } + return false + }*/ + + /*matchRange := func(lower byte, upper byte) bool { + if c := buffer[position]; c >= lower && c <= upper { + position++ + return true + } + return false + }*/ + + _rules = [...]func() bool{ + nil, + /* 0 Calls <- <(sp (Call sp)* !.)> */ + func() bool { + position0, tokenIndex0 := position, tokenIndex + { + position1 := position + if !_rules[rulesp]() { + goto l0 + } + l2: + { + position3, tokenIndex3 := position, tokenIndex + if !_rules[ruleCall]() { + goto l3 + } + if !_rules[rulesp]() { + goto l3 + } + goto l2 + l3: + position, tokenIndex = position3, tokenIndex3 + } + { + position4, tokenIndex4 := position, tokenIndex + if !matchDot() { + goto l4 + } + goto l0 + l4: + position, tokenIndex = position4, tokenIndex4 + } + add(ruleCalls, position1) + } + return true + l0: + position, tokenIndex = position0, tokenIndex0 + return false + }, + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + func() bool { + position5, tokenIndex5 := position, tokenIndex + { + position6 := position + { + position7, tokenIndex7 := position, tokenIndex + if buffer[position] != rune('S') { + goto l8 + } + position++ + if buffer[position] != rune('e') { + goto l8 + } + position++ + if buffer[position] != rune('t') { + goto l8 + } + position++ + { + add(ruleAction0, position) + } + if !_rules[ruleopen]() { + goto l8 + } + if !_rules[rulecol]() { + goto l8 + } + if !_rules[rulecomma]() { + goto l8 + } + if !_rules[ruleargs]() { + goto l8 + } + { + position10, tokenIndex10 := position, tokenIndex + if !_rules[rulecomma]() { + goto l10 + } + { + position12 := position + { + position13 := position + if !_rules[ruletimestampfmt]() { + goto l10 + } + add(rulePegText, position13) + } + { + add(ruleAction43, position) + } + add(ruletimestamp, position12) + } + goto l11 + l10: + position, tokenIndex = position10, tokenIndex10 + } + l11: + if !_rules[ruleclose]() { + goto l8 + } + { + add(ruleAction1, position) + } + goto l7 + l8: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('S') { + goto l16 + } + position++ + if buffer[position] != rune('e') { + goto l16 + } + position++ + if buffer[position] != rune('t') { + goto l16 + } + position++ + if buffer[position] != rune('R') { + goto l16 + } + position++ + if buffer[position] != rune('o') { + goto l16 + } + position++ + if buffer[position] != rune('w') { + goto l16 + } + position++ + if buffer[position] != rune('A') { + goto l16 + } + position++ + if buffer[position] != rune('t') { + goto l16 + } + position++ + if buffer[position] != rune('t') { + goto l16 + } + position++ + if buffer[position] != rune('r') { + goto l16 + } + position++ + if buffer[position] != rune('s') { + goto l16 + } + position++ + { + add(ruleAction2, position) + } + if !_rules[ruleopen]() { + goto l16 + } + if !_rules[ruleposfield]() { + goto l16 + } + if !_rules[rulecomma]() { + goto l16 + } + { + position18 := position + { + position19 := position + if !_rules[ruleuint]() { + goto l16 + } + add(rulePegText, position19) + } + { + add(ruleAction40, position) + } + add(ruleuintrow, position18) + } + if !_rules[rulecomma]() { + goto l16 + } + if !_rules[ruleargs]() { + goto l16 + } + if !_rules[ruleclose]() { + goto l16 + } + { + add(ruleAction3, position) + } + goto l7 + l16: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('S') { + goto l22 + } + position++ + if buffer[position] != rune('e') { + goto l22 + } + position++ + if buffer[position] != rune('t') { + goto l22 + } + position++ + if buffer[position] != rune('C') { + goto l22 + } + position++ + if buffer[position] != rune('o') { + goto l22 + } + position++ + if buffer[position] != rune('l') { + goto l22 + } + position++ + if buffer[position] != rune('u') { + goto l22 + } + position++ + if buffer[position] != rune('m') { + goto l22 + } + position++ + if buffer[position] != rune('n') { + goto l22 + } + position++ + if buffer[position] != rune('A') { + goto l22 + } + position++ + if buffer[position] != rune('t') { + goto l22 + } + position++ + if buffer[position] != rune('t') { + goto l22 + } + position++ + if buffer[position] != rune('r') { + goto l22 + } + position++ + if buffer[position] != rune('s') { + goto l22 + } + position++ + { + add(ruleAction4, position) + } + if !_rules[ruleopen]() { + goto l22 + } + if !_rules[rulecol]() { + goto l22 + } + if !_rules[rulecomma]() { + goto l22 + } + if !_rules[ruleargs]() { + goto l22 + } + if !_rules[ruleclose]() { + goto l22 + } + { + add(ruleAction5, position) + } + goto l7 + l22: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('C') { + goto l25 + } + position++ + if buffer[position] != rune('l') { + goto l25 + } + position++ + if buffer[position] != rune('e') { + goto l25 + } + position++ + if buffer[position] != rune('a') { + goto l25 + } + position++ + if buffer[position] != rune('r') { + goto l25 + } + position++ + { + add(ruleAction6, position) + } + if !_rules[ruleopen]() { + goto l25 + } + if !_rules[rulecol]() { + goto l25 + } + if !_rules[rulecomma]() { + goto l25 + } + if !_rules[ruleargs]() { + goto l25 + } + if !_rules[ruleclose]() { + goto l25 + } + { + add(ruleAction7, position) + } + goto l7 + l25: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('T') { + goto l28 + } + position++ + if buffer[position] != rune('o') { + goto l28 + } + position++ + if buffer[position] != rune('p') { + goto l28 + } + position++ + if buffer[position] != rune('N') { + goto l28 + } + position++ + { + add(ruleAction8, position) + } + if !_rules[ruleopen]() { + goto l28 + } + if !_rules[ruleposfield]() { + goto l28 + } + { + position30, tokenIndex30 := position, tokenIndex + if !_rules[rulecomma]() { + goto l30 + } + if !_rules[ruleallargs]() { + goto l30 + } + goto l31 + l30: + position, tokenIndex = position30, tokenIndex30 + } + l31: + if !_rules[ruleclose]() { + goto l28 + } + { + add(ruleAction9, position) + } + goto l7 + l28: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l33 + } + position++ + if buffer[position] != rune('a') { + goto l33 + } + position++ + if buffer[position] != rune('n') { + goto l33 + } + position++ + if buffer[position] != rune('g') { + goto l33 + } + position++ + if buffer[position] != rune('e') { + goto l33 + } + position++ + { + add(ruleAction10, position) + } + if !_rules[ruleopen]() { + goto l33 + } + { + position35, tokenIndex35 := position, tokenIndex + { + position37 := position + if !_rules[rulefield]() { + goto l36 + } + if !_rules[rulesp]() { + goto l36 + } + if buffer[position] != rune('=') { + goto l36 + } + position++ + if !_rules[rulesp]() { + goto l36 + } + if !_rules[rulevalue]() { + goto l36 + } + if !_rules[rulecomma]() { + goto l36 + } + { + position38 := position + if !_rules[ruletimestampfmt]() { + goto l36 + } + add(rulePegText, position38) + } + { + add(ruleAction26, position) + } + if !_rules[rulecomma]() { + goto l36 + } + { + position40 := position + if !_rules[ruletimestampfmt]() { + goto l36 + } + add(rulePegText, position40) + } + { + add(ruleAction27, position) + } + add(ruletimerange, position37) + } + goto l35 + l36: + position, tokenIndex = position35, tokenIndex35 + { + position43 := position + { + add(ruleAction21, position) + } + if !_rules[rulecondint]() { + goto l42 + } + if !_rules[rulecondLT]() { + goto l42 + } + { + position45 := position + { + position46 := position + if !_rules[rulefieldExpr]() { + goto l42 + } + add(rulePegText, position46) + } + if !_rules[rulesp]() { + goto l42 + } + { + add(ruleAction25, position) + } + add(rulecondfield, position45) + } + if !_rules[rulecondLT]() { + goto l42 + } + if !_rules[rulecondint]() { + goto l42 + } + { + add(ruleAction22, position) + } + add(ruleconditional, position43) + } + goto l35 + l42: + position, tokenIndex = position35, tokenIndex35 + if !_rules[rulearg]() { + goto l33 + } + } + l35: + if !_rules[ruleclose]() { + goto l33 + } + { + add(ruleAction11, position) + } + goto l7 + l33: + position, tokenIndex = position7, tokenIndex7 + { + position50 := position + { + position51 := position + { + position52, tokenIndex52 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l53 + } + position++ + goto l52 + l53: + position, tokenIndex = position52, tokenIndex52 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l5 + } + position++ + } + l52: + l54: + { + position55, tokenIndex55 := position, tokenIndex + { + position56, tokenIndex56 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l57 + } + position++ + goto l56 + l57: + position, tokenIndex = position56, tokenIndex56 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l58 + } + position++ + goto l56 + l58: + position, tokenIndex = position56, tokenIndex56 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l55 + } + position++ + } + l56: + goto l54 + l55: + position, tokenIndex = position55, tokenIndex55 + } + add(ruleIDENT, position51) + } + add(rulePegText, position50) + } + { + add(ruleAction12, position) + } + if !_rules[ruleopen]() { + goto l5 + } + if !_rules[ruleallargs]() { + goto l5 + } + { + position60, tokenIndex60 := position, tokenIndex + if !_rules[rulecomma]() { + goto l60 + } + goto l61 + l60: + position, tokenIndex = position60, tokenIndex60 + } + l61: + if !_rules[ruleclose]() { + goto l5 + } + { + add(ruleAction13, position) + } + } + l7: + add(ruleCall, position6) + } + return true + l5: + position, tokenIndex = position5, tokenIndex5 + return false + }, + /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ + func() bool { + position63, tokenIndex63 := position, tokenIndex + { + position64 := position + { + position65, tokenIndex65 := position, tokenIndex + if !_rules[ruleCall]() { + goto l66 + } + l67: + { + position68, tokenIndex68 := position, tokenIndex + if !_rules[rulecomma]() { + goto l68 + } + if !_rules[ruleCall]() { + goto l68 + } + goto l67 + l68: + position, tokenIndex = position68, tokenIndex68 + } + { + position69, tokenIndex69 := position, tokenIndex + if !_rules[rulecomma]() { + goto l69 + } + if !_rules[ruleargs]() { + goto l69 + } + goto l70 + l69: + position, tokenIndex = position69, tokenIndex69 + } + l70: + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 + if !_rules[ruleargs]() { + goto l71 + } + goto l65 + l71: + position, tokenIndex = position65, tokenIndex65 + if !_rules[rulesp]() { + goto l63 + } + } + l65: + add(ruleallargs, position64) + } + return true + l63: + position, tokenIndex = position63, tokenIndex63 + return false + }, + /* 3 args <- <(arg (comma args)? sp)> */ + func() bool { + position72, tokenIndex72 := position, tokenIndex + { + position73 := position + if !_rules[rulearg]() { + goto l72 + } + { + position74, tokenIndex74 := position, tokenIndex + if !_rules[rulecomma]() { + goto l74 + } + if !_rules[ruleargs]() { + goto l74 + } + goto l75 + l74: + position, tokenIndex = position74, tokenIndex74 + } + l75: + if !_rules[rulesp]() { + goto l72 + } + add(ruleargs, position73) + } + return true + l72: + position, tokenIndex = position72, tokenIndex72 + return false + }, + /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ + func() bool { + position76, tokenIndex76 := position, tokenIndex + { + position77 := position + { + position78, tokenIndex78 := position, tokenIndex + if !_rules[rulefield]() { + goto l79 + } + if !_rules[rulesp]() { + goto l79 + } + if buffer[position] != rune('=') { + goto l79 + } + position++ + if !_rules[rulesp]() { + goto l79 + } + if !_rules[rulevalue]() { + goto l79 + } + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 + if !_rules[rulefield]() { + goto l76 + } + if !_rules[rulesp]() { + goto l76 + } + { + position80 := position + { + position81, tokenIndex81 := position, tokenIndex + if buffer[position] != rune('>') { + goto l82 + } + position++ + if buffer[position] != rune('<') { + goto l82 + } + position++ + { + add(ruleAction14, position) + } + goto l81 + l82: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('<') { + goto l84 + } + position++ + if buffer[position] != rune('=') { + goto l84 + } + position++ + { + add(ruleAction15, position) + } + goto l81 + l84: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('>') { + goto l86 + } + position++ + if buffer[position] != rune('=') { + goto l86 + } + position++ + { + add(ruleAction16, position) + } + goto l81 + l86: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('=') { + goto l88 + } + position++ + if buffer[position] != rune('=') { + goto l88 + } + position++ + { + add(ruleAction17, position) + } + goto l81 + l88: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('!') { + goto l90 + } + position++ + if buffer[position] != rune('=') { + goto l90 + } + position++ + { + add(ruleAction18, position) + } + goto l81 + l90: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('<') { + goto l92 + } + position++ + { + add(ruleAction19, position) + } + goto l81 + l92: + position, tokenIndex = position81, tokenIndex81 + if buffer[position] != rune('>') { + goto l76 + } + position++ + { + add(ruleAction20, position) + } + } + l81: + add(ruleCOND, position80) + } + if !_rules[rulesp]() { + goto l76 + } + if !_rules[rulevalue]() { + goto l76 + } + } + l78: + add(rulearg, position77) + } + return true + l76: + position, tokenIndex = position76, tokenIndex76 + return false + }, + /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ + nil, + /* 6 conditional <- <(Action21 condint condLT condfield condLT condint Action22)> */ + nil, + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ + func() bool { + position97, tokenIndex97 := position, tokenIndex + { + position98 := position + { + position99 := position + { + position100, tokenIndex100 := position, tokenIndex + { + position102, tokenIndex102 := position, tokenIndex + if buffer[position] != rune('-') { + goto l102 + } + position++ + goto l103 + l102: + position, tokenIndex = position102, tokenIndex102 + } + l103: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l101 + } + position++ + l104: + { + position105, tokenIndex105 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l105 + } + position++ + goto l104 + l105: + position, tokenIndex = position105, tokenIndex105 + } + goto l100 + l101: + position, tokenIndex = position100, tokenIndex100 + if buffer[position] != rune('0') { + goto l97 + } + position++ + } + l100: + add(rulePegText, position99) + } + if !_rules[rulesp]() { + goto l97 + } + { + add(ruleAction23, position) + } + add(rulecondint, position98) + } + return true + l97: + position, tokenIndex = position97, tokenIndex97 + return false + }, + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ + func() bool { + position107, tokenIndex107 := position, tokenIndex + { + position108 := position + { + position109 := position + { + position110, tokenIndex110 := position, tokenIndex + if buffer[position] != rune('<') { + goto l111 + } + position++ + if buffer[position] != rune('=') { + goto l111 + } + position++ + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 + if buffer[position] != rune('<') { + goto l107 + } + position++ + } + l110: + add(rulePegText, position109) + } + if !_rules[rulesp]() { + goto l107 + } + { + add(ruleAction24, position) + } + add(rulecondLT, position108) + } + return true + l107: + position, tokenIndex = position107, tokenIndex107 + return false + }, + /* 9 condfield <- <( sp Action25)> */ + nil, + /* 10 timerange <- <(field sp '=' sp value comma Action26 comma Action27)> */ + nil, + /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ + func() bool { + position115, tokenIndex115 := position, tokenIndex + { + position116 := position + { + position117, tokenIndex117 := position, tokenIndex + if !_rules[ruleitem]() { + goto l118 + } + goto l117 + l118: + position, tokenIndex = position117, tokenIndex117 + { + position119 := position + if buffer[position] != rune('[') { + goto l115 + } + position++ + if !_rules[rulesp]() { + goto l115 + } + add(rulelbrack, position119) + } + { + add(ruleAction28, position) + } + if !_rules[rulelist]() { + goto l115 + } + { + position121 := position + if !_rules[rulesp]() { + goto l115 + } + if buffer[position] != rune(']') { + goto l115 + } + position++ + if !_rules[rulesp]() { + goto l115 + } + add(rulerbrack, position121) + } + { + add(ruleAction29, position) + } + } + l117: + add(rulevalue, position116) + } + return true + l115: + position, tokenIndex = position115, tokenIndex115 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position123, tokenIndex123 := position, tokenIndex + { + position124 := position + if !_rules[ruleitem]() { + goto l123 + } + { + position125, tokenIndex125 := position, tokenIndex + if !_rules[rulecomma]() { + goto l125 + } + if !_rules[rulelist]() { + goto l125 + } + goto l126 + l125: + position, tokenIndex = position125, tokenIndex125 + } + l126: + add(rulelist, position124) + } + return true + l123: + position, tokenIndex = position123, tokenIndex123 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ + func() bool { + position127, tokenIndex127 := position, tokenIndex + { + position128 := position + { + position129, tokenIndex129 := position, tokenIndex + if buffer[position] != rune('n') { + goto l130 + } + position++ + if buffer[position] != rune('u') { + goto l130 + } + position++ + if buffer[position] != rune('l') { + goto l130 + } + position++ + if buffer[position] != rune('l') { + goto l130 + } + position++ + { + position131, tokenIndex131 := position, tokenIndex + { + position132, tokenIndex132 := position, tokenIndex + if !_rules[rulecomma]() { + goto l133 + } + goto l132 + l133: + position, tokenIndex = position132, tokenIndex132 + if !_rules[rulesp]() { + goto l130 + } + if !_rules[ruleclose]() { + goto l130 + } + } + l132: + position, tokenIndex = position131, tokenIndex131 + } + { + add(ruleAction30, position) + } + goto l129 + l130: + position, tokenIndex = position129, tokenIndex129 + if buffer[position] != rune('t') { + goto l135 + } + position++ + if buffer[position] != rune('r') { + goto l135 + } + position++ + if buffer[position] != rune('u') { + goto l135 + } + position++ + if buffer[position] != rune('e') { + goto l135 + } + position++ + { + position136, tokenIndex136 := position, tokenIndex + { + position137, tokenIndex137 := position, tokenIndex + if !_rules[rulecomma]() { + goto l138 + } + goto l137 + l138: + position, tokenIndex = position137, tokenIndex137 + if !_rules[rulesp]() { + goto l135 + } + if !_rules[ruleclose]() { + goto l135 + } + } + l137: + position, tokenIndex = position136, tokenIndex136 + } + { + add(ruleAction31, position) + } + goto l129 + l135: + position, tokenIndex = position129, tokenIndex129 + if buffer[position] != rune('f') { + goto l140 + } + position++ + if buffer[position] != rune('a') { + goto l140 + } + position++ + if buffer[position] != rune('l') { + goto l140 + } + position++ + if buffer[position] != rune('s') { + goto l140 + } + position++ + if buffer[position] != rune('e') { + goto l140 + } + position++ + { + position141, tokenIndex141 := position, tokenIndex + { + position142, tokenIndex142 := position, tokenIndex + if !_rules[rulecomma]() { + goto l143 + } + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 + if !_rules[rulesp]() { + goto l140 + } + if !_rules[ruleclose]() { + goto l140 + } + } + l142: + position, tokenIndex = position141, tokenIndex141 + } + { + add(ruleAction32, position) + } + goto l129 + l140: + position, tokenIndex = position129, tokenIndex129 + { + position146 := position + { + position147, tokenIndex147 := position, tokenIndex + if buffer[position] != rune('-') { + goto l147 + } + position++ + goto l148 + l147: + position, tokenIndex = position147, tokenIndex147 + } + l148: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l145 + } + position++ + l149: + { + position150, tokenIndex150 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l150 + } + position++ + goto l149 + l150: + position, tokenIndex = position150, tokenIndex150 + } + { + position151, tokenIndex151 := position, tokenIndex + if buffer[position] != rune('.') { + goto l151 + } + position++ + l153: + { + position154, tokenIndex154 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l154 + } + position++ + goto l153 + l154: + position, tokenIndex = position154, tokenIndex154 + } + goto l152 + l151: + position, tokenIndex = position151, tokenIndex151 + } + l152: + add(rulePegText, position146) + } + { + add(ruleAction33, position) + } + goto l129 + l145: + position, tokenIndex = position129, tokenIndex129 + { + position157 := position + { + position158, tokenIndex158 := position, tokenIndex + if buffer[position] != rune('-') { + goto l158 + } + position++ + goto l159 + l158: + position, tokenIndex = position158, tokenIndex158 + } + l159: + if buffer[position] != rune('.') { + goto l156 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l156 + } + position++ + l160: + { + position161, tokenIndex161 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l161 + } + position++ + goto l160 + l161: + position, tokenIndex = position161, tokenIndex161 + } + add(rulePegText, position157) + } + { + add(ruleAction34, position) + } + goto l129 + l156: + position, tokenIndex = position129, tokenIndex129 + { + position164 := position + { + position167, tokenIndex167 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l168 + } + position++ + goto l167 + l168: + position, tokenIndex = position167, tokenIndex167 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l169 + } + position++ + goto l167 + l169: + position, tokenIndex = position167, tokenIndex167 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l170 + } + position++ + goto l167 + l170: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune('-') { + goto l171 + } + position++ + goto l167 + l171: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune('_') { + goto l172 + } + position++ + goto l167 + l172: + position, tokenIndex = position167, tokenIndex167 + if buffer[position] != rune(':') { + goto l163 + } + position++ + } + l167: + l165: + { + position166, tokenIndex166 := position, tokenIndex + { + position173, tokenIndex173 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l174 + } + position++ + goto l173 + l174: + position, tokenIndex = position173, tokenIndex173 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l175 + } + position++ + goto l173 + l175: + position, tokenIndex = position173, tokenIndex173 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l176 + } + position++ + goto l173 + l176: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune('-') { + goto l177 + } + position++ + goto l173 + l177: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune('_') { + goto l178 + } + position++ + goto l173 + l178: + position, tokenIndex = position173, tokenIndex173 + if buffer[position] != rune(':') { + goto l166 + } + position++ + } + l173: + goto l165 + l166: + position, tokenIndex = position166, tokenIndex166 + } + add(rulePegText, position164) + } + { + add(ruleAction35, position) + } + goto l129 + l163: + position, tokenIndex = position129, tokenIndex129 + if buffer[position] != rune('"') { + goto l180 + } + position++ + { + position181 := position + if !_rules[ruledoublequotedstring]() { + goto l180 + } + add(rulePegText, position181) + } + if buffer[position] != rune('"') { + goto l180 + } + position++ + { + add(ruleAction36, position) + } + goto l129 + l180: + position, tokenIndex = position129, tokenIndex129 + if buffer[position] != rune('\'') { + goto l127 + } + position++ + { + position183 := position + { + position184 := position + l185: + { + position186, tokenIndex186 := position, tokenIndex + { + position187, tokenIndex187 := position, tokenIndex + { + position189, tokenIndex189 := position, tokenIndex + { + position190, tokenIndex190 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l191 + } + position++ + goto l190 + l191: + position, tokenIndex = position190, tokenIndex190 + if buffer[position] != rune('\\') { + goto l192 + } + position++ + goto l190 + l192: + position, tokenIndex = position190, tokenIndex190 + if buffer[position] != rune('\n') { + goto l189 + } + position++ + } + l190: + goto l188 + l189: + position, tokenIndex = position189, tokenIndex189 + } + if !matchDot() { + goto l188 + } + goto l187 + l188: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l193 + } + position++ + if buffer[position] != rune('n') { + goto l193 + } + position++ + goto l187 + l193: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l194 + } + position++ + if buffer[position] != rune('"') { + goto l194 + } + position++ + goto l187 + l194: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l195 + } + position++ + if buffer[position] != rune('\'') { + goto l195 + } + position++ + goto l187 + l195: + position, tokenIndex = position187, tokenIndex187 + if buffer[position] != rune('\\') { + goto l186 + } + position++ + if buffer[position] != rune('\\') { + goto l186 + } + position++ + } + l187: + goto l185 + l186: + position, tokenIndex = position186, tokenIndex186 + } + add(rulesinglequotedstring, position184) + } + add(rulePegText, position183) + } + if buffer[position] != rune('\'') { + goto l127 + } + position++ + { + add(ruleAction37, position) + } + } + l129: + add(ruleitem, position128) + } + return true + l127: + position, tokenIndex = position127, tokenIndex127 + return false + }, + /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + func() bool { + { + position198 := position + l199: + { + position200, tokenIndex200 := position, tokenIndex + { + position201, tokenIndex201 := position, tokenIndex + { + position203, tokenIndex203 := position, tokenIndex + { + position204, tokenIndex204 := position, tokenIndex + if buffer[position] != rune('"') { + goto l205 + } + position++ + goto l204 + l205: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\\') { + goto l206 + } + position++ + goto l204 + l206: + position, tokenIndex = position204, tokenIndex204 + if buffer[position] != rune('\n') { + goto l203 + } + position++ + } + l204: + goto l202 + l203: + position, tokenIndex = position203, tokenIndex203 + } + if !matchDot() { + goto l202 + } + goto l201 + l202: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l207 + } + position++ + if buffer[position] != rune('n') { + goto l207 + } + position++ + goto l201 + l207: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l208 + } + position++ + if buffer[position] != rune('"') { + goto l208 + } + position++ + goto l201 + l208: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l209 + } + position++ + if buffer[position] != rune('\'') { + goto l209 + } + position++ + goto l201 + l209: + position, tokenIndex = position201, tokenIndex201 + if buffer[position] != rune('\\') { + goto l200 + } + position++ + if buffer[position] != rune('\\') { + goto l200 + } + position++ + } + l201: + goto l199 + l200: + position, tokenIndex = position200, tokenIndex200 + } + add(ruledoublequotedstring, position198) + } + return true + }, + /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ + nil, + /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + func() bool { + position211, tokenIndex211 := position, tokenIndex + { + position212 := position + { + position213, tokenIndex213 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l214 + } + position++ + goto l213 + l214: + position, tokenIndex = position213, tokenIndex213 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l211 + } + position++ + } + l213: + l215: + { + position216, tokenIndex216 := position, tokenIndex + { + position217, tokenIndex217 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l218 + } + position++ + goto l217 + l218: + position, tokenIndex = position217, tokenIndex217 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l219 + } + position++ + goto l217 + l219: + position, tokenIndex = position217, tokenIndex217 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l220 + } + position++ + goto l217 + l220: + position, tokenIndex = position217, tokenIndex217 + if buffer[position] != rune('_') { + goto l221 + } + position++ + goto l217 + l221: + position, tokenIndex = position217, tokenIndex217 + if buffer[position] != rune('-') { + goto l216 + } + position++ + } + l217: + goto l215 + l216: + position, tokenIndex = position216, tokenIndex216 + } + add(rulefieldExpr, position212) + } + return true + l211: + position, tokenIndex = position211, tokenIndex211 + return false + }, + /* 17 field <- <(<(fieldExpr / reserved)> Action38)> */ + func() bool { + position222, tokenIndex222 := position, tokenIndex + { + position223 := position + { + position224 := position + { + position225, tokenIndex225 := position, tokenIndex + if !_rules[rulefieldExpr]() { + goto l226 + } + goto l225 + l226: + position, tokenIndex = position225, tokenIndex225 + { + position227 := position + { + position228, tokenIndex228 := position, tokenIndex + if buffer[position] != rune('_') { + goto l229 + } + position++ + if buffer[position] != rune('r') { + goto l229 + } + position++ + if buffer[position] != rune('o') { + goto l229 + } + position++ + if buffer[position] != rune('w') { + goto l229 + } + position++ + goto l228 + l229: + position, tokenIndex = position228, tokenIndex228 + if buffer[position] != rune('_') { + goto l230 + } + position++ + if buffer[position] != rune('c') { + goto l230 + } + position++ + if buffer[position] != rune('o') { + goto l230 + } + position++ + if buffer[position] != rune('l') { + goto l230 + } + position++ + goto l228 + l230: + position, tokenIndex = position228, tokenIndex228 + if buffer[position] != rune('_') { + goto l231 + } + position++ + if buffer[position] != rune('s') { + goto l231 + } + position++ + if buffer[position] != rune('t') { + goto l231 + } + position++ + if buffer[position] != rune('a') { + goto l231 + } + position++ + if buffer[position] != rune('r') { + goto l231 + } + position++ + if buffer[position] != rune('t') { + goto l231 + } + position++ + goto l228 + l231: + position, tokenIndex = position228, tokenIndex228 + if buffer[position] != rune('_') { + goto l232 + } + position++ + if buffer[position] != rune('e') { + goto l232 + } + position++ + if buffer[position] != rune('n') { + goto l232 + } + position++ + if buffer[position] != rune('d') { + goto l232 + } + position++ + goto l228 + l232: + position, tokenIndex = position228, tokenIndex228 + if buffer[position] != rune('_') { + goto l233 + } + position++ + if buffer[position] != rune('t') { + goto l233 + } + position++ + if buffer[position] != rune('i') { + goto l233 + } + position++ + if buffer[position] != rune('m') { + goto l233 + } + position++ + if buffer[position] != rune('e') { + goto l233 + } + position++ + if buffer[position] != rune('s') { + goto l233 + } + position++ + if buffer[position] != rune('t') { + goto l233 + } + position++ + if buffer[position] != rune('a') { + goto l233 + } + position++ + if buffer[position] != rune('m') { + goto l233 + } + position++ + if buffer[position] != rune('p') { + goto l233 + } + position++ + goto l228 + l233: + position, tokenIndex = position228, tokenIndex228 + if buffer[position] != rune('_') { + goto l222 + } + position++ + if buffer[position] != rune('f') { + goto l222 + } + position++ + if buffer[position] != rune('i') { + goto l222 + } + position++ + if buffer[position] != rune('e') { + goto l222 + } + position++ + if buffer[position] != rune('l') { + goto l222 + } + position++ + if buffer[position] != rune('d') { + goto l222 + } + position++ + } + l228: + add(rulereserved, position227) + } + } + l225: + add(rulePegText, position224) + } + { + add(ruleAction38, position) + } + add(rulefield, position223) + } + return true + l222: + position, tokenIndex = position222, tokenIndex222 + return false + }, + /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ + nil, + /* 19 posfield <- <( Action39)> */ + func() bool { + position236, tokenIndex236 := position, tokenIndex + { + position237 := position + { + position238 := position + if !_rules[rulefieldExpr]() { + goto l236 + } + add(rulePegText, position238) + } + { + add(ruleAction39, position) + } + add(ruleposfield, position237) + } + return true + l236: + position, tokenIndex = position236, tokenIndex236 + return false + }, + /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ + func() bool { + position240, tokenIndex240 := position, tokenIndex + { + position241 := position + { + position242, tokenIndex242 := position, tokenIndex + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l243 + } + position++ + l244: + { + position245, tokenIndex245 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l245 + } + position++ + goto l244 + l245: + position, tokenIndex = position245, tokenIndex245 + } + goto l242 + l243: + position, tokenIndex = position242, tokenIndex242 + if buffer[position] != rune('0') { + goto l240 + } + position++ + } + l242: + add(ruleuint, position241) + } + return true + l240: + position, tokenIndex = position240, tokenIndex240 + return false + }, + /* 21 uintrow <- <( Action40)> */ + nil, + /* 22 col <- <(( Action41) / ('"' '"' Action42))> */ + func() bool { + position247, tokenIndex247 := position, tokenIndex + { + position248 := position + { + position249, tokenIndex249 := position, tokenIndex + { + position251 := position + if !_rules[ruleuint]() { + goto l250 + } + add(rulePegText, position251) + } + { + add(ruleAction41, position) + } + goto l249 + l250: + position, tokenIndex = position249, tokenIndex249 + if buffer[position] != rune('"') { + goto l247 + } + position++ + { + position253 := position + if !_rules[ruledoublequotedstring]() { + goto l247 + } + add(rulePegText, position253) + } + if buffer[position] != rune('"') { + goto l247 + } + position++ + { + add(ruleAction42, position) + } + } + l249: + add(rulecol, position248) + } + return true + l247: + position, tokenIndex = position247, tokenIndex247 + return false + }, + /* 23 open <- <('(' sp)> */ + func() bool { + position255, tokenIndex255 := position, tokenIndex + { + position256 := position + if buffer[position] != rune('(') { + goto l255 + } + position++ + if !_rules[rulesp]() { + goto l255 + } + add(ruleopen, position256) + } + return true + l255: + position, tokenIndex = position255, tokenIndex255 + return false + }, + /* 24 close <- <(')' sp)> */ + func() bool { + position257, tokenIndex257 := position, tokenIndex + { + position258 := position + if buffer[position] != rune(')') { + goto l257 + } + position++ + if !_rules[rulesp]() { + goto l257 + } + add(ruleclose, position258) + } + return true + l257: + position, tokenIndex = position257, tokenIndex257 + return false + }, + /* 25 sp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position260 := position + l261: + { + position262, tokenIndex262 := position, tokenIndex + { + position263, tokenIndex263 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l264 + } + position++ + goto l263 + l264: + position, tokenIndex = position263, tokenIndex263 + if buffer[position] != rune('\t') { + goto l265 + } + position++ + goto l263 + l265: + position, tokenIndex = position263, tokenIndex263 + if buffer[position] != rune('\n') { + goto l262 + } + position++ + } + l263: + goto l261 + l262: + position, tokenIndex = position262, tokenIndex262 + } + add(rulesp, position260) + } + return true + }, + /* 26 comma <- <(sp ',' sp)> */ + func() bool { + position266, tokenIndex266 := position, tokenIndex + { + position267 := position + if !_rules[rulesp]() { + goto l266 + } + if buffer[position] != rune(',') { + goto l266 + } + position++ + if !_rules[rulesp]() { + goto l266 + } + add(rulecomma, position267) + } + return true + l266: + position, tokenIndex = position266, tokenIndex266 + return false + }, + /* 27 lbrack <- <('[' sp)> */ + nil, + /* 28 rbrack <- <(sp ']' sp)> */ + nil, + /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + nil, + /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + func() bool { + position271, tokenIndex271 := position, tokenIndex + { + position272 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if buffer[position] != rune('-') { + goto l271 + } + position++ + { + position273, tokenIndex273 := position, tokenIndex + if buffer[position] != rune('0') { + goto l274 + } + position++ + goto l273 + l274: + position, tokenIndex = position273, tokenIndex273 + if buffer[position] != rune('1') { + goto l271 + } + position++ + } + l273: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if buffer[position] != rune('-') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if buffer[position] != rune('T') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if buffer[position] != rune(':') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l271 + } + position++ + add(ruletimestampbasicfmt, position272) + } + return true + l271: + position, tokenIndex = position271, tokenIndex271 + return false + }, + /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + func() bool { + position275, tokenIndex275 := position, tokenIndex + { + position276 := position + { + position277, tokenIndex277 := position, tokenIndex + if buffer[position] != rune('"') { + goto l278 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l278 + } + if buffer[position] != rune('"') { + goto l278 + } + position++ + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\'') { + goto l279 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l279 + } + if buffer[position] != rune('\'') { + goto l279 + } + position++ + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 + if !_rules[ruletimestampbasicfmt]() { + goto l275 + } + } + l277: + add(ruletimestampfmt, position276) + } + return true + l275: + position, tokenIndex = position275, tokenIndex275 + return false + }, + /* 32 timestamp <- <( Action43)> */ + nil, + /* 34 Action0 <- <{p.startCall("Set")}> */ + nil, + /* 35 Action1 <- <{p.endCall()}> */ + nil, + /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ + nil, + /* 37 Action3 <- <{p.endCall()}> */ + nil, + /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + nil, + /* 39 Action5 <- <{p.endCall()}> */ + nil, + /* 40 Action6 <- <{p.startCall("Clear")}> */ + nil, + /* 41 Action7 <- <{p.endCall()}> */ + nil, + /* 42 Action8 <- <{p.startCall("TopN")}> */ + nil, + /* 43 Action9 <- <{p.endCall()}> */ + nil, + /* 44 Action10 <- <{p.startCall("Range")}> */ + nil, + /* 45 Action11 <- <{p.endCall()}> */ + nil, + nil, + /* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + nil, + /* 48 Action13 <- <{ p.endCall() }> */ + nil, + /* 49 Action14 <- <{ p.addBTWN() }> */ + nil, + /* 50 Action15 <- <{ p.addLTE() }> */ + nil, + /* 51 Action16 <- <{ p.addGTE() }> */ + nil, + /* 52 Action17 <- <{ p.addEQ() }> */ + nil, + /* 53 Action18 <- <{ p.addNEQ() }> */ + nil, + /* 54 Action19 <- <{ p.addLT() }> */ + nil, + /* 55 Action20 <- <{ p.addGT() }> */ + nil, + /* 56 Action21 <- <{p.startConditional()}> */ + nil, + /* 57 Action22 <- <{p.endConditional()}> */ + nil, + /* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + nil, + /* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + nil, + /* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + nil, + /* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + nil, + /* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + nil, + /* 63 Action28 <- <{ p.startList() }> */ + nil, + /* 64 Action29 <- <{ p.endList() }> */ + nil, + /* 65 Action30 <- <{ p.addVal(nil) }> */ + nil, + /* 66 Action31 <- <{ p.addVal(true) }> */ + nil, + /* 67 Action32 <- <{ p.addVal(false) }> */ + nil, + /* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + nil, + /* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + nil, + /* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + nil, + /* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + nil, + /* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + nil, + /* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 78 Action43 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + nil, + } + p.rules = _rules +} diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go new file mode 100644 index 000000000..ad40364b3 --- /dev/null +++ b/pql/pqlpeg_test.go @@ -0,0 +1,546 @@ +package pql + +import ( + "reflect" + "strconv" + "testing" +) + +func TestPEG(t *testing.T) { + p := PQL{Buffer: ` +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} + p.Init() + err := p.Parse() + if err != nil { + t.Fatalf("parse error: %v", err) + } + p.Execute() + + p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`} + p.Init() + err = p.Parse() + if err == nil { + t.Fatalf("should have been an error because of the interior unescaped double quote") + } + + q, err := ParseString("TopN(blah, Bitmap(id==other), field=f, n=0)") + if err != nil { + t.Fatalf("should have parsed: %v", err) + } + if q.String() != `TopN(Bitmap(id == "other"), _field="blah", field="f", n=0)` { + t.Fatalf("Failed, got: %s", q) + } + + q, err = ParseString("C(a=falsen0)") + if err != nil { + t.Fatalf("falsen0 should have been parsed as a string") + } + + q, err = ParseString("Bitmap(row=4, did==other)") + if err != nil { + t.Fatalf("should have parsed: %v", err) + } + + if q.String() != `Bitmap(did == "other", row=4)` { + t.Fatalf("got %s", q) + } + +} + +func TestOldPQL(t *testing.T) { + _, err := ParseString(`SetBit(f=11, col=1)`) + if err != nil { + t.Fatalf("should have parsed: %v", err) + } +} + +func TestPEGWorking(t *testing.T) { + tests := []struct { + name string + input string + ncalls int + }{ + { + name: "Empty", + input: "", + ncalls: 0}, + { + name: "Set", + input: "Set(2, f=10)", + ncalls: 1}, + { + name: "SetTime", + input: "Set(2, f=1, 1999-12-31T00:00)", + ncalls: 1}, + { + name: "DoubleSet", + input: "Set(1, a=4)Set(2, a=4)", + ncalls: 2}, + { + name: "DoubleSetSpc", + input: "Set(1, a=4) Set(2, a=4)", + ncalls: 2}, + { + name: "DoubleSetNewline", + input: "Set(1, a=4) \n Set(2, a=4)", + ncalls: 2}, + { + name: "SetWithArbCall", + input: "Set(1, a=4)Blerg(z=ha)", + ncalls: 2}, + { + name: "SetArbSet", + input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)", + ncalls: 3}, + { + name: "ArbSetArb", + input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)", + ncalls: 3}, + { + name: "SetStringArg", + input: "Set(1, a=zoom)", + ncalls: 1}, + { + name: "SetManyArgs", + input: "Set(1, a=4, b=5)", + ncalls: 1}, + { + name: "SetManyMixedArgs", + input: "Set(1, a=4, bsd=haha)", + ncalls: 1}, + { + name: "SetTimestamp", + input: "Set(1, a=4, 2017-04-03T19:34)", + ncalls: 1}, + { + name: "Union()", + input: "Union()", + ncalls: 1}, + { + name: "UnionOneRow", + input: "Union(Row(a=1))", + ncalls: 1}, + { + name: "UnionTwoRows", + input: "Union(Row(a=1), Row(z=44))", + ncalls: 1}, + { + name: "UnionNested", + input: "Union(Intersect(Row(), Union(Row(), Row())), Row())", + ncalls: 1}, + { + name: "TopN no args", + input: "TopN(boondoggle)", + ncalls: 1}, + { + name: "TopN with args", + input: "TopN(boon, doggle=9)", + ncalls: 1}, + { + name: "double quoted args", + input: `B(a="zm''e")`, + ncalls: 1}, + { + name: "single quoted args", + input: `B(a='zm""e')`, + ncalls: 1}, + { + name: "SetRowAttrs", + input: "SetRowAttrs(blah, 9, a=47)", + ncalls: 1}, + { + name: "SetRowAttrs2args", + input: "SetRowAttrs(blah, 9, a=47, b=bval)", + ncalls: 1}, + { + name: "SetColumnAttrs", + input: "SetColumnAttrs(9, a=47)", + ncalls: 1}, + { + name: "SetColumnAttrs2args", + input: "SetColumnAttrs(9, a=47, b=bval)", + ncalls: 1}, + { + name: "Clear", + input: "Clear(1, a=53)", + ncalls: 1}, + { + name: "Clear2args", + input: "Clear(1, a=53, b=33)", + ncalls: 1}, + { + name: "TopN", + input: "TopN(myfield, n=44)", + ncalls: 1}, + { + name: "TopNBitmap", + input: "TopN(myfield, Row(a=47), n=10)", + ncalls: 1}, + { + name: "RangeLT", + input: "Range(a < 4)", + ncalls: 1}, + { + name: "RangeGT", + input: "Range(a > 4)", + ncalls: 1}, + { + name: "RangeLTE", + input: "Range(a <= 4)", + ncalls: 1}, + { + name: "RangeGTE", + input: "Range(a >= 4)", + ncalls: 1}, + { + name: "RangeEQ", + input: "Range(a == 4)", + ncalls: 1}, + { + name: "RangeNEQ", + input: "Range(a != null)", + ncalls: 1}, + { + name: "RangeLTLT", + input: "Range(4 < a < 9)", + ncalls: 1}, + { + name: "RangeLTLTE", + input: "Range(4 < a <= 9)", + ncalls: 1}, + { + name: "RangeLTELT", + input: "Range(4 <= a < 9)", + ncalls: 1}, + { + name: "RangeLTELTE", + input: "Range(4 <= a <= 9)", + ncalls: 1}, + { + name: "RangeTime", + input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)", + ncalls: 1}, + { + name: "RangeTimeQuotes", + input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`, + ncalls: 1}, + { + name: "Dashed Frame", + input: "Set(1, my-frame=9)", + ncalls: 1}, + { + name: "newlines", + input: `Set( +1, +my-frame +=9)`, + ncalls: 1}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.input) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.input, err) + } + if len(q.Calls) != test.ncalls { + t.Fatalf("wrong number of calls for '%s': %#v", test.input, q.Calls) + } + }) + } +} + +func TestPEGErrors(t *testing.T) { + tests := []struct { + name string + input string + }{ + { + name: "SetNoParens", + input: "Set"}, + { + name: "SetBadTimestamp", + input: "Set(1, a=4, 2017-94-03T19:34)"}, + { + name: "SetTimestampNoArg", + input: "Set(1, 2017-04-03T19:34)"}, + { + name: "SetStartingComma", + input: "Set(, 1, a=4)"}, + { + name: "StartinCommaArb", + input: "Zeeb(, a=4)"}, + { + name: "SetRowAttrs0args", + input: "SetRowAttrs(blah, 9)"}, + { + name: "Clear0args", + input: "Clear(9)"}, + { + name: "RangeTimeGT", + input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, + { + name: "RangeTimeOneStamp", + input: "Range(a=4, 2010-07-04T00:00)"}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.input) + if err == nil { + t.Fatalf("parsing query '%s' - expected error, got: %s", test.input, q) + } + }) + } +} + +func TestPQLDeepEquality(t *testing.T) { + tests := []struct { + name string + call string + exp *Call + }{ + { + name: "Set", + call: "Set(1, a=7, 2010-07-08T14:44)", + exp: &Call{ + Name: "Set", + Args: map[string]interface{}{ + "a": int64(7), + "_col": int64(1), + "_timestamp": "2010-07-08T14:44", + }, + }}, + { + name: "SetRowAttrs", + call: "SetRowAttrs(myfield, 9, z=4)", + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": int64(9), + }, + }}, + { + name: "SetColumnAttrs", + call: "SetColumnAttrs(9, z=4)", + exp: &Call{ + Name: "SetColumnAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_col": int64(9), + }, + }}, + { + name: "Clear", + call: "Clear(1, a=7)", + exp: &Call{ + Name: "Clear", + Args: map[string]interface{}{ + "a": int64(7), + "_col": int64(1), + }, + }}, + { + name: "TopN", + call: "TopN(myfield, Row(), a=7)", + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "a": int64(7), + "_field": "myfield", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "RangeEQ", + call: "Range(a==7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: EQ, + Value: int64(7), + }, + }, + }}, + { + name: "RangeLT", + call: "Range(a<7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: LT, + Value: int64(7), + }, + }, + }}, + { + name: "RangeLTE", + call: "Range(a<=7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: LTE, + Value: int64(7), + }, + }, + }}, + { + name: "RangeGTE", + call: "Range(a>=7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: GTE, + Value: int64(7), + }, + }, + }}, + { + name: "RangeGT", + call: "Range(a>7)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: GT, + Value: int64(7), + }, + }, + }}, + { + name: "RangeNEQ", + call: "Range(a!=null)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: NEQ, + Value: nil, + }, + }, + }}, + { + name: "RangeLTELT", + call: "Range(4 <= a < 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(4), int64(9)}, + }, + }, + }}, + { + name: "RangeLTLT", + call: "Range(4 < a < 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(9)}, + }, + }, + }}, + { + name: "RangeLTELTE", + call: "Range(4 <= a <= 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(4), int64(10)}, + }, + }, + }}, + { + name: "RangeLTLTE", + call: "Range(4 < a <= 9)", + exp: &Call{ + Name: "Range", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(10)}, + }, + }, + }}, + { + name: "Sum", + call: "Sum(field=f)", + exp: &Call{ + Name: "Sum", + Args: map[string]interface{}{ + "field": "f", + }, + }}, + { + name: "Weird dash", + call: "Sum(field-=f)", + exp: &Call{ + Name: "Sum", + Args: map[string]interface{}{ + "field-": "f", + }, + }}, + { + name: "SumChild", + call: "Sum(Row(), field=f)", + exp: &Call{ + Name: "Sum", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "MinChild", + call: "Min(Row(), field=f)", + exp: &Call{ + Name: "Min", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + { + name: "MaxChild", + call: "Max(Row(), field=f)", + exp: &Call{ + Name: "Max", + Args: map[string]interface{}{ + "field": "f", + }, + Children: []*Call{ + {Name: "Row"}, + }, + }}, + } + + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + q, err := ParseString(test.call) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.call, err) + } + + if !reflect.DeepEqual(test.exp, q.Calls[0]) { + t.Fatalf("unexpected call:\n%s\ninstead of:\n%s\n'%#v'\ninstead of:\n'%#v'", q.Calls[0], test.exp, q.Calls[0], test.exp) + } + }) + } +} diff --git a/pql/scanner.go b/pql/scanner.go deleted file mode 100644 index 5a24b6af2..000000000 --- a/pql/scanner.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql - -import ( - "bufio" - "bytes" - "io" - "unicode" -) - -// Scanner represents a PQL lexical scanner. -type Scanner struct { - r io.RuneScanner - pos Pos -} - -// NewScanner returns a new instance of Scanner. -func NewScanner(r io.Reader) *Scanner { - return &Scanner{r: bufio.NewReader(r)} -} - -// Scan returns the next token and position from the underlying reader. -func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { - pos = s.pos - - // Read next code point. - ch := s.read() - - // If we see whitespace then consume all contiguous whitespace. - // If we see a letter, or certain acceptable special characters, then consume - // as an ident or reserved word. If we see quotes, then scan as string. - if isWhitespace(ch) { - s.unread() - return s.scanWhitespace() - } else if isIdentFirstChar(ch) { - s.unread() - return s.scanIdent() - } else if isDigit(ch) || ch == '-' { - s.unread() - return s.scanNumber() - } else if ch == '"' || ch == '\'' { - s.unread() - return s.scanString() - } - - // Otherwise parse individual characters. - switch ch { - case eof: - return EOF, pos, "" - case '=': - if next := s.read(); next == '=' { - return EQ, pos, "==" - } - s.unread() - return ASSIGN, pos, string(ch) - case '!': - if next := s.read(); next == '=' { - return NEQ, pos, "!=" - } - s.unread() - return ASSIGN, pos, string(ch) - case '<': - if next := s.read(); next == '=' { - return LTE, pos, "<=" - } - s.unread() - return LT, pos, string(ch) - case '>': - next := s.read() - if next == '=' { - return GTE, pos, ">=" - } else if next == '<' { - return BETWEEN, pos, "><" - } - s.unread() - return GT, pos, string(ch) - case ',': - return COMMA, pos, string(ch) - case '(': - return LPAREN, pos, string(ch) - case ')': - return RPAREN, pos, string(ch) - case '[': - return LBRACK, pos, string(ch) - case ']': - return RBRACK, pos, string(ch) - default: - return ILLEGAL, pos, string(ch) - } -} - -// read returns the next code point from the underlying reader and updates the pos. -func (s *Scanner) read() rune { - // Read next rune from underlying reader. - ch, _, err := s.r.ReadRune() - if err != nil { - return eof - } - - // Update position information. - if ch == '\n' { - s.pos.Line++ - s.pos.Char = 0 - } else { - s.pos.Char++ - } - - return ch -} - -// unread pushes the previously read rune back onto the reader. -func (s *Scanner) unread() { - if s.pos.Char == 0 { - s.pos.Line-- - } else { - s.pos.Char-- - } - - s.r.UnreadRune() -} - -// scanWhitespace consumes the current rune and all contiguous whitespace. -func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isWhitespace(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - - return WS, pos, buf.String() -} - -func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { - pos = s.pos - - var buf bytes.Buffer - for { - ch := s.read() - if ch == eof { - break - } else if !isIdentChar(ch) { - s.unread() - break - } - buf.WriteRune(ch) - } - lit = buf.String() - - // If the literal matches a keyword then return that keyword. - if tok = Lookup(lit); tok != IDENT { - return tok, pos, lit - } - - return IDENT, pos, lit -} - -// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. -func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { - pos = s.pos - tok = INTEGER - - var buf bytes.Buffer - var seenDot bool - first := true - for { - ch := s.read() - if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { - s.unread() - break - } - if ch == '.' { - seenDot = true - tok = FLOAT - } - buf.WriteRune(ch) - first = false - } - return tok, pos, buf.String() -} - -// scanString consumes a single-quoted or double-quoted string. -func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { - pos = s.pos - - // This must be either a single- or double-quote. - ending := s.read() - - var buf bytes.Buffer - for { - ch := s.read() - if ch == ending { - break - } else if ch == '\n' || ch == eof { - return BADSTRING, pos, buf.String() - } else if ch == '\\' { - next := s.read() - if next == 'n' { - buf.WriteRune('\n') - } else if next == '\\' { - buf.WriteRune('\\') - } else if next == '"' { - buf.WriteRune('"') - } else if next == '\'' { - buf.WriteRune('\'') - } else { - return BADSTRING, pos, buf.String() - } - } else { - buf.WriteRune(ch) - } - } - - return STRING, pos, buf.String() -} - -// bufScanner represents a wrapper for scanner to add a buffer. -// It provides a fixed-length circular buffer that can be unread. -type bufScanner struct { - s *Scanner - i int // buffer index - n int // buffer size - buf [8]struct { - tok Token - pos Pos - lit string - } -} - -// newBufScanner returns a new buffered scanner for a reader. -func newBufScanner(r io.Reader) *bufScanner { - return &bufScanner{s: NewScanner(r)} -} - -// Scan reads the next token from the scanner. -func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { - // If we have unread tokens then read them off the buffer first. - if s.n > 0 { - s.n-- - return s.curr() - } - - // Move buffer position forward and save the token. - s.i = (s.i + 1) % len(s.buf) - buf := &s.buf[s.i] - buf.tok, buf.pos, buf.lit = s.s.Scan() - - return s.curr() -} - -// unscan pushes the previously token back onto the buffer. -func (s *bufScanner) unscan() { s.n++ } - -// curr returns the last read token. -func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { - buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] - return buf.tok, buf.pos, buf.lit -} - -// pos returns the current position. -func (s *bufScanner) pos() Pos { - _, pos, _ := s.curr() - return pos -} - -// isWhitespace returns true if the rune a Unicode space character. -func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } - -// isLetter returns true if the rune is a letter. -func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } - -// isDigit returns true if the rune is a digit. -func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } - -// isIdentChar returns true if the rune can be used in an unquoted identifier. -func isIdentChar(ch rune) bool { - return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' -} - -// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. -func isIdentFirstChar(ch rune) bool { return isLetter(ch) } - -const eof = rune(0) diff --git a/pql/scanner_test.go b/pql/scanner_test.go deleted file mode 100644 index e48896748..000000000 --- a/pql/scanner_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pql_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/pql" -) - -func TestScanner_Scan(t *testing.T) { - var tests = []struct { - name string - s string - tok pql.Token - lit string - pos pql.Pos - }{ - // Special tokens (EOF, ILLEGAL, WS) - {name: "EOF", s: ``, tok: pql.EOF}, - {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, - {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, - {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, - {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, - - {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, - {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, - {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, - {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, - {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, - {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, - {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, - {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, - {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, - {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, - {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, - {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, - {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, - - {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, - {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, - {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, - - {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, - {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := pql.NewScanner(strings.NewReader(tt.s)) - tok, pos, lit := s.Scan() - if tt.tok != tok { - t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) - } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { - t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) - } else if tt.lit != lit { - t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) - } - }) - } -} diff --git a/pql/token.go b/pql/token.go index 6997f17af..51eea410d 100644 --- a/pql/token.go +++ b/pql/token.go @@ -14,28 +14,12 @@ package pql -import "strings" - // Token is a lexical token of the PQL language. type Token int const ( // Special tokens ILLEGAL Token = iota - EOF - WS - - literal_beg - IDENT // main - STRING // "foo" - BADSTRING // bad escape or unclosed string - INTEGER // 12345 - FLOAT // 100.2 - literal_end - - keyword_beg - ALL - keyword_end ASSIGN // = EQ // == @@ -45,23 +29,10 @@ const ( GT // > GTE // >= BETWEEN // >< - COMMA // , - LPAREN // ( - RPAREN // ) - LBRACK // ( - RBRACK // ) ) var tokens = [...]string{ ILLEGAL: "ILLEGAL", - EOF: "EOF", - WS: "WS", - - IDENT: "IDENT", - INTEGER: "INTEGER", - FLOAT: "FLOAT", - - ALL: "ALL", ASSIGN: "=", EQ: "==", @@ -71,20 +42,6 @@ var tokens = [...]string{ GT: ">", GTE: ">=", BETWEEN: "><", - COMMA: ",", - LPAREN: "(", - RPAREN: ")", - LBRACK: "(", - RBRACK: ")", -} - -var keywords map[string]Token - -func init() { - keywords = make(map[string]Token) - for tok := keyword_beg + 1; tok < keyword_end; tok++ { - keywords[strings.ToLower(tokens[tok])] = tok - } } // String returns the string representation of the token. @@ -94,18 +51,3 @@ func (tok Token) String() string { } return "" } - -// Lookup returns the token associated with a given string. -func Lookup(ident string) Token { - if tok, ok := keywords[strings.ToLower(ident)]; ok { - return tok - } - return IDENT -} - -// Pos specifies the line and character position of a token. -// The Char and Line are both zero-based indexes. -type Pos struct { - Line int - Char int -} diff --git a/roaring/containers.go b/roaring/containers.go index 1cf366efc..5275d4f04 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -14,18 +14,18 @@ package roaring -type SliceContainers struct { +type sliceContainers struct { keys []uint64 containers []*Container lastKey uint64 lastContainer *Container } -func NewSliceContainers() *SliceContainers { - return &SliceContainers{} +func newSliceContainers() *sliceContainers { + return &sliceContainers{} } -func (sc *SliceContainers) Get(key uint64) *Container { +func (sc *sliceContainers) Get(key uint64) *Container { i := search64(sc.keys, key) if i < 0 { return nil @@ -33,7 +33,7 @@ func (sc *SliceContainers) Get(key uint64) *Container { return sc.containers[i] } -func (sc *SliceContainers) Put(key uint64, c *Container) { +func (sc *sliceContainers) Put(key uint64, c *Container) { i := search64(sc.keys, key) // If index is negative then there's not an exact match @@ -46,7 +46,7 @@ func (sc *SliceContainers) Put(key uint64, c *Container) { } -func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { i := search64(sc.keys, key) if i < 0 { c := NewContainer() @@ -63,7 +63,7 @@ func (sc *SliceContainers) PutContainerValues(key uint64, containerType byte, n } -func (sc *SliceContainers) Remove(key uint64) { +func (sc *sliceContainers) Remove(key uint64) { i := search64(sc.keys, key) if i < 0 { return @@ -72,7 +72,7 @@ func (sc *SliceContainers) Remove(key uint64) { sc.containers = append(sc.containers[:i], sc.containers[i+1:]...) } -func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { +func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) { sc.keys = append(sc.keys, 0) copy(sc.keys[i+1:], sc.keys[i:]) sc.keys[i] = key @@ -82,7 +82,7 @@ func (sc *SliceContainers) insertAt(key uint64, c *Container, i int) { sc.containers[i] = c } -func (sc *SliceContainers) GetOrCreate(key uint64) *Container { +func (sc *sliceContainers) GetOrCreate(key uint64) *Container { // Check the last* cache for same container. if key == sc.lastKey && sc.lastContainer != nil { return sc.lastContainer @@ -101,8 +101,8 @@ func (sc *SliceContainers) GetOrCreate(key uint64) *Container { return sc.lastContainer } -func (sc *SliceContainers) Clone() Containers { - other := NewSliceContainers() +func (sc *sliceContainers) Clone() Containers { + other := newSliceContainers() other.keys = make([]uint64, len(sc.keys)) other.containers = make([]*Container, len(sc.containers)) copy(other.keys, sc.keys) @@ -112,26 +112,34 @@ func (sc *SliceContainers) Clone() Containers { return other } -func (sc *SliceContainers) Last() (key uint64, c *Container) { +func (sc *sliceContainers) Last() (key uint64, c *Container) { if len(sc.keys) == 0 { return 0, nil } return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1] } -func (sc *SliceContainers) Size() int { +func (sc *sliceContainers) Size() int { return len(sc.keys) } -func (sc *SliceContainers) Reset() { +func (sc *sliceContainers) Count() uint64 { + n := uint64(0) + for i := range sc.containers { + n += uint64(sc.containers[i].n) + } + return n +} + +func (sc *sliceContainers) Reset() { sc.keys = sc.keys[:0] sc.containers = sc.containers[:0] sc.lastContainer = nil sc.lastKey = 0 } -func (sc *SliceContainers) seek(key uint64) (int, bool) { +func (sc *sliceContainers) seek(key uint64) (int, bool) { i := search64(sc.keys, key) found := true if i < 0 { @@ -141,28 +149,29 @@ func (sc *SliceContainers) seek(key uint64) (int, bool) { return i, found } -func (sc *SliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { +func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) { i, found := sc.seek(key) - return &SliceIterator{e: sc, i: i}, found + return &sliceIterator{e: sc, i: i}, found } -type SliceIterator struct { - e *SliceContainers +type sliceIterator struct { + e *sliceContainers i int key uint64 value *Container } -func (si *SliceIterator) Next() bool { +func (si *sliceIterator) Next() bool { if si.e == nil || si.i > len(si.e.keys)-1 { return false } si.key = si.e.keys[si.i] si.value = si.e.containers[si.i] si.i++ + return true } -func (si *SliceIterator) Value() (uint64, *Container) { +func (si *sliceIterator) Value() (uint64, *Container) { return si.key, si.value } diff --git a/roaring/roaring.go b/roaring/roaring.go index 42d06eb50..3756d1db7 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -51,14 +51,14 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - //ContainerArray indicates a container of bit position values - ContainerArray = byte(1) + //containerArray indicates a container of bit position values + containerArray = byte(1) - //ContainerBitmap indicates a container of bits packed in a uint64 array block - ContainerBitmap = byte(2) + //containerBitmap indicates a container of bits packed in a uint64 array block + containerBitmap = byte(2) - //ContainerRun indicates a container of run encoded bits - ContainerRun = byte(3) + //containerRun indicates a container of run encoded bits + containerRun = byte(3) maxContainerVal = 0xffff ) @@ -93,9 +93,11 @@ type Containers interface { // return the first container at or after key. found will be true if a // container is found at key. Iterator(key uint64) (citer ContainerIterator, found bool) + + Count() uint64 + //Reset will clear the containers collection to allow for recycling during snapshot Reset() - } type ContainerIterator interface { @@ -117,7 +119,7 @@ type Bitmap struct { // NewBitmap returns a Bitmap with an initial set of values. func NewBitmap(a ...uint64) *Bitmap { b := &Bitmap{ - Containers: NewSliceContainers(), + Containers: newSliceContainers(), } b.Add(a...) return b @@ -221,12 +223,7 @@ func (b *Bitmap) Max() uint64 { // Count returns the number of bits set in the bitmap. func (b *Bitmap) Count() (n uint64) { - citer, _ := b.Containers.Iterator(0) - for citer.Next() { - _, c := citer.Value() - n += uint64(c.n) - } - return n + return b.Containers.Count() } // CountRange returns the number of bits set between [start, end). @@ -505,7 +502,7 @@ func (b *Bitmap) Optimize() { citer, _ := b.Containers.Iterator(0) for citer.Next() { _, c := citer.Value() - c.Optimize() + c.optimize() } } @@ -662,18 +659,18 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { citer.Next() _, c := citer.Value() switch c.containerType { - case ContainerRun: + case containerRun: c.array = nil c.bitmap = nil runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount] opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size - case ContainerArray: + case containerArray: c.runs = nil c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32) - case ContainerBitmap: + case containerBitmap: c.array = nil c.runs = nil c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] @@ -695,6 +692,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // FIXME(benbjohnson): return error with position so file can be trimmed. return err } + opr.apply(b) // Increase the op count. @@ -729,10 +727,10 @@ func (b *Bitmap) Iterator() *Iterator { } // Info returns stats for the bitmap. -func (b *Bitmap) Info() BitmapInfo { - info := BitmapInfo{ +func (b *Bitmap) Info() bitmapInfo { + info := bitmapInfo{ OpN: b.opN, - Containers: make([]ContainerInfo, 0, b.Containers.Size()), + Containers: make([]containerInfo, 0, b.Containers.Size()), } citer, _ := b.Containers.Iterator(0) @@ -792,10 +790,10 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap { return result } -// BitmapInfo represents a point-in-time snapshot of bitmap stats. -type BitmapInfo struct { +// bitmapInfo represents a point-in-time snapshot of bitmap stats. +type bitmapInfo struct { OpN int - Containers []ContainerInfo + Containers []containerInfo } // Iterator represents an iterator over a Bitmap. @@ -991,8 +989,8 @@ func (itr *Iterator) peek() uint64 { // ArrayMaxSize represents the maximum size of array containers. const ArrayMaxSize = 4096 -// RunMaxSize represents the maximum size of run length encoded containers. -const RunMaxSize = 2048 +// runMaxSize represents the maximum size of run length encoded containers. +const runMaxSize = 2048 // Container represents a Container for uint16 integers. // @@ -1025,7 +1023,7 @@ func (iv interval16) runlen() int { // newContainer returns a new instance of container. func NewContainer() *Container { - return &Container{containerType: ContainerArray} + return &Container{containerType: containerArray} } // Mapped returns true if the container is mapped directly to a byte slice @@ -1033,6 +1031,11 @@ func (c *Container) Mapped() bool { return c.mapped } +// N returns the cached bit count of the container +func (c *Container) N() int { + return c.n +} + // Update updates the container func (c *Container) Update(containerType byte, n int, mapped bool) { c.containerType = containerType @@ -1042,17 +1045,17 @@ func (c *Container) Update(containerType byte, n int, mapped bool) { // isArray returns true if the container is an array container. func (c *Container) isArray() bool { - return c.containerType == ContainerArray + return c.containerType == containerArray } // isBitmap returns true if the container is a bitmap container. func (c *Container) isBitmap() bool { - return c.containerType == ContainerBitmap + return c.containerType == containerBitmap } // isRun returns true if the container is a run-length-encoded container. func (c *Container) isRun() bool { - return c.containerType == ContainerRun + return c.containerType == containerRun } // unmap creates copies of the containers data in the heap. @@ -1065,15 +1068,15 @@ func (c *Container) unmap() { } switch c.containerType { - case ContainerArray: + case containerArray: tmp := make([]uint16, len(c.array)) copy(tmp, c.array) c.array = tmp - case ContainerBitmap: + case containerBitmap: tmp := make([]uint64, len(c.bitmap)) copy(tmp, c.bitmap) c.bitmap = tmp - case ContainerRun: + case containerRun: tmp := make([]interval16, len(c.runs)) copy(tmp, c.runs) c.runs = tmp @@ -1314,40 +1317,40 @@ func (c *Container) countRuns() (r int) { return 0 } -// Optimize converts the container to the type which will take up the least +// optimize converts the container to the type which will take up the least // amount of space. -func (c *Container) Optimize() { +func (c *Container) optimize() { if c.n == 0 { return } runs := c.countRuns() var newType byte - if runs <= RunMaxSize && runs <= c.n/2 { - newType = ContainerRun + if runs <= runMaxSize && runs <= c.n/2 { + newType = containerRun } else if c.n < ArrayMaxSize { - newType = ContainerArray + newType = containerArray } else { - newType = ContainerBitmap + newType = containerBitmap } // Then convert accordingly. if c.isArray() { - if newType == ContainerBitmap { + if newType == containerBitmap { c.arrayToBitmap() - } else if newType == ContainerRun { + } else if newType == containerRun { c.arrayToRun() } } else if c.isBitmap() { - if newType == ContainerArray { + if newType == containerArray { c.bitmapToArray() - } else if newType == ContainerRun { + } else if newType == containerRun { c.bitmapToRun() } } else if c.isRun() { - if newType == ContainerBitmap { + if newType == containerBitmap { c.runToBitmap() - } else if newType == ContainerArray { + } else if newType == containerArray { c.runToArray() } } @@ -1486,7 +1489,7 @@ func (c *Container) runMax() uint16 { // bitmapToArray converts from bitmap format to array format. func (c *Container) bitmapToArray() { c.array = make([]uint16, 0, c.n) - c.containerType = ContainerArray + c.containerType = containerArray // return early if empty if c.n == 0 { @@ -1509,7 +1512,7 @@ func (c *Container) bitmapToArray() { // arrayToBitmap converts from array format to bitmap format. func (c *Container) arrayToBitmap() { c.bitmap = make([]uint64, bitmapN) - c.containerType = ContainerBitmap + c.containerType = containerBitmap // return early if empty if c.n == 0 { @@ -1528,7 +1531,7 @@ func (c *Container) arrayToBitmap() { // runToBitmap converts from RLE format to bitmap format. func (c *Container) runToBitmap() { c.bitmap = make([]uint64, bitmapN) - c.containerType = ContainerBitmap + c.containerType = containerBitmap // return early if empty if c.n == 0 { @@ -1550,7 +1553,7 @@ func (c *Container) runToBitmap() { // bitmapToRun converts from bitmap format to RLE format. func (c *Container) bitmapToRun() { - c.containerType = ContainerRun + c.containerType = containerRun // return early if empty if c.n == 0 { c.runs = make([]interval16, 0) @@ -1606,7 +1609,7 @@ func (c *Container) bitmapToRun() { // arrayToRun converts from array format to RLE format. func (c *Container) arrayToRun() { - c.containerType = ContainerRun + c.containerType = containerRun // return early if empty if c.n == 0 { c.runs = make([]interval16, 0) @@ -1633,7 +1636,7 @@ func (c *Container) arrayToRun() { // runToArray converts from RLE format to array format. func (c *Container) runToArray() { - c.containerType = ContainerArray + c.containerType = containerArray c.array = make([]uint16, 0, c.n) // return early if empty @@ -1657,13 +1660,13 @@ func (c *Container) Clone() *Container { other := &Container{n: c.n, containerType: c.containerType} switch c.containerType { - case ContainerArray: + case containerArray: other.array = make([]uint16, len(c.array)) copy(other.array, c.array) - case ContainerBitmap: + case containerBitmap: other.bitmap = make([]uint64, len(c.bitmap)) copy(other.bitmap, c.bitmap) - case ContainerRun: + case containerRun: other.runs = make([]interval16, len(c.runs)) copy(other.runs, c.runs) } @@ -1729,8 +1732,8 @@ func (c *Container) size() int { } // info returns the current stats about the container. -func (c *Container) info() ContainerInfo { - info := ContainerInfo{N: c.n} +func (c *Container) info() containerInfo { + info := containerInfo{N: c.n} if c.isArray() { info.Type = "array" @@ -1786,8 +1789,8 @@ func (c *Container) check() error { return a } -// ContainerInfo represents a point-in-time snapshot of container stats. -type ContainerInfo struct { +// containerInfo represents a point-in-time snapshot of container stats. +type containerInfo struct { Key uint64 // container key Type string // container type (array, bitmap, or run) N int // number of bits @@ -1815,7 +1818,7 @@ func flipArray(b *Container) *Container { } func flipBitmap(b *Container) *Container { - other := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + other := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i, bitmap := range b.bitmap { other.bitmap[i] = ^bitmap @@ -1976,7 +1979,7 @@ func intersect(a, b *Container) *Container { } func intersectArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1997,7 +2000,7 @@ func intersectArrayArray(a, b *Container) *Container { // container. The return is always an array container (since it's guaranteed to // be low-cardinality) func intersectArrayRun(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -2016,7 +2019,7 @@ func intersectArrayRun(a, b *Container) *Container { // intersectRunRun computes the intersect of two run containers. func intersectRunRun(a, b *Container) *Container { - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -2046,7 +2049,7 @@ func intersectRunRun(a, b *Container) *Container { } if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2058,7 +2061,7 @@ func intersectBitmapRun(a, b *Container) *Container { var output *Container if b.n < ArrayMaxSize { // output is array container - output = &Container{containerType: ContainerArray} + output = &Container{containerType: containerArray} for _, iv := range b.runs { for i := iv.start; i <= iv.last; i++ { if a.bitmapContains(i) { @@ -2077,7 +2080,7 @@ func intersectBitmapRun(a, b *Container) *Container { // the bitmap which are between runs. output = &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for j := 0; j < len(b.runs); j++ { vb := b.runs[j] @@ -2118,7 +2121,7 @@ func intersectBitmapRun(a, b *Container) *Container { } func intersectArrayBitmap(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2133,7 +2136,7 @@ func intersectArrayBitmap(a, b *Container) *Container { } func intersectBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i := range a.bitmap { v := a.bitmap[i] & b.bitmap[i] @@ -2141,7 +2144,7 @@ func intersectBitmapBitmap(a, b *Container) *Container { output.n += int(popcount(v)) } - output.Optimize() + output.optimize() return output } @@ -2174,7 +2177,7 @@ func union(a, b *Container) *Container { } func unionArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; ; { if i >= na && j >= nb { @@ -2210,7 +2213,7 @@ func unionArrayRun(a, b *Container) *Container { if b.n == maxContainerVal+1 { return b.Clone() } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -2231,7 +2234,7 @@ func unionArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2273,7 +2276,7 @@ func unionRunRun(a, b *Container) *Container { na, nb := len(a.runs), len(b.runs) output := &Container{ runs: make([]interval16, 0, na+nb), - containerType: ContainerRun, + containerType: containerRun, } var va, vb interval16 for i, j := 0, 0; i < na || j < nb; { @@ -2291,7 +2294,7 @@ func unionRunRun(a, b *Container) *Container { j++ } } - if len(output.runs) > RunMaxSize { + if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2386,7 +2389,7 @@ func (c *Container) equals(c2 *Container) bool { if c.mapped != c2.mapped || c.containerType != c2.containerType || c.n != c2.n { return false } - if c.containerType == ContainerArray { + if c.containerType == containerArray { if len(c.array) != len(c2.array) { return false } @@ -2395,7 +2398,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.containerType == ContainerBitmap { + } else if c.containerType == containerBitmap { if len(c.bitmap) != len(c2.bitmap) { return false } @@ -2404,7 +2407,7 @@ func (c *Container) equals(c2 *Container) bool { return false } } - } else if c.containerType == ContainerRun { + } else if c.containerType == containerRun { if len(c.runs) != len(c2.runs) { return false } @@ -2433,7 +2436,7 @@ func unionArrayBitmap(a, b *Container) *Container { func unionBitmapBitmap(a, b *Container) *Container { output := &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for i := 0; i < bitmapN; i++ { @@ -2475,7 +2478,7 @@ func difference(a, b *Container) *Container { // differenceArrayArray computes the difference bween two arrays. func differenceArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na; { va := a.array[i] @@ -2506,7 +2509,7 @@ func differenceArrayRun(a, b *Container) *Container { return a.Clone() } - output := &Container{array: make([]uint16, 0, a.n), containerType: ContainerArray} + output := &Container{array: make([]uint16, 0, a.n), containerType: containerArray} // cardinality upper bound: card(A) i := 0 // array index @@ -2541,7 +2544,7 @@ func differenceArrayRun(a, b *Container) *Container { // keep all array elements after end of runs // It's possible that output was converted from array to bitmap in output.add() // so check container type before proceeding. - if output.containerType == ContainerArray { + if output.containerType == containerArray { output.array = append(output.array, a.array[i:]...) // TODO: consider handling container.n mutations in one place // like we do with container.add(). @@ -2574,7 +2577,7 @@ func differenceRunArray(a, b *Container) *Container { if a.n == 0 || b.n == 0 { return a.Clone() } - output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} + output := &Container{runs: make([]interval16, 0, len(a.runs)), containerType: containerRun} bidx := 0 vb := b.array[bidx] @@ -2620,7 +2623,7 @@ RUNLOOP: output.n += int(run.last - start + 1) } } - output.Optimize() + output.optimize() return output } @@ -2630,7 +2633,7 @@ func differenceRunBitmap(a, b *Container) *Container { if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return flipBitmap(b) } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} output.n = a.n if len(a.runs) == 0 { return output @@ -2675,7 +2678,7 @@ func differenceRunBitmap(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -2696,7 +2699,7 @@ func differenceRunRun(a, b *Container) *Container { alen := len(a.runs) blen := len(b.runs) - output := &Container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else + output := &Container{runs: make([]interval16, 0, alen+blen), containerType: containerRun} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -2746,7 +2749,7 @@ func differenceRunRun(a, b *Container) *Container { } func differenceArrayBitmap(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2777,7 +2780,7 @@ func differenceBitmapArray(a, b *Container) *Container { } func differenceBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} for i := range a.bitmap { v := a.bitmap[i] & (^b.bitmap[i]) @@ -2820,7 +2823,7 @@ func xor(a, b *Container) *Container { } func xorArrayArray(a, b *Container) *Container { - output := &Container{containerType: ContainerArray} + output := &Container{containerType: containerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { @@ -2860,7 +2863,7 @@ func xorArrayBitmap(a, b *Container) *Container { // It's possible that output was converted from bitmap to array in output.remove() // so we only do this conversion if output is still a bitmap container. - if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { + if output.containerType == containerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -2870,7 +2873,7 @@ func xorArrayBitmap(a, b *Container) *Container { func xorBitmapBitmap(a, b *Container) *Container { output := &Container{ bitmap: make([]uint64, bitmapN), - containerType: ContainerBitmap, + containerType: containerBitmap, } for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] @@ -3078,7 +3081,7 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) { // xorArrayRun computes the exclusive or of an array and a run container. func xorArrayRun(a, b *Container) *Container { - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -3134,7 +3137,7 @@ func xorArrayRun(a, b *Container) *Container { } if output.n < ArrayMaxSize { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -3247,7 +3250,7 @@ func xorRunRun(a, b *Container) *Container { if nb == 0 { return a.Clone() } - output := &Container{containerType: ContainerRun} + output := &Container{containerType: containerRun} lastI, lastJ := -1, -1 @@ -3280,7 +3283,7 @@ func xorRunRun(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output @@ -3295,7 +3298,7 @@ func xorBitmapRun(a, b *Container) *Container { if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() - } else if len(output.runs) > RunMaxSize { + } else if len(output.runs) > runMaxSize { output.runToBitmap() } return output diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 24db8e1ca..cd22978fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -236,11 +236,11 @@ func doContainer(containerType byte, data interface{}) *Container { } switch containerType { - case ContainerArray: + case containerArray: c.array = data.([]uint16) - case ContainerBitmap: + case containerBitmap: c.bitmap = data.([]uint64) - case ContainerRun: + case containerRun: c.runs = data.([]interval16) } c.n = c.count() @@ -253,45 +253,45 @@ func setupContainerTests() map[byte]map[string]*Container { cts := make(map[byte]map[string]*Container) // array containers - cts[ContainerArray] = map[string]*Container{ - "empty": doContainer(ContainerArray, arrayEmpty()), - "full": doContainer(ContainerArray, arrayFull()), - "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), - "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), - "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), - "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), - "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), - "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), - "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), - "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), + cts[containerArray] = map[string]*Container{ + "empty": doContainer(containerArray, arrayEmpty()), + "full": doContainer(containerArray, arrayFull()), + "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(containerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), } // bitmap containers - cts[ContainerBitmap] = map[string]*Container{ - "empty": doContainer(ContainerBitmap, bitmapEmpty()), - "full": doContainer(ContainerBitmap, bitmapFull()), - "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), - "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), - "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), - "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), - "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), - "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), - "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), - "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), + cts[containerBitmap] = map[string]*Container{ + "empty": doContainer(containerBitmap, bitmapEmpty()), + "full": doContainer(containerBitmap, bitmapFull()), + "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), } // run containers - cts[ContainerRun] = map[string]*Container{ - "empty": doContainer(ContainerRun, runEmpty()), - "full": doContainer(ContainerRun, runFull()), - "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), - "lastBitSet": doContainer(ContainerRun, runLastBitSet()), - "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), - "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), - "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), - "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), - "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), - "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), + cts[containerRun] = map[string]*Container{ + "empty": doContainer(containerRun, runEmpty()), + "full": doContainer(containerRun, runFull()), + "firstBitSet": doContainer(containerRun, runFirstBitSet()), + "lastBitSet": doContainer(containerRun, runLastBitSet()), + "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(containerRun, runLastBitUnset()), + "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(containerRun, runOddBitsSet()), + "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), } return cts diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 6e1f105bf..5836782b8 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -33,7 +33,7 @@ func (c *Container) String() string { } func TestRunAppendInterval(t *testing.T) { - a := Container{containerType: ContainerRun} + a := Container{containerType: containerRun} tests := []struct { base []interval16 app interval16 @@ -82,7 +82,7 @@ func TestInterval16RunLen(t *testing.T) { } func TestContainerRunAdd(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} tests := []struct { op uint16 exp []interval16 @@ -113,7 +113,7 @@ func TestContainerRunAdd(t *testing.T) { } func TestContainerRunAdd2(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs) @@ -128,7 +128,7 @@ func TestContainerRunAdd2(t *testing.T) { } func TestRunCountRange(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} cnt := c.runCountRange(2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) @@ -181,7 +181,7 @@ func TestRunCountRange(t *testing.T) { } func TestRunContains(t *testing.T) { - c := Container{runs: make([]interval16, 0), containerType: ContainerRun} + c := Container{runs: make([]interval16, 0), containerType: containerRun} if c.runContains(5) { t.Fatalf("empty run container should not contain 5") } @@ -203,7 +203,7 @@ func TestRunContains(t *testing.T) { } func TestBitmapCountRange(t *testing.T) { - c := Container{containerType: ContainerBitmap} + c := Container{containerType: containerBitmap} tests := []struct { start int end int @@ -229,11 +229,11 @@ func TestBitmapCountRange(t *testing.T) { func TestIntersectionCountArrayBitmap3(t *testing.T) { a, b := &Container{}, &Container{} - a.containerType = ContainerBitmap + a.containerType = containerBitmap a.bitmap = getFullBitmap() a.n = maxContainerVal + 1 - b.containerType = ContainerBitmap + b.containerType = containerBitmap b.bitmap = getFullBitmap() b.n = maxContainerVal + 1 res := intersectBitmapBitmap(a, b) @@ -290,9 +290,9 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { for i, test := range tests { a.array = test.array - a.containerType = ContainerArray + a.containerType = containerArray b.bitmap = test.bitmap - b.containerType = ContainerBitmap + b.containerType = containerBitmap ret := intersectionCountArrayBitmap(a, b) if ret != test.exp { t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp) @@ -301,7 +301,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} tests := []struct { op uint16 exp []interval16 @@ -335,7 +335,7 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} + c := Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun} max := c.max() if max != 16 { t.Fatalf("max for %v should be 16", c.runs) @@ -349,8 +349,8 @@ func TestRunMax(t *testing.T) { } func TestIntersectionCountArrayRun(t *testing.T) { - a := &Container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} - b := &Container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + a := &Container{containerType: containerArray, array: []uint16{1, 5, 10, 11, 12}} + b := &Container{containerType: containerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -359,16 +359,16 @@ func TestIntersectionCountArrayRun(t *testing.T) { } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} - b := &Container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} + a := &Container{containerType: containerBitmap, bitmap: []uint64{0x8000000000000000}} + b := &Container{containerType: containerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) if ret != 1 { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) } - a = &Container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &Container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + a = &Container{containerType: containerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + b = &Container{containerType: containerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -416,8 +416,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun a.runs = test.aruns b.runs = test.bruns ret := intersectionCountRunRun(a, b) @@ -458,8 +458,8 @@ func TestIntersectArrayRun(t *testing.T) { } for i, test := range tests { - a.containerType = ContainerArray - b.containerType = ContainerRun + a.containerType = containerArray + b.containerType = containerRun a.array = test.array b.runs = test.runs ret := intersectArrayRun(a, b) @@ -516,8 +516,8 @@ func TestIntersectRunRun(t *testing.T) { }, } for i, test := range tests { - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun a.runs = test.aruns b.runs = test.bruns ret := intersectRunRun(a, b) @@ -581,8 +581,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { for i, v := range test.exp { exp[i] = v } - a.containerType = ContainerBitmap - b.containerType = ContainerRun + a.containerType = containerBitmap + b.containerType = containerRun ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -642,8 +642,8 @@ func TestIntersectBitmapRunArray(t *testing.T) { a.bitmap[i] = v } b.runs = test.runs - a.containerType = ContainerBitmap - b.containerType = ContainerRun + a.containerType = containerBitmap + b.containerType = containerRun ret := intersectBitmapRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -660,19 +660,19 @@ func TestUnionMixed(t *testing.T) { // array container a := &Container{} a.array = []uint16{1, 4, 5, 7, 10, 11, 12} - a.containerType = ContainerArray + a.containerType = containerArray a.n = 7 // bitmap container b := &Container{bitmap: make([]uint64, bitmapN)} b.bitmap[0] = uint64(0x3) b.n = 2 - b.containerType = ContainerBitmap + b.containerType = containerBitmap // run container r := &Container{} r.runs = []interval16{{start: 5, last: 10}} - r.containerType = ContainerRun + r.containerType = containerRun r.n = 6 t.Run("various container Unions", func(t *testing.T) { @@ -713,10 +713,10 @@ func TestIntersectMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = 6 - a.containerType = ContainerRun + a.containerType = containerRun b.array = []uint16{1, 4, 5, 7, 10, 11, 12} b.n = 7 - b.containerType = ContainerArray + b.containerType = containerArray res := intersect(a, b) if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) @@ -732,7 +732,7 @@ func TestIntersectMixed(t *testing.T) { } c.bitmap = []uint64{0x60} c.n = 2 - c.containerType = ContainerBitmap + c.containerType = containerBitmap res = intersect(c, a) if !reflect.DeepEqual(res.array, []uint16{5, 6}) { @@ -762,15 +762,15 @@ func TestDifferenceMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = a.runCountRange(0, 100) - a.containerType = ContainerRun + a.containerType = containerRun b.array = []uint16{0, 2, 4, 6, 8, 10, 12} b.n = len(b.array) - b.containerType = ContainerArray + b.containerType = containerArray d.array = []uint16{1, 3, 5, 7, 9, 11, 12} d.n = len(d.array) - d.containerType = ContainerArray + d.containerType = containerArray res := difference(a, b) @@ -790,7 +790,7 @@ func TestDifferenceMixed(t *testing.T) { c.bitmap = []uint64{0x64} c.n = c.countRange(0, 100) - c.containerType = ContainerBitmap + c.containerType = containerBitmap res = difference(c, a) if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) { t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) @@ -885,8 +885,8 @@ func TestUnionRunRun(t *testing.T) { for i, test := range tests { a.runs = test.aruns b.runs = test.bruns - a.containerType = ContainerRun - b.containerType = ContainerRun + a.containerType = containerRun + b.containerType = containerRun ret := unionRunRun(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -927,8 +927,8 @@ func TestUnionArrayRun(t *testing.T) { for i, test := range tests { a.array = test.array b.runs = test.runs - a.containerType = ContainerArray - b.containerType = ContainerRun + a.containerType = containerArray + b.containerType = containerRun ret := unionArrayRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -937,7 +937,7 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -977,7 +977,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} tests := []struct { array []uint16 exp []uint64 @@ -1008,7 +1008,7 @@ func TestArrayToBitmap(t *testing.T) { } func TestBitmapToArray(t *testing.T) { - a := &Container{containerType: ContainerBitmap} + a := &Container{containerType: containerBitmap} tests := []struct { bitmap []uint64 exp []uint16 @@ -1039,7 +1039,7 @@ func TestBitmapToArray(t *testing.T) { } func TestRunToBitmap(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} tests := []struct { runs []interval16 exp []uint64 @@ -1093,7 +1093,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap} + a := &Container{containerType: containerBitmap} tests := []struct { bitmap []uint64 exp []interval16 @@ -1171,7 +1171,7 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := &Container{containerType: ContainerArray} + a := &Container{containerType: containerArray} tests := []struct { array []uint16 exp []interval16 @@ -1205,7 +1205,7 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} tests := []struct { runs []interval16 exp []uint16 @@ -1239,7 +1239,7 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1283,8 +1283,8 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1322,7 +1322,7 @@ func TestUnionBitmapRun(t *testing.T) { } func TestBitmapCountRuns(t *testing.T) { - c := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 exp int @@ -1372,7 +1372,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &Container{containerType: ContainerArray} + c := &Container{containerType: containerArray} tests := []struct { array []uint16 exp int @@ -1413,8 +1413,8 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &Container{containerType: ContainerArray} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerArray} + b := &Container{containerType: containerRun} tests := []struct { array []uint16 runs []interval16 @@ -1439,8 +1439,8 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerArray} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerArray} tests := []struct { runs []interval16 array []uint16 @@ -1520,8 +1520,8 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 bitmap []uint64 @@ -1583,8 +1583,8 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1666,8 +1666,8 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := &Container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - a := &Container{containerType: ContainerArray} + b := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} + a := &Container{containerType: containerArray} tests := []struct { bitmap []uint64 array []uint16 @@ -1716,8 +1716,8 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - b := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + a := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + b := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1746,8 +1746,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -1780,7 +1780,7 @@ func TestDifferenceRunRun(t *testing.T) { } func TestWriteReadArray(t *testing.T) { - ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} + ca := &Container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: containerArray} ba := NewFileBitmap() ba.Containers.Put(0, ca) ba2 := NewFileBitmap() @@ -1800,7 +1800,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: containerBitmap} for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } @@ -1823,7 +1823,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap} + cb := &Container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: containerBitmap} for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } @@ -1852,7 +1852,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} + cr := &Container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: containerRun} br := NewFileBitmap() br.Containers.Put(0, cr) br2 := NewFileBitmap() @@ -1877,21 +1877,21 @@ func TestXorArrayRun(t *testing.T) { exp *Container }{ { - a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12}, + a: &Container{array: []uint16{1, 5, 10, 11, 12}, containerType: containerArray}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: containerArray, n: 12}, }, { - a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray}, - b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12}, + a: &Container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: containerArray}, + b: &Container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: containerRun}, + exp: &Container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: containerArray, n: 12}, }, { - a: &Container{array: []uint16{65535}, containerType: ContainerArray}, - b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{65534}, containerType: ContainerArray, n: 1}, + a: &Container{array: []uint16{65535}, containerType: containerArray}, + b: &Container{runs: []interval16{{start: 65534, last: 65535}}, containerType: containerRun}, + exp: &Container{array: []uint16{65534}, containerType: containerArray, n: 1}, }, { - a: &Container{array: []uint16{65535}, containerType: ContainerArray}, - b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, - exp: &Container{array: []uint16{}, containerType: ContainerArray, n: 0}, + a: &Container{array: []uint16{65535}, containerType: containerArray}, + b: &Container{runs: []interval16{{start: 65535, last: 65535}}, containerType: containerRun}, + exp: &Container{array: []uint16{}, containerType: containerArray, n: 0}, }, } @@ -1912,8 +1912,8 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} a.runs = []interval16{{start: 4, last: 10}} b.runs = []interval16{{start: 5, last: 10}} ret := xorRunRun(a, b) @@ -1927,8 +1927,8 @@ func TestXorRunRun1(t *testing.T) { } func TestXorRunRun(t *testing.T) { - a := &Container{containerType: ContainerRun} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerRun} + b := &Container{containerType: containerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -2025,7 +2025,7 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := &Container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + c := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} tests := []struct { bitmap []uint64 start uint64 @@ -2093,8 +2093,8 @@ func TestBitmapXorRange(t *testing.T) { } func TestXorBitmapRun(t *testing.T) { - a := &Container{containerType: ContainerBitmap} - b := &Container{containerType: ContainerRun} + a := &Container{containerType: containerBitmap} + b := &Container{containerType: containerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -2546,8 +2546,8 @@ func TestSearch64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &Container{containerType: ContainerArray}, &Container{ - containerType: ContainerBitmap, + a, b := &Container{containerType: containerArray}, &Container{ + containerType: containerBitmap, bitmap: make([]uint64, bitmapN), } tests := []struct { @@ -2594,11 +2594,11 @@ func TestIntersectArrayBitmap(t *testing.T) { for i, test := range tests { a.array = test.array - a.containerType = ContainerArray + a.containerType = containerArray for i, bmval := range test.bitmap { b.bitmap[i] = bmval } - b.containerType = ContainerBitmap + b.containerType = containerBitmap ret := intersectArrayBitmap(a, b).array if len(ret) == 0 && len(test.exp) == 0 { continue @@ -2722,13 +2722,13 @@ func TestContainerCombinations(t *testing.T) { cts := setupContainerTests() - containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + containerTypes := []byte{containerArray, containerBitmap, containerRun} // map used for a more descriptive print cm := map[byte]string{ - ContainerArray: "array", - ContainerBitmap: "bitmap", - ContainerRun: "run", + containerArray: "array", + containerBitmap: "bitmap", + containerRun: "run", } testOps := []testOp{ @@ -3198,7 +3198,7 @@ func TestContainerCombinations(t *testing.T) { // Convert to all container types and check result. for _, ct := range containerTypes { clone := ret.Clone() - if ct == ContainerArray { + if ct == containerArray { if clone.isBitmap() { clone.bitmapToArray() } else if clone.isRun() { @@ -3212,7 +3212,7 @@ func TestContainerCombinations(t *testing.T) { if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) } - } else if ct == ContainerBitmap { + } else if ct == containerBitmap { if clone.isArray() { clone.arrayToBitmap() } else if clone.isRun() { @@ -3224,7 +3224,7 @@ func TestContainerCombinations(t *testing.T) { if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) } - } else if ct == ContainerRun { + } else if ct == containerRun { if clone.isArray() { clone.arrayToRun() } else if clone.isBitmap() { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c1291d1f7..d29f1226f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -471,10 +471,10 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.ShardWidth+5, pilosa.ShardWidth+7) + bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.ShardWidth+1) result := bm0.Difference(bm1) - if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { + if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.ShardWidth + 5, pilosa.ShardWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) } } @@ -1161,7 +1161,7 @@ func BenchmarkContainerLinear(b *testing.B) { bm := roaring.NewFileBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1172,7 +1172,7 @@ func BenchmarkContainerReverse(b *testing.B) { bm := roaring.NewFileBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1183,7 +1183,7 @@ func BenchmarkContainerColumn(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1196,8 +1196,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) { for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1209,8 +1209,8 @@ func BenchmarkContainerInsideOut(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { - bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1219,7 +1219,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(0); col < pilosa.SliceWidth; col++ { + for col := uint64(0); col < pilosa.ShardWidth; col++ { bm.Add(col) } } @@ -1228,7 +1228,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { + for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { bm.Add(col) } } diff --git a/row.go b/row.go index ec2c42726..38ba24f9b 100644 --- a/row.go +++ b/row.go @@ -18,14 +18,16 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/roaring" ) // Row is a set of integers (the associated columns), and attributes which are // arbitrary key/value pairs storing metadata about what the row represents. type Row struct { - segments []RowSegment + segments []rowSegment + + // String keys translated to/from segment columns. + Keys []string // Attributes associated with the row. Attrs map[string]interface{} @@ -42,7 +44,7 @@ func NewRow(columns ...uint64) *Row { // Merge merges data from other into r. func (r *Row) Merge(other *Row) { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -61,11 +63,11 @@ func (r *Row) Merge(other *Row) { } r.segments = segments - r.InvalidateCount() + r.invalidateCount() } -// IntersectionCount returns the number of intersections between r and other. -func (r *Row) IntersectionCount(other *Row) uint64 { +// intersectionCount returns the number of intersections between r and other. +func (r *Row) intersectionCount(other *Row) uint64 { var n uint64 itr := newMergeSegmentIterator(r.segments, other.segments) @@ -80,9 +82,9 @@ func (r *Row) IntersectionCount(other *Row) uint64 { return n } -// Intersect returns the itersection of r and other. -func (r *Row) Intersect(other *Row) *Row { - var segments []RowSegment +// intersect returns the itersection of r and other. +func (r *Row) intersect(other *Row) *Row { + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -98,7 +100,7 @@ func (r *Row) Intersect(other *Row) *Row { // Xor returns the xor of r and other. func (r *Row) Xor(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -118,7 +120,7 @@ func (r *Row) Xor(other *Row) *Row { // Union returns the bitwise union of r and other. func (r *Row) Union(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { if s1 == nil { @@ -136,7 +138,7 @@ func (r *Row) Union(other *Row) *Row { // Difference returns the diff of r and other. func (r *Row) Difference(other *Row) *Row { - var segments []RowSegment + var segments []rowSegment itr := newMergeSegmentIterator(r.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { @@ -154,79 +156,65 @@ func (r *Row) Difference(other *Row) *Row { // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { - return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i) + return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } -// ClearBit clears the i-th column of the row. -func (r *Row) ClearBit(i uint64) (changed bool) { - s := r.segment(i / SliceWidth) +// clearBit clears the i-th column of the row. +func (r *Row) clearBit(i uint64) (changed bool) { + s := r.segment(i / ShardWidth) if s == nil { return false } return s.ClearBit(i) } -// segment returns a segment for a given slice. +// Segments returns a list of all segments in the row. +func (r *Row) Segments() []rowSegment { + return r.segments +} + +// segment returns a segment for a given shard. // Returns nil if segment does not exist. -func (r *Row) segment(slice uint64) *RowSegment { +func (r *Row) segment(shard uint64) *rowSegment { if i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice - }); i < len(r.segments) && r.segments[i].slice == slice { + return r.segments[i].shard >= shard + }); i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } return nil } -func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment { +func (r *Row) createSegmentIfNotExists(shard uint64) *rowSegment { i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice + return r.segments[i].shard >= shard }) // Return exact match. - if i < len(r.segments) && r.segments[i].slice == slice { + if i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } // Insert new segment. - r.segments = append(r.segments, RowSegment{data: *roaring.NewBitmap()}) + r.segments = append(r.segments, rowSegment{data: *roaring.NewBitmap()}) if i < len(r.segments) { copy(r.segments[i+1:], r.segments[i:]) } - r.segments[i] = RowSegment{ + r.segments[i] = rowSegment{ data: *roaring.NewBitmap(), - slice: slice, + shard: shard, writable: true, } return &r.segments[i] } -// InvalidateCount updates the cached count in the row. -func (r *Row) InvalidateCount() { +// invalidateCount updates the cached count in the row. +func (r *Row) invalidateCount() { for i := range r.segments { r.segments[i].InvalidateCount() } } -// IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased. -func (r *Row) IncrementCount(i uint64) { - seg := r.segment(i / SliceWidth) - if seg != nil { - seg.n++ - } - -} - -// DecrementCount decrements the row cached counter. -func (r *Row) DecrementCount(i uint64) { - seg := r.segment(i / SliceWidth) - if seg != nil { - if seg.n > 0 { - seg.n-- - } - } -} - // Count returns the number of columns in the row. func (r *Row) Count() uint64 { var n uint64 @@ -241,8 +229,10 @@ func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` Columns []uint64 `json:"columns"` + Keys []string `json:"keys,omitempty"` } o.Columns = r.Columns() + o.Keys = r.Keys o.Attrs = r.Attrs if o.Attrs == nil { @@ -261,47 +251,12 @@ func (r *Row) Columns() []uint64 { return a } -// encodeRow converts r into its internal representation. -func encodeRow(r *Row) *internal.Row { - if r == nil { - return nil - } - - return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), - } -} - -// decodeRow converts r from its internal representation. -func decodeRow(pr *internal.Row) *Row { - if pr == nil { - return nil - } - - r := NewRow() - r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Columns { - r.SetBit(v) - } - return r -} - -// Union performs a union on a slice of rows. -func Union(rows []*Row) *Row { - other := rows[0] - for _, r := range rows[1:] { - other = other.Union(r) - } - return other -} - -// RowSegment holds a subset of a row. +// rowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The -// width of the segment will always match the slice width. -type RowSegment struct { - // Slice this segment belongs to - slice uint64 +// width of the segment will always match the shard width. +type rowSegment struct { + // Shard this segment belongs to + shard uint64 // Underlying raw bitmap implementation. // This is an mmapped bitmap if writable is false. Otherwise @@ -315,7 +270,7 @@ type RowSegment struct { // Merge adds chunks from other to s. // Chunks in s are overwritten if they exist in other. -func (s *RowSegment) Merge(other *RowSegment) { +func (s *rowSegment) Merge(other *rowSegment) { s.ensureWritable() itr := other.data.Iterator() @@ -325,56 +280,56 @@ func (s *RowSegment) Merge(other *RowSegment) { } // IntersectionCount returns the number of intersections between s and other. -func (s *RowSegment) IntersectionCount(other *RowSegment) uint64 { +func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { return s.data.IntersectionCount(&other.data) } // Intersect returns the itersection of s and other. -func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { +func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { data := s.data.Intersect(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } // Union returns the bitwise union of s and other. -func (s *RowSegment) Union(other *RowSegment) *RowSegment { +func (s *rowSegment) Union(other *rowSegment) *rowSegment { data := s.data.Union(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } // Difference returns the diff of s and other. -func (s *RowSegment) Difference(other *RowSegment) *RowSegment { +func (s *rowSegment) Difference(other *rowSegment) *rowSegment { data := s.data.Difference(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } // Xor returns the xor of s and other. -func (s *RowSegment) Xor(other *RowSegment) *RowSegment { +func (s *rowSegment) Xor(other *rowSegment) *rowSegment { data := s.data.Xor(&other.data) - return &RowSegment{ + return &rowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } // SetBit sets the i-th column of the row. -func (s *RowSegment) SetBit(i uint64) (changed bool) { +func (s *rowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() changed, _ = s.data.Add(i) if changed { @@ -384,7 +339,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) { } // ClearBit clears the i-th column of the row. -func (s *RowSegment) ClearBit(i uint64) (changed bool) { +func (s *rowSegment) ClearBit(i uint64) (changed bool) { s.ensureWritable() changed, _ = s.data.Remove(i) @@ -395,12 +350,12 @@ func (s *RowSegment) ClearBit(i uint64) (changed bool) { } // InvalidateCount updates the cached count in the row. -func (s *RowSegment) InvalidateCount() { +func (s *rowSegment) InvalidateCount() { s.n = s.data.Count() } // Columns returns a list of all columns set in the segment. -func (s *RowSegment) Columns() []uint64 { +func (s *rowSegment) Columns() []uint64 { a := make([]uint64, 0, s.Count()) itr := s.data.Iterator() for v, eof := itr.Next(); !eof; v, eof = itr.Next() { @@ -410,10 +365,10 @@ func (s *RowSegment) Columns() []uint64 { } // Count returns the number of set columns in the row. -func (s *RowSegment) Count() uint64 { return s.n } +func (s *rowSegment) Count() uint64 { return s.n } // ensureWritable clones the segment if it is pointing to non-writable data. -func (s *RowSegment) ensureWritable() { +func (s *rowSegment) ensureWritable() { if s.writable { return } @@ -424,16 +379,16 @@ func (s *RowSegment) ensureWritable() { // mergeSegmentIterator produces an iterator that loops through two sets of segments. type mergeSegmentIterator struct { - a0, a1 []RowSegment + a0, a1 []rowSegment } // newMergeSegmentIterator returns a new instance of mergeSegmentIterator. -func newMergeSegmentIterator(a0, a1 []RowSegment) mergeSegmentIterator { +func newMergeSegmentIterator(a0, a1 []rowSegment) mergeSegmentIterator { return mergeSegmentIterator{a0: a0, a1: a1} } // next returns the next set of segments. -func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) { +func (itr *mergeSegmentIterator) next() (s0, s1 *rowSegment) { // Find current segments. if len(itr.a0) > 0 { s0 = &itr.a0[0] @@ -454,15 +409,15 @@ func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) { } // Otherwise determine which is first. - if s0.slice < s1.slice { + if s0.shard < s1.shard { itr.a0 = itr.a0[1:] return s0, nil - } else if s0.slice > s1.slice { + } else if s0.shard > s1.shard { itr.a1 = itr.a1[1:] return s1, nil } - // Return both if slices are equal. + // Return both if shards are equal. itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:] return s0, s1 } diff --git a/row_test.go b/row_test.go index bc1cc0c68..7f1279ceb 100644 --- a/row_test.go +++ b/row_test.go @@ -30,7 +30,7 @@ func TestRow_Merge(t *testing.T) { exp uint64 }{ { - r1: pilosa.NewRow(1, 2, 3, SliceWidth+1, 2*SliceWidth), + r1: pilosa.NewRow(1, 2, 3, ShardWidth+1, 2*ShardWidth), r2: pilosa.NewRow(3, 4, 5), exp: 7, }, @@ -56,9 +56,9 @@ func TestRow_Merge(t *testing.T) { // Ensure a row can Xor'ed func TestRow_Xor(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth, 2 * ShardWidth} res := r1.Xor(r2) if res.Count() != 3 { @@ -78,9 +78,9 @@ func TestRow_Xor(t *testing.T) { } func TestRow_Union_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{0, 1, ShardWidth, 2 * ShardWidth} res := r1.Union(r2) if res.Count() != 4 { @@ -99,9 +99,9 @@ func TestRow_Union_Segment(t *testing.T) { } func TestRow_Difference_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth} res := r1.Difference(r2) if res.Count() != 2 { diff --git a/server.go b/server.go index 320fd174c..91c2d813a 100644 --- a/server.go +++ b/server.go @@ -18,8 +18,6 @@ import ( "context" "fmt" "log" - "net" - "net/http" "os" "os/exec" "path/filepath" @@ -29,8 +27,6 @@ import ( "sync" "time" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -38,13 +34,11 @@ import ( // Default server settings. const ( - DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" + defaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Ensure Server implements interfaces. -var _ Broadcaster = &Server{} -var _ BroadcastHandler = &Server{} -var _ StatusHandler = &Server{} +var _ broadcaster = &Server{} // Server represents a holder wrapped by a running HTTP server. type Server struct { @@ -53,34 +47,40 @@ type Server struct { closing chan struct{} // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - executor *Executor + holder *Holder + cluster *cluster + translateFile *TranslateFile + diagnostics *diagnosticsCollector + executor *executor + hosts []string + clusterDisabled bool + serializer Serializer // External - handler *Handler - Broadcaster Broadcaster - BroadcastReceiver BroadcastReceiver - Gossiper Gossiper - remoteClient *http.Client - systemInfo SystemInfo - gcNotifier GCNotifier - NewAttrStore func(string) AttrStore - logger Logger - ln net.Listener + systemInfo SystemInfo + gcNotifier GCNotifier + logger Logger - NodeID string - URI URI + nodeID string + uri URI antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration maxWritesPerRequest int + isCoordinator bool + syncer holderSyncer + + primaryTranslateStore TranslateStore defaultClient InternalClient dataDir string } +// TODO: have this return an interface for Holder instead of concrete object? +func (s *Server) Holder() *Holder { + return s.holder +} + // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error @@ -93,7 +93,7 @@ func OptServerLogger(l Logger) ServerOption { func OptServerReplicaN(n int) ServerOption { return func(s *Server) error { - s.Cluster.ReplicaN = n + s.cluster.ReplicaN = n return nil } } @@ -107,8 +107,7 @@ func OptServerDataDir(dir string) ServerOption { func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { return func(s *Server) error { - s.NewAttrStore = af - s.Holder.NewAttrStore = af + s.holder.NewAttrStore = af return nil } } @@ -122,14 +121,7 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { func OptServerLongQueryTime(dur time.Duration) ServerOption { return func(s *Server) error { - s.Cluster.LongQueryTime = dur - return nil - } -} - -func OptServerHandler(h *Handler) ServerOption { - return func(s *Server) error { - s.handler = h + s.cluster.longQueryTime = dur return nil } } @@ -162,19 +154,25 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } -func OptServerRemoteClient(c *http.Client) ServerOption { +func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { - s.executor = NewExecutor(c) - s.remoteClient = c - s.defaultClient = NewInternalHTTPClientFromURI(nil, c) - s.Cluster.RemoteClient = c + s.executor = newExecutor(optExecutorInternalQueryClient(c)) + s.defaultClient = c + s.cluster.InternalClient = c + return nil + } +} + +func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { + return func(s *Server) error { + s.primaryTranslateStore = store return nil } } func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { - s.Holder.Stats = sc + s.holder.Stats = sc return nil } } @@ -186,41 +184,62 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } -func OptServerListener(ln net.Listener) ServerOption { +func OptServerURI(uri *URI) ServerOption { return func(s *Server) error { - s.ln = ln - + s.uri = *uri return nil } } -func OptServerURI(uri *URI) ServerOption { +// OptClusterDisabled tells the server whether to use a static cluster with the +// defined hosts. Mostly used for testing. +func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { return func(s *Server) error { - s.URI = *uri + s.hosts = hosts + s.clusterDisabled = disabled + return nil + } +} + +func OptServerSerializer(ser Serializer) ServerOption { + return func(s *Server) error { + s.serializer = ser + return nil + } +} + +func OptServerIsCoordinator(is bool) ServerOption { + return func(s *Server) error { + s.isCoordinator = is + return nil + } +} + +func OptServerNodeID(nodeID string) ServerOption { + return func(s *Server) error { + s.nodeID = nodeID + return nil + } +} + +func OptServerClusterHasher(h Hasher) ServerOption { + return func(s *Server) error { + s.cluster.Hasher = h return nil } } // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { - handler, err := NewHandler() - if err != nil { - return nil, errors.Wrap(err, "initializing handler") - } s := &Server{ - closing: make(chan struct{}), - Cluster: NewCluster(), - Holder: NewHolder(), - handler: handler, - Broadcaster: NopBroadcaster, - BroadcastReceiver: NopBroadcastReceiver, - diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), - systemInfo: NewNopSystemInfo(), + closing: make(chan struct{}), + cluster: newCluster(), + holder: NewHolder(), + diagnostics: newDiagnosticsCollector(defaultDiagnosticServer), + systemInfo: newNopSystemInfo(), gcNotifier: NopGCNotifier, - NewAttrStore: NewNopAttrStore, - antiEntropyInterval: time.Minute * 10, metricInterval: 0, diagnosticInterval: 0, @@ -241,34 +260,55 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, err } - s.Holder.Path = path - s.Holder.Logger = s.logger - s.Holder.Stats.SetLogger(s.logger) + s.holder.Path = path + s.holder.Logger = s.logger + s.holder.Stats.SetLogger(s.logger) - s.Cluster.Path = path - s.Cluster.Logger = s.logger - s.Cluster.Holder = s.Holder + s.cluster.Path = path + s.cluster.logger = s.logger + s.cluster.holder = s.holder - // update URI port with actual listener port. TODO this should probably be done outside of here. - if s.URI.Port() == 0 { - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + // Initialize translation database. + s.translateFile = NewTranslateFile() + s.translateFile.Path = filepath.Join(path, ".keys") + s.translateFile.PrimaryTranslateStore = s.primaryTranslateStore + + // Get or create NodeID. + s.nodeID = s.loadNodeID() + if s.isCoordinator { + s.cluster.Coordinator = s.nodeID } - s.NodeID = s.LoadNodeID() // Set Cluster Node. node := &Node{ - ID: s.NodeID, - URI: s.URI, - IsCoordinator: s.Cluster.Coordinator == s.NodeID, + ID: s.nodeID, + URI: s.uri, + IsCoordinator: s.cluster.Coordinator == s.nodeID, + } + s.cluster.Node = node + if s.clusterDisabled { + err := s.cluster.setStatic(s.hosts) + if err != nil { + return nil, errors.Wrap(err, "setting cluster static") + } } - s.Cluster.Node = node - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) - s.executor.Holder = s.Holder + // Append the NodeID tag to stats. + s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.nodeID)) + + s.executor.Holder = s.holder s.executor.Node = node - s.executor.Cluster = s.Cluster + s.executor.Cluster = s.cluster + s.executor.TranslateStore = s.translateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest - s.handler.API.Executor = s.executor + s.cluster.broadcaster = s + s.cluster.maxWritesPerRequest = s.maxWritesPerRequest + s.holder.broadcaster = s + + err = s.cluster.setup() + if err != nil { + return nil, errors.Wrap(err, "setting up cluster") + } return s, nil } @@ -276,67 +316,28 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Open opens and initializes the server. func (s *Server) Open() error { s.logger.Printf("open server") - // s.ln can be configured prior to Open() via s.OpenListener(). - if s.ln == nil { - return errors.New("Must pass a listener option to NewServer") - } // Log startup - err := s.Holder.logStartup() + err := s.holder.logStartup() if err != nil { log.Println(errors.Wrap(err, "logging startup")) } - // Get or create NodeID. - - // Append the NodeID tag to stats. - - // Create default HTTP client - - // Create executor for executing queries. - - // Cluster settings. - s.Cluster.Broadcaster = s.Broadcaster - s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest - - // Initialize HTTP handler. - s.handler.API.Holder = s.Holder - s.handler.API.Broadcaster = s.Broadcaster - s.handler.API.BroadcastHandler = s - s.handler.API.StatusHandler = s - s.handler.API.Cluster = s.Cluster - - // Initialize Holder. - s.Holder.Broadcaster = s.Broadcaster - - // Serve HTTP. - go func() { - server := &http.Server{Handler: s.handler} - go func() { - <-s.closing - server.Close() - }() - err := server.Serve(s.ln) - if err != nil && err.Error() != "http: Server closed" { - s.logger.Printf("HTTP handler terminated with error: %s\n", err) - } - }() - - // Start the BroadcastReceiver. - if err := s.BroadcastReceiver.Start(s); err != nil { - return fmt.Errorf("starting BroadcastReceiver: %v", err) + // Initialize id-key storage. + if err := s.translateFile.Open(); err != nil { + return err } // Open Cluster management. - if err := s.Cluster.Open(); err != nil { + if err := s.cluster.waitForStarted(); err != nil { return fmt.Errorf("opening Cluster: %v", err) } // Open holder. - if err := s.Holder.Open(); err != nil { + if err := s.holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + if err := s.cluster.setNodeState(nodeStateReady); err != nil { return fmt.Errorf("setting nodeState: %v", err) } @@ -345,7 +346,13 @@ func (s *Server) Open() error { // the cluster without waiting for data to load on the coordinator. Before // this starts, the joins are queued up in the Cluster.joiningLeavingNodes // buffered channel. - s.Cluster.ListenForJoins() + s.cluster.listenForJoins() + + s.syncer.Holder = s.holder + s.syncer.Node = s.cluster.Node + s.syncer.Cluster = s.cluster + s.syncer.Closing = s.closing + s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") // Start background monitoring. s.wg.Add(3) @@ -362,69 +369,62 @@ func (s *Server) Close() error { close(s.closing) s.wg.Wait() - if s.ln != nil { - s.ln.Close() + if s.cluster != nil { + s.cluster.close() } - if s.Cluster != nil { - s.Cluster.Close() + if s.holder != nil { + s.holder.Close() } - if s.Holder != nil { - s.Holder.Close() + if s.translateFile != nil { + s.translateFile.Close() } return nil } -// LoadNodeID gets NodeID from disk, or creates a new value. +// loadNodeID gets NodeID from disk, or creates a new value. // If server.NodeID is already set, a new ID is not created. -func (s *Server) LoadNodeID() string { - if s.NodeID != "" { - return s.NodeID +func (s *Server) loadNodeID() string { + if s.nodeID != "" { + return s.nodeID } - nodeID, err := s.Holder.loadNodeID() + nodeID, err := s.holder.loadNodeID() if err != nil { s.logger.Printf("loading NodeID: %v", err) - return s.NodeID + return s.nodeID } return nodeID } -// Addr returns the address of the listener. -func (s *Server) Addr() net.Addr { - if s.ln == nil { - return nil - } - return s.ln.Addr() +// SyncData manually invokes the anti entropy process which makes sure that this +// node has the data from all replicas across the cluster. +func (s *Server) SyncData() error { + return errors.Wrap(s.syncer.SyncHolder(), "syncing holder") } func (s *Server) monitorAntiEntropy() { + if s.antiEntropyInterval == 0 { + return // anti entropy disabled + } ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) + // Initialize syncer with local holder and remote client. for { // Wait for tick or a close. select { case <-s.closing: return case <-ticker.C: - s.Holder.Stats.Count("AntiEntropy", 1, 1.0) + s.holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.logger.Printf("holder sync beginning") - - // Initialize syncer with local holder and remote client. - var syncer HolderSyncer - syncer.Holder = s.Holder - syncer.Node = s.Cluster.Node - syncer.Cluster = s.Cluster - syncer.Closing = s.closing - syncer.RemoteClient = s.remoteClient - syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. - if err := syncer.SyncHolder(); err != nil { + s.logger.Printf("holder sync beginning") + if err := s.syncer.SyncHolder(); err != nil { s.logger.Printf("holder sync error: err=%s", err) continue } @@ -432,119 +432,115 @@ func (s *Server) monitorAntiEntropy() { // Record successful sync in log. s.logger.Printf("holder sync complete") dif := time.Since(t) - s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) + s.holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } } -// ReceiveMessage represents an implementation of BroadcastHandler. -func (s *Server) ReceiveMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateSliceMessage: - idx := s.Holder.Index(obj.Index) +// receiveMessage represents an implementation of BroadcastHandler. +func (s *Server) receiveMessage(m Message) error { + switch obj := m.(type) { + case *CreateShardMessage: + idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - idx.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateIndexMessage: + idx.setRemoteMaxShard(obj.Shard) + case *CreateIndexMessage: opt := IndexOptions{} - _, err := s.Holder.CreateIndex(obj.Index, opt) + _, err := s.holder.CreateIndex(obj.Index, opt) if err != nil { return err } - case *internal.DeleteIndexMessage: - if err := s.Holder.DeleteIndex(obj.Index); err != nil { + case *DeleteIndexMessage: + if err := s.holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFrameMessage: - idx := s.Holder.Index(obj.Index) + case *CreateFieldMessage: + idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFrameOptions(obj.Meta) - _, err := idx.CreateFrame(obj.Frame, *opt) + opt := obj.Meta + _, err := idx.createField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFrameMessage: - idx := s.Holder.Index(obj.Index) - if err := idx.DeleteFrame(obj.Frame); err != nil { + case *DeleteFieldMessage: + idx := s.holder.Index(obj.Index) + if err := idx.DeleteField(obj.Field); err != nil { return err } - case *internal.CreateFieldMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - field := decodeField(obj.Field) - if err := f.CreateField(field); err != nil { - return err - } - case *internal.DeleteFieldMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - if err := f.DeleteField(obj.Field); err != nil { - return err - } - case *internal.CreateViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + case *CreateViewMessage: + f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { return err } - case *internal.DeleteViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + case *DeleteViewMessage: + f := s.holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } - err := f.DeleteView(obj.View) + err := f.deleteView(obj.View) if err != nil { return err } - case *internal.ClusterStatus: - err := s.Cluster.MergeClusterStatus(obj) + case *ClusterStatus: + err := s.cluster.mergeClusterStatus(obj) if err != nil { return err } - case *internal.ResizeInstruction: - err := s.Cluster.FollowResizeInstruction(obj) + case *ResizeInstruction: + err := s.cluster.followResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: - err := s.Cluster.MarkResizeInstructionComplete(obj) + case *ResizeInstructionComplete: + err := s.cluster.markResizeInstructionComplete(obj) if err != nil { return err } - case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeNode(obj.New)) - case *internal.UpdateCoordinatorMessage: - s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) - case *internal.NodeStateMessage: - err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) + case *SetCoordinatorMessage: + s.cluster.setCoordinator(obj.New) + case *UpdateCoordinatorMessage: + s.cluster.updateCoordinator(obj.New) + case *NodeStateMessage: + err := s.cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } - case *internal.RecalculateCaches: - s.Holder.RecalculateCaches() - case *internal.NodeEventMessage: - s.Cluster.ReceiveEvent(DecodeNodeEvent(obj)) + case *RecalculateCaches: + s.holder.recalculateCaches() + case *NodeEvent: + s.cluster.ReceiveEvent(obj) + case *NodeStatus: + s.handleRemoteStatus(obj) } return nil } // SendSync represents an implementation of Broadcaster. -func (s *Server) SendSync(pb proto.Message) error { +func (s *Server) SendSync(m Message) error { var eg errgroup.Group - for _, node := range s.Cluster.Nodes { + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) + for _, node := range s.cluster.Nodes { node := node s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. - if s.URI == node.URI { + if s.uri == node.URI { continue } eg.Go(func() error { - return s.defaultClient.SendMessage(context.Background(), &node.URI, pb) + return s.defaultClient.SendMessage(context.Background(), &node.URI, msg) }) } @@ -552,92 +548,69 @@ func (s *Server) SendSync(pb proto.Message) error { } // SendAsync represents an implementation of Broadcaster. -func (s *Server) SendAsync(pb proto.Message) error { - return s.Gossiper.SendAsync(pb) +func (s *Server) SendAsync(m Message) error { + return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. -func (s *Server) SendTo(to *Node, pb proto.Message) error { +func (s *Server) SendTo(to *Node, m Message) error { s.logger.Printf("SendTo: %s", to.URI) - return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) + msg, err := s.serializer.Marshal(m) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + msg = append([]byte{getMessageType(m)}, msg...) + return s.defaultClient.SendMessage(context.Background(), &to.URI, msg) } -// Server implements StatusHandler. -// LocalStatus is used to periodically sync information -// between nodes. Under normal conditions, nodes should -// remain in sync through Broadcast messages. For cases -// where a node fails to receive a Broadcast message, or -// when a new (empty) node needs to get in sync with the -// rest of the cluster, two things are shared via gossip: -// - MaxSlice by Index -// - Schema -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) LocalStatus() (proto.Message, error) { - if s.Cluster == nil { - return nil, errors.New("Server.Cluster is nil") - } - if s.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Node: EncodeNode(s.Cluster.Node), - MaxSlices: s.Holder.EncodeMaxSlices(), - Schema: s.Holder.EncodeSchema(), - } - - return &ns, nil +// node returns the pilosa.node object. It is used by membership protocols to +// get this node's name(ID), location(URI), and coordinator status. +func (s *Server) node() Node { + return *s.cluster.Node } -// ClusterStatus returns the ClusterState and NodeSet for the cluster. -func (s *Server) ClusterStatus() (proto.Message, error) { - return s.Cluster.Status(), nil -} - -// HandleRemoteStatus receives incoming NodeStatus from remote nodes. -func (s *Server) HandleRemoteStatus(pb proto.Message) error { +// handleRemoteStatus receives incoming NodeStatus from remote nodes. +func (s *Server) handleRemoteStatus(pb Message) { // Ignore NodeStatus messages until the cluster is in a Normal state. - if s.Cluster.State() != ClusterStateNormal { - return nil + if s.cluster.State() != ClusterStateNormal { + return } go func() { // Make sure the holder has opened. - <-s.Holder.opened + <-s.holder.opened - err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) + err := s.mergeRemoteStatus(pb.(*NodeStatus)) if err != nil { s.logger.Printf("merge remote status: %s", err) } }() - - return nil } -func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { +func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { // Ignore status updates from self. - if s.NodeID == DecodeNode(ns.Node).ID { + if s.nodeID == ns.Node.ID { return nil } // Sync schema. - if err := s.Holder.ApplySchema(ns.Schema); err != nil { + if err := s.holder.applySchema(ns.Schema); err != nil { return errors.Wrap(err, "applying schema") } - // Sync maxSlices. - oldmaxslices := s.Holder.MaxSlices() - for index, newMax := range ns.MaxSlices.Standard { - localIndex := s.Holder.Index(index) + // Sync maxShards. + oldmaxshards := s.holder.maxShards() + for index, newMax := range ns.MaxShards { + localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation + // indexes should be created and synced prior to shard creation if localIndex == nil { s.logger.Printf("Local Index not found: %s", index) continue } - if newMax > oldmaxslices[index] { - oldmaxslices[index] = newMax - localIndex.SetRemoteMaxSlice(newMax) + if newMax > oldmaxshards[index] { + oldmaxshards[index] = newMax + localIndex.setRemoteMaxShard(newMax) } } @@ -656,17 +629,17 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) - s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) - s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) + s.diagnostics.Set("Host", s.uri.Host) + s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ",")) + s.diagnostics.Set("NumNodes", len(s.cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) - s.diagnostics.Set("NodeID", s.NodeID) - s.diagnostics.Set("ClusterID", s.Cluster.ID) + s.diagnostics.Set("NodeID", s.nodeID) + s.diagnostics.Set("ClusterID", s.cluster.id) s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } @@ -716,31 +689,31 @@ func (s *Server) monitorRuntime() { return case <-s.gcNotifier.AfterGC(): // GC just ran. - s.Holder.Stats.Count("garbage_collection", 1, 1.0) + s.holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: } // Record the number of go routines. - s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) + s.holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() // Open File handles. if err == nil { - s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) + s.holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) } // Runtime memory metrics. runtime.ReadMemStats(&m) - s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) - s.Holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) - s.Holder.Stats.Gauge("StackInuse", float64(m.StackInuse), 1.0) - s.Holder.Stats.Gauge("Mallocs", float64(m.Mallocs), 1.0) - s.Holder.Stats.Gauge("Frees", float64(m.Frees), 1.0) + s.holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0) + s.holder.Stats.Gauge("HeapInuse", float64(m.HeapInuse), 1.0) + s.holder.Stats.Gauge("StackInuse", float64(m.StackInuse), 1.0) + s.holder.Stats.Gauge("Mallocs", float64(m.Mallocs), 1.0) + s.holder.Stats.Gauge("Frees", float64(m.Frees), 1.0) } } -// CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() (int, error) { +// countOpenFiles on operating systems that support lsof. +func countOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks @@ -754,21 +727,12 @@ func CountOpenFiles() (int, error) { return len(lines), nil case "windows": // TODO: count open file handles on windows - return 0, errors.New("CountOpenFiles() on Windows is not supported") + return 0, errors.New("countOpenFiles() on Windows is not supported") default: - return 0, errors.New("CountOpenFiles() on this OS is not supported") + return 0, errors.New("countOpenFiles() on this OS is not supported") } } -// StatusHandler specifies the methods which an object must implement to share -// state in the cluster. These are used by the GossipMemberSet to implement the -// LocalState and MergeRemoteState methods of memberlist.Delegate -type StatusHandler interface { - LocalStatus() (proto.Message, error) - ClusterStatus() (proto.Message, error) - HandleRemoteStatus(proto.Message) error -} - func expandDirName(path string) (string, error) { prefix := "~" + string(filepath.Separator) if strings.HasPrefix(path, prefix) { diff --git a/server/cluster_test.go b/server/cluster_test.go index e6777d90c..aff676ae0 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -24,25 +24,19 @@ import ( "testing" "time" - "golang.org/x/sync/errgroup" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" + "golang.org/x/sync/errgroup" ) // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { - ms := test.MustRunMainWithCluster(t, 2) + ms := test.MustRunCluster(t, 2) m0, m1 := ms[0], ms[1] defer m0.Close() defer m1.Close() - m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) - m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - // Expected indexes and Frames + // Expected indexes and Fields expected := map[string][]string{ "i": []string{"f"}, } @@ -51,14 +45,14 @@ func TestMain_SendReceiveMessage(t *testing.T) { client0 := m0.Client() client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { t.Fatal(err) } - // Make sure node0 knows about the index and frame created. + // Make sure node0 knows about the index and field created. schema0, err := client0.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -66,15 +60,15 @@ func TestMain_SendReceiveMessage(t *testing.T) { received0 := map[string][]string{} for _, idx := range schema0 { received0[idx.Name] = []string{} - for _, frame := range idx.Frames { - received0[idx.Name] = append(received0[idx.Name], frame.Name) + for _, field := range idx.Fields { + received0[idx.Name] = append(received0[idx.Name], field.Name) } } if !reflect.DeepEqual(received0, expected) { t.Fatalf("unexpected schema on node0: %s", received0) } - // Make sure node1 knows about the index and frame created. + // Make sure node1 knows about the index and field created. schema1, err := client1.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -82,8 +76,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { received1 := map[string][]string{} for _, idx := range schema1 { received1[idx.Name] = []string{} - for _, frame := range idx.Frames { - received1[idx.Name] = append(received1[idx.Name], frame.Name) + for _, field := range idx.Fields { + received1[idx.Name] = append(received1[idx.Name], field.Name) } } if !reflect.DeepEqual(received1, expected) { @@ -92,8 +86,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + Set(1, f=1) + Set(2400000, f=1) `); err != nil { t.Fatal(err) } @@ -101,242 +95,172 @@ func TestMain_SendReceiveMessage(t *testing.T) { // We have to wait for the broadcast message to be sent before checking state. time.Sleep(1 * time.Second) - // Make sure node0 knows about the latest MaxSlice. - maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + // Make sure node0 knows about the latest MaxShard. + maxShards0, err := client0.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices0["i"] != 2 { - t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + if maxShards0["i"] != 2 { + t.Fatalf("unexpected maxShard on node0: %d", maxShards0["i"]) } - // Make sure node1 knows about the latest MaxSlice. - maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + // Make sure node1 knows about the latest MaxShard. + maxShards1, err := client1.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices1["i"] != 2 { - t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + if maxShards1["i"] != 2 { + t.Fatalf("unexpected maxShard on node1: %d", maxShards1["i"]) } } // Ensure that an empty node comes up in a NORMAL state. func TestClusterResize_EmptyNode(t *testing.T) { - m0 := test.MustRunMain() + m0 := test.MustRunCommand() defer m0.Close() - if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { - t.Fatalf("unexpected cluster state: %s", m0.Server.Cluster.State()) + if m0.API.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected cluster state: %s", m0.API.State()) } } // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { - // Configure node0 - m0 := test.NewMainWithCluster(true) - defer m0.Close() + clus := test.MustRunCluster(t, 2) + defer clus[0].Close() + defer clus[1].Close() - gossipHost := "localhost" - gossipPort := 0 - seed, err := m0.RunWithTransport(gossipHost, gossipPort, []string{}) - if err != nil { - t.Fatal(err) - } - - // Configure node1 - m1 := test.NewMainWithCluster(false) - defer m1.Close() - - seed, err = m1.RunWithTransport(gossipHost, gossipPort, []string{seed}) - if err != nil { - t.Fatal(err) - } - - if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + if clus[0].API.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) + } else if clus[1].API.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State()) } } // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { - // Configure node0 - m0 := test.NewMainWithCluster(true) - defer m0.Close() + clus := test.MustRunCluster(t, 2) - seed, err := m0.RunWithTransport("localhost", 0, []string{}) - if err != nil { - t.Fatal(err) - } - - // Configure node1 - m1 := test.NewMainWithCluster(false) - defer m1.Close() - - var eg errgroup.Group - eg.Go(func() error { - _, err = m1.RunWithTransport("localhost", 0, []string{seed}) - if err != nil { - return err - } - return nil - }) - if err := eg.Wait(); err != nil { - t.Fatal(err) - } - - if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) + } else if !checkClusterState(clus[1], pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State()) } }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster(true) + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() - seed, err := m0.RunWithTransport("localhost", 0, []string{}) - if err != nil { - t.Fatal(err) - } + seed := m0.GossipAddress() // Create a client for each node. client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } defer m1.Close() - var eg errgroup.Group - eg.Go(func() error { - _, err = m1.RunWithTransport("localhost", 0, []string{seed}) - if err != nil { - return err - } - return nil - }) - if err := eg.Wait(); err != nil { - t.Fatal(err) - } - - if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("ContinuousSlices", func(t *testing.T) { - + t.Run("ContinuousShards", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster(true) + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() - seed, err := m0.RunWithTransport("localhost", 0, []string{}) - if err != nil { - t.Fatal(err) - } + seed := m0.GossipAddress() // Create a client for each node. client0 := m0.Client() - //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { t.Fatal(err) } // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=1300000) + Set(1, f=1) + Set(1300000, f=1) `); err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } defer m1.Close() - var eg errgroup.Group - eg.Go(func() error { - _, err = m1.RunWithTransport("localhost", 0, []string{seed}) - if err != nil { - return err - } - return nil - }) - if err := eg.Wait(); err != nil { - t.Fatal(err) - } - - if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("SkippedSlice", func(t *testing.T) { - + t.Run("SkippedShard", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster(true) + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() - seed, err := m0.RunWithTransport("localhost", 0, []string{}) - if err != nil { - t.Fatal(err) - } + seed := m0.GossipAddress() // Create a client for each node. client0 := m0.Client() - //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { t.Fatal(err) } - // Write data on first node. Note that no data is placed on slice 1. + // Write data on first node. Note that no data is placed on shard 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + Set(1, f=1) + Set(2400000, f=1) `); err != nil { t.Fatal(err) } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } defer m1.Close() - var eg errgroup.Group - eg.Go(func() error { - _, err = m1.RunWithTransport("localhost", 0, []string{seed}) - if err != nil { - return err - } - return nil - }) - if err := eg.Wait(); err != nil { - t.Fatal(err) - } - - if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) } @@ -345,37 +269,37 @@ func TestClusterResize_AddNode(t *testing.T) { func TestCluster_GossipMembership(t *testing.T) { t.Run("Node0Down", func(t *testing.T) { // Configure node0 - m0 := test.NewMainWithCluster(true) + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() - seed, err := m0.RunWithTransport("localhost", 0, []string{}) - if err != nil { - t.Fatal(err) - } - - // Configure node1 - m1 := test.NewMainWithCluster(false) - defer m1.Close() + seed := m0.GossipAddress() var eg errgroup.Group + + // Configure node1 + m1 := test.NewCommandNode(false) + defer m1.Close() eg.Go(func() error { + m1.Config.Gossip.Port = "0" // Pass invalid seed as first in list - _, err := m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) + m1.Config.Gossip.Seeds = []string{"http://localhost:8765", seed} + err := m1.Start() if err != nil { - return err + t.Fatalf("starting second main: %v", err) } return nil }) - // Configure node2 - m2 := test.NewMainWithCluster(false) + // Configure node1 + m2 := test.NewCommandNode(false) defer m2.Close() - eg.Go(func() error { - // Pass invalid seed as last in list - _, err := m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) + m2.Config.Gossip.Port = "0" + // Pass invalid seed as first in list + m2.Config.Gossip.Seeds = []string{seed, "http://localhost:8765"} + err := m2.Start() if err != nil { - return err + t.Fatalf("starting second main: %v", err) } return nil }) @@ -384,15 +308,15 @@ func TestCluster_GossipMembership(t *testing.T) { t.Fatal(err) } - if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) - } else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) - } else if !checkClusterState(m2.Server.Cluster, pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State()) + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) + } else if !checkClusterState(m2, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node2 cluster state: %s", m2.API.State()) } - numNodes := len(m0.Server.Cluster.Status().Nodes) + numNodes := len(m0.API.Hosts(context.Background())) if numNodes != 3 { t.Fatalf("Expected 3 nodes, got %d", numNodes) } @@ -400,7 +324,7 @@ func TestCluster_GossipMembership(t *testing.T) { } func TestClusterResize_RemoveNode(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3) + cluster := test.MustRunCluster(t, 3) m0 := cluster[0] m1 := cluster[1] @@ -455,10 +379,10 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) { client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { t.Fatal(err) } @@ -466,7 +390,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.ShardWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { @@ -486,9 +410,9 @@ func TestClusterResize_RemoveNode(t *testing.T) { // checkClusterState polls a given cluster for its state until it // receives a matching state. It polls up to n times before returning. -func checkClusterState(c *pilosa.Cluster, state string, n int) bool { +func checkClusterState(m *test.Command, state string, n int) bool { for i := 0; i < n; i++ { - if c.State() == state { + if m.API.State() == state { return true } time.Sleep(10 * time.Millisecond) diff --git a/server/config.go b/server/config.go index 663e9e18c..73f0b289e 100644 --- a/server/config.go +++ b/server/config.go @@ -21,13 +21,6 @@ import ( "github.com/pilosa/pilosa/toml" ) -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) @@ -47,7 +40,7 @@ type Config struct { Bind string `toml:"bind"` // MaxWritesPerRequest limits the number of mutating commands that can be in - // a single request to the server. This includes SetBit, ClearBit, + // a single request to the server. This includes Set, Clear, // SetRowAttrs & SetColumnAttrs. MaxWritesPerRequest int `toml:"max-writes-per-request"` @@ -61,7 +54,7 @@ type Config struct { Handler struct { // CORS Allowed Origins AllowedOrigins []string `toml:"allowed-origins"` - } + } `toml:"handler"` // TLS TLS TLSConfig `toml:"tls"` @@ -78,6 +71,11 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` + // Translation config supports translation store replication. + Translation struct { + PrimaryURL string `toml:"primary-url"` + } + AntiEntropy struct { Interval toml.Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/server/default.go b/server/default.go index a7131c82a..ce2fe8aaa 100644 --- a/server/default.go +++ b/server/default.go @@ -20,5 +20,5 @@ package server import "time" -// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. -const DefaultDiagnosticsInterval = time.Duration(0) +// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. A value of 0 disables diagnostics. +const defaultDiagnosticsInterval = time.Duration(0) diff --git a/server/handler_test.go b/server/handler_test.go new file mode 100644 index 000000000..9670df2c2 --- /dev/null +++ b/server/handler_test.go @@ -0,0 +1,674 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http/httptest" + "reflect" + "strings" + "testing" + + gohttp "net/http" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/test" +) + +// Ensure the handler returns "not found" for invalid paths. +func TestHandler_Endpoints(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + h := cmd.Handler.(*http.Handler).Handler + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + + t.Run("Not Found", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) + if w.Code != gohttp.StatusNotFound { + t.Fatalf("invalid status: %d", w.Code) + } + }) + + t.Run("Info", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != fmt.Sprintf("{\"shardWidth\":%d}\n", pilosa.ShardWidth) { + t.Fatalf("unexpected body: %s", body) + } + }) + + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { + t.Fatal(err) + } + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + t.Run("Schema", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + + t.Run("Status", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + ret := mustJSONDecode(t, w.Body) + if ret["state"].(string) != "NORMAL" { + t.Fatalf("wrong state from /status: %#v", ret) + } + if len(ret["nodes"].([]interface{})) != 1 { + t.Fatalf("wrong length nodes list: %#v", ret) + } + }) + + t.Run("Abort no resize job", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) + if w.Code != gohttp.StatusInternalServerError { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } + // TODO need to test aborting a cluster resize job. this may not be the right place + }) + + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.ShardWidth)+4) + + hldr.SetBit("i0", "f0", 31, 1) + + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+8) + + t.Run("Max Shard", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/internal/shards/max", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + + t.Run("Shards args", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1", strings.NewReader("Count(Row(f0=30))"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + + t.Run("Shards args protobuf", func(t *testing.T) { + // Generate request body. + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ + Query: "Count(Row(f0=30))", + Shards: []uint64{0, 1}, + }) + if err != nil { + t.Fatal(err) + } + + // Generate protobuf request. + req := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/json") + + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + + }) + + t.Run("Query args error", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=a,b", strings.NewReader("Count(Row(f0=30))"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"invalid shard argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + + t.Run("Query params err", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + + t.Run("Uint64 protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Row(f0=30))")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } + }) + + t.Run("Row JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + + f0 := i0.Field("f0") + if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+1, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { + t.Fatal(err) + } else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil { + t.Fatal(err) + } + + t.Run("ColumnAttrs_JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)"))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + + t.Run("Row pbuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) + } + }) + + t.Run("Row columnattrs protobuf", func(t *testing.T) { + // Encode request body. + buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{ + Query: "Row(f0=30)", + ColumnAttrs: true, + }) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(buf)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { + t.Fatalf("unexpected columns: %+v", columns) + } else if _, ok := resp.Results[0].(*pilosa.Row); !ok { + t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 { + t.Fatalf("unexpected attr length: %d", len(attrs)) + } else if attrs["a"] != "b" { + t.Fatalf("unexpected attr[a]: %v", attrs["a"]) + } else if attrs["c"] != int64(1) { + t.Fatalf("unexpected attr[c]: %v", attrs["c"]) + } else if !attrs["d"].(bool) { + t.Fatalf("unexpected attr[d]: %v", attrs["d"]) + } + + if a := resp.ColumnAttrSets; len(a) != 2 { + t.Fatalf("unexpected column attributes length: %d", len(a)) + } else if a[0].ID != pilosa.ShardWidth+1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { + t.Fatalf("unexpected column attr length: %d", len(a)) + } else if a[0].Attrs["x"] != "y" { + t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"]) + } + }) + + t.Run("Query Pairs JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + + t.Run("Query Pairs protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { + t.Fatalf("unexpected pair length: %d", len(a)) + } + }) + + t.Run("Query err JSON", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"executing: field not found"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + + t.Run("Query err protobuf", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`)) + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + var resp pilosa.QueryResponse + if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } else if s := resp.Err.Error(); s != `executing: field not found` { + t.Fatalf("unexpected error: %s", s) + } + }) + + t.Run("Method not allowed", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil)) + if w.Code != gohttp.StatusMethodNotAllowed { + t.Fatalf("invalid status: %d", w.Code) + } + }) + + t.Run("Err Parse", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?shards=0,1", strings.NewReader("bad_fn("))) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + }) + + t.Run("delete index", func(t *testing.T) { + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected response body: %s", w.Body.String()) + } + // Verify index is gone. + if hldr.Index("i") != nil { + t.Fatal("expected nil index") + } + }) + + t.Run("Field delete", func(t *testing.T) { + i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := i.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } else if body := w.Body.String(); body != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } + }) + + i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if err := i.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + t.Fatal(err) + } else if err := i.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := i.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + t.Fatal(err) + } + + t.Run("AttrStore Diff", func(t *testing.T) { + blks, err := i.ColumnAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } + + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") + + // Send block checksums to determine diff. + req := test.MustNewHTTPRequest( + "POST", + "/internal/index/i/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } + + // Read and validate body. + if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", w.Body.String()) + } + }) + + meta, err := i.CreateFieldIfNotExists("meta", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + if err := meta.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + t.Fatal(err) + } else if err := meta.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + t.Fatal(err) + } else if err := meta.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + t.Fatal(err) + } + + t.Run("field attrstore diff", func(t *testing.T) { + blks, err := meta.RowAttrStore().Blocks() + if err != nil { + t.Fatal(err) + } + blks = blks[1:] + blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") + + // Send block checksums to determine diff. + req := test.MustNewHTTPRequest( + "POST", + "/internal/index/i/field/meta/attr/diff", + strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), + ) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) + } + + // Read and validate body. + if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" { + t.Fatalf("unexpected body: %s", w.Body.String()) + } + }) + + t.Run("Version", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/version", nil) + h.ServeHTTP(w, r) + version := strings.TrimPrefix(pilosa.Version, "v") + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + }) + + t.Run("Fragment Nodes", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?index=i&shard=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + body := mustJSONDecodeSlice(t, w.Body) + bmap := body[0].(map[string]interface{}) + if bmap["isCoordinator"] != true { + t.Fatalf("expected true coordinator") + } + + // invalid argument should return BadRequest + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?db=X&shard=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?shard=0", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) + + t.Run("Expvars", func(t *testing.T) { + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) + + t.Run("Recalculate Caches", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) + if w.Code != gohttp.StatusNoContent { + t.Fatalf("unexpected status code: %d", w.Code) + } + }) + + t.Run("CORS", func(t *testing.T) { + req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) + req.Header.Add("Origin", "http://test/") + req.Header.Add("Access-Control-Request-Method", "POST") + + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + result := w.Result() + + // This handler does not support CORS, return Method Not Allowed (405) + if result.StatusCode != 405 { + t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) + } + + clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) + w = httptest.NewRecorder() + h := clus[0].Handler.(*http.Handler).Handler + h.ServeHTTP(w, req) + result = w.Result() + + if result.StatusCode != 200 { + t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) + } + if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { + t.Fatal("CORS header not present") + } + }) + + t.Run("index handlers", func(t *testing.T) { + // create index + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/idx1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // create index again + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/idx1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusConflict { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"index already exists"}}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // create field + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/idx1/field/fld1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // create field again + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/idx1/field/fld1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusConflict { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"field already exists"}}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // delete field + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("DELETE", "/index/idx1/field/fld1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // delete field again + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("DELETE", "/index/idx1/field/fld1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"field not found"}}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // delete index + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("DELETE", "/index/idx1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // delete index again + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("DELETE", "/index/idx1", strings.NewReader("")) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":false,"error":{"message":"index not found"}}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + }) +} + +func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { + dec := json.NewDecoder(r) + err := dec.Decode(&ret) + if err != nil { + t.Fatalf("decoding response: %v", err) + } + return ret +} + +func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) { + dec := json.NewDecoder(r) + err := dec.Decode(&ret) + if err != nil { + t.Fatalf("decoding response: %v", err) + } + return ret +} diff --git a/server/release.go b/server/release.go index 54d00b244..d988f4f81 100644 --- a/server/release.go +++ b/server/release.go @@ -20,5 +20,5 @@ package server import "time" -// DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. -const DefaultDiagnosticsInterval = 1 * time.Hour +// defaultDiagnosticsInterval is the default sync frequency diagnostic metrics. +const defaultDiagnosticsInterval = 1 * time.Hour diff --git a/server/server.go b/server/server.go index 345807ec2..d71139096 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,6 @@ import ( "log" "math/rand" "net" - "net/http" "os" "os/signal" "strconv" @@ -36,10 +35,11 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" - "github.com/pilosa/pilosa/statik" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/statsd" "github.com/pkg/errors" ) @@ -61,31 +61,56 @@ type Command struct { Config *Config // Gossip transport - GossipTransport *gossip.Transport + gossipTransport *gossip.Transport // Standard input/output *pilosa.CmdIO - // Started will be closed once Command.Run is finished. + // Started will be closed once Command.Start is finished. Started chan struct{} - // Done will be closed when Command.Close() is called - Done chan struct{} + // done will be closed when Command.Close() is called + done chan struct{} // Passed to the Gossip implementation. logOutput io.Writer logger loggerLogger + + Handler pilosa.Handler + API *pilosa.API + ln net.Listener + + serverOptions []pilosa.ServerOption +} + +type CommandOption func(c *Command) error + +func OptCommandServerOptions(opts ...pilosa.ServerOption) CommandOption { + return func(c *Command) error { + c.serverOptions = append(c.serverOptions, opts...) + return nil + } } // NewCommand returns a new instance of Main. -func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { - return &Command{ +func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command { + c := &Command{ Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), Started: make(chan struct{}), - Done: make(chan struct{}), + done: make(chan struct{}), } + + for _, opt := range opts { + err := opt(c) + if err != nil { + panic(err) + // TODO: Return error instead of panic? + } + } + + return c } // Start starts the pilosa server - it returns once the server is running. @@ -99,17 +124,23 @@ func (m *Command) Start() (err error) { } // SetupNetworking - err = m.SetupNetworking() + err = m.setupNetworking() if err != nil { return errors.Wrap(err, "setting up networking") } + go func() { + err := m.Handler.Serve() + if err != nil { + m.logger.Printf("Handler serve error: %v", err) + } + }() // Initialize server. if err = m.Server.Open(); err != nil { return errors.Wrap(err, "opening server") } - m.logger.Printf("Listening as %s\n", m.Server.URI) + m.logger.Printf("Listening as %s\n", m.API.Node().URI) return nil } @@ -126,7 +157,7 @@ func (m *Command) Wait() error { // Second signal causes a hard shutdown. go func() { <-c; os.Exit(1) }() return errors.Wrap(m.Close(), "closing command") - case <-m.Done: + case <-m.done: m.logger.Printf("Server closed externally") return nil } @@ -165,15 +196,6 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) - handler, err := pilosa.NewHandler(pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins)) - if err != nil { - return errors.Wrap(err, "wrapping handler") - } - handler.Logger = m.logger - handler.FileSystem = &statik.FileSystem{} - handler.API = pilosa.NewAPI() - handler.API.Logger = m.logger - uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { return errors.Wrap(err, "processing bind address") @@ -181,7 +203,7 @@ func (m *Command) SetupServer() error { // Setup TLS var TLSConfig *tls.Config - if uri.Scheme() == "https" { + if uri.Scheme == "https" { if m.Config.TLS.CertificatePath == "" { return errors.New("certificate path is required for TLS sockets") } @@ -200,23 +222,39 @@ func (m *Command) SetupServer() error { diagnosticsInterval := time.Duration(0) if m.Config.Metric.Diagnostics { - diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval) + diagnosticsInterval = time.Duration(defaultDiagnosticsInterval) } - statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) if err != nil { return errors.Wrap(err, "new stats client") } - ln, err := getListener(*uri, TLSConfig) + m.ln, err = getListener(*uri, TLSConfig) if err != nil { return errors.Wrap(err, "getting listener") } - c := GetHTTPClient(TLSConfig) - handler.API.RemoteClient = c + // If port is 0, get auto-allocated port from listener + if uri.Port == 0 { + uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port)) + } - m.Server, err = pilosa.NewServer( + c := http.GetHTTPClient(TLSConfig) + + // Setup connection to primary store if this is a replica. + var primaryTranslateStore pilosa.TranslateStore + if m.Config.Translation.PrimaryURL != "" { + primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) + } + + // Set Coordinator. + coordinatorOpt := pilosa.OptServerIsCoordinator(false) + if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { + coordinatorOpt = pilosa.OptServerIsCoordinator(true) + } + + serverOptions := []pilosa.ServerOption{ pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), pilosa.OptServerDataDir(m.Config.DataDir), @@ -227,60 +265,43 @@ func (m *Command) SetupServer() error { pilosa.OptServerLogger(m.logger), pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore), - pilosa.OptServerHandler(handler), pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), pilosa.OptServerStatsClient(statsClient), - pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), - pilosa.OptServerRemoteClient(c), + pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), + pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), + pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerSerializer(proto.Serializer{}), + coordinatorOpt, + } + + serverOptions = append(serverOptions, m.serverOptions...) + + m.Server, err = pilosa.NewServer(serverOptions...) + + if err != nil { + return errors.Wrap(err, "new server") + } + + m.API, err = pilosa.NewAPI(pilosa.OptAPIServer(m.Server)) + if err != nil { + return errors.Wrap(err, "new api") + } + + m.Handler, err = http.NewHandler( + http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + http.OptHandlerAPI(m.API), + http.OptHandlerLogger(m.logger), + http.OptHandlerListener(m.ln), ) + return errors.Wrap(err, "new handler") - return errors.Wrap(err, "new server") } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} - -// SetupNetworking sets up internode communication based on the configuration. -func (m *Command) SetupNetworking() error { - - m.Server.NodeID = m.Server.LoadNodeID() - +// setupNetworking sets up internode communication based on the configuration. +func (m *Command) setupNetworking() error { if m.Config.Cluster.Disabled { - m.Server.Cluster.Static = true - m.Server.Cluster.Coordinator = m.Server.NodeID - for _, address := range m.Config.Cluster.Hosts { - uri, err := pilosa.NewURIFromAddress(address) - if err != nil { - return errors.Wrap(err, "getting URI") - } - m.Server.Cluster.Nodes = append(m.Server.Cluster.Nodes, &pilosa.Node{ - URI: *uri, - }) - } - - m.Server.Broadcaster = pilosa.NopBroadcaster - m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes) - m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - m.Server.Gossiper = pilosa.NopGossiper return nil } @@ -290,63 +311,48 @@ func (m *Command) SetupNetworking() error { } // get the host portion of addr to use for binding - gossipHost := m.Server.URI.Host() - var transport *gossip.Transport - if m.GossipTransport != nil { - transport = m.GossipTransport - } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) - if err != nil { - return errors.Wrap(err, "getting transport") - } + gossipHost := m.API.Node().URI.Host + m.gossipTransport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) + if err != nil { + return errors.Wrap(err, "getting transport") } - // Set Coordinator. - if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { - m.Server.Cluster.Coordinator = m.Server.NodeID - m.Server.Cluster.Node.IsCoordinator = true - } - - gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) - m.Server.Cluster.EventReceiver = gossipEventReceiver gossipMemberSet, err := gossip.NewGossipMemberSet( - m.Server.NodeID, - m.Server.URI.Host(), m.Config.Gossip, - gossipEventReceiver, - m.Server, + m.API, gossip.WithLogger(m.logger.Logger()), - gossip.WithTransport(transport), + gossip.WithTransport(m.gossipTransport), ) if err != nil { return errors.Wrap(err, "getting memberset") } - gossipMemberSet.Logger = m.logger - m.Server.Cluster.MemberSet = gossipMemberSet - m.Server.Broadcaster = m.Server - m.Server.BroadcastReceiver = gossipMemberSet - m.Server.Gossiper = gossipMemberSet - return nil + return errors.Wrap(gossipMemberSet.Open(), "opening gossip memberset") +} + +// GossipTransport allows a caller to return the gossip transport created when +// setting up the GossipMemberSet. This is useful if one needs to determine the +// allocated ephemeral port programmatically. (usually used in tests) +func (m *Command) GossipTransport() *gossip.Transport { + return m.gossipTransport } // Close shuts down the server. func (m *Command) Close() error { var logErr error + handlerErr := m.Handler.Close() serveErr := m.Server.Close() if closer, ok := m.logOutput.(io.Closer); ok { logErr = closer.Close() } - close(m.Done) - if serveErr != nil && logErr != nil { - return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr) - } else if logErr != nil { - return logErr + close(m.done) + if serveErr != nil || logErr != nil || handlerErr != nil { + return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr) } - return serveErr + return nil } -// NewStatsClient creates a stats client from the config -func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { +// newStatsClient creates a stats client from the config +func newStatsClient(name string, host string) (pilosa.StatsClient, error) { switch name { case "expvar": return pilosa.NewExpvarStatsClient(), nil @@ -355,26 +361,26 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { case "nop", "none": return pilosa.NopStatsClient, nil default: - return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].", name) } } // getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS - if uri.Scheme() == "https" && tlsconf != nil { + if uri.Scheme == "https" && tlsconf != nil { ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) if err != nil { return nil, errors.Wrap(err, "tls.Listener") } - } else if uri.Scheme() == "http" { + } else if uri.Scheme == "http" { // Open HTTP listener to determine port (if specified as :0). ln, err = net.Listen("tcp", uri.HostPort()) if err != nil { return nil, errors.Wrap(err, "net.Listen") } } else { - return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme) } return ln, nil diff --git a/server/server_test.go b/server/server_test.go index 9092212b2..506807f70 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -21,14 +21,14 @@ import ( "io/ioutil" "math/rand" "reflect" - "runtime" "sort" "strings" "testing" "testing/quick" - "github.com/BurntSushi/toml" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -40,31 +40,31 @@ func TestMain_Set_Quick(t *testing.T) { } if err := quick.Check(func(cmds []SetCommand) bool { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } - // Execute SetBit() commands. + // Execute Set() commands. for _, cmd := range cmds { - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") { + if err := client.CreateField(context.Background(), "i", cmd.Field); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil { t.Fatal(err) } } // Validate data. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -86,8 +86,8 @@ func TestMain_Set_Quick(t *testing.T) { } // Validate data after reopening. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -116,52 +116,52 @@ func TestMain_Set_Quick(t *testing.T) { // Ensure program can set row attributes and retrieve them. func TestMain_SetRowAttrs(t *testing.T) { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x"); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "z"); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "neg"); err != nil { t.Fatal(err) } - // Set columns on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + // Set columns on different rows in different fields. + if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `Set(100, neg=3)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(x, 1, x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(x, 2, x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(z, 2, x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(neg, 3, x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=2)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(neg=3)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Row(x=2)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -193,31 +193,31 @@ func TestMain_SetRowAttrs(t *testing.T) { // Ensure program can set column attributes and retrieve them. func TestMain_SetColumnAttrs(t *testing.T) { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x"); err != nil { t.Fatal(err) } // Set columns on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `Set(101, x=1)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(100, foo="bar")`); err != nil { t.Fatal(err) } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -262,39 +262,25 @@ func tempMkdir(t *testing.T) string { return dir } -// Ensure the file handle count is working -func TestCountOpenFiles(t *testing.T) { - // Windows is not supported yet - if runtime.GOOS == "windows" { - t.Skip("Skipping unsupported CountOpenFiles test on Windows.") - } - count, err := pilosa.CountOpenFiles() - if err != nil { - t.Errorf("CountOpenFiles failed: %s", err) - } - if count == 0 { - t.Error("CountOpenFiles returned invalid value 0.") - } -} - func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 - cluster := test.MustRunMainWithCluster(t, clusterSize) + cluster := test.MustRunCluster(t, clusterSize) // Create the schema. client0 := cluster[0].Client() if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } - if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil { - t.Fatal("create frame:", err) + if err := client0.CreateField(context.Background(), "i", "f"); err != nil { + t.Fatal("create field:", err) } + return // Set some columns data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { @@ -311,7 +297,7 @@ func TestMain_RecalculateHashes(t *testing.T) { // Run a TopN query on all nodes. The result should be the same as the target. for _, m := range cluster { - res, err := m.Query("i", "", `TopN(frame="f")`) + res, err := m.Query("i", "", `TopN(f)`) if err != nil { t.Fatal(err) } @@ -325,37 +311,37 @@ func TestMain_RecalculateHashes(t *testing.T) { // SetCommand represents a command to set a column. type SetCommand struct { ID uint64 - Frame string + Field string ColumnID uint64 } type SetCommands []SetCommand -// Frames returns the set of column ids for each frame/row. -func (a SetCommands) Frames() map[string]map[uint64][]uint64 { +// Fields returns the set of column ids for each field/row. +func (a SetCommands) Fields() map[string]map[uint64][]uint64 { // Create a set of unique commands. m := make(map[SetCommand]struct{}) for _, cmd := range a { m[cmd] = struct{}{} } - // Build unique ids for each frame & row. - frames := make(map[string]map[uint64][]uint64) + // Build unique ids for each field & row. + fields := make(map[string]map[uint64][]uint64) for cmd := range m { - if frames[cmd.Frame] == nil { - frames[cmd.Frame] = make(map[uint64][]uint64) + if fields[cmd.Field] == nil { + fields[cmd.Field] = make(map[uint64][]uint64) } - frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID) + fields[cmd.Field][cmd.ID] = append(fields[cmd.Field][cmd.ID], cmd.ColumnID) } // Sort each set of column ids. - for _, frame := range frames { - for id := range frame { - sort.Sort(uint64Slice(frame[id])) + for _, field := range fields { + for id := range field { + sort.Sort(uint64Slice(field[id])) } } - return frames + return fields } // GenerateSetCommands generates random SetCommand objects. @@ -364,7 +350,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { for i := range cmds { cmds[i] = SetCommand{ ID: uint64(rand.Intn(1000)), - Frame: "x", + Field: "x", ColumnID: uint64(rand.Intn(10)), } } @@ -374,7 +360,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { // ParseConfig parses s into a Config. func ParseConfig(s string) (server.Config, error) { var c server.Config - _, err := toml.Decode(s, &c) + err := toml.Unmarshal([]byte(s), &c) return c, err } diff --git a/test/client.go b/server_internal_test.go similarity index 60% rename from test/client.go rename to server_internal_test.go index 9e391e88b..e64f80a0f 100644 --- a/test/client.go +++ b/server_internal_test.go @@ -12,24 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package test +package pilosa import ( - "net/http" - - "github.com/pilosa/pilosa" + "runtime" + "testing" ) -// Client represents a test wrapper for pilosa.Client. -type Client struct { - *pilosa.InternalHTTPClient -} - -// MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *http.Client) *Client { - c, err := pilosa.NewInternalHTTPClient(host, h) - if err != nil { - panic(err) +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + if runtime.GOOS == "windows" { + t.Skip("Skipping unsupported countOpenFiles test on Windows.") + } + count, err := countOpenFiles() + if err != nil { + t.Errorf("countOpenFiles failed: %s", err) + } + if count == 0 { + t.Error("countOpenFiles returned invalid value 0.") } - return &Client{InternalHTTPClient: c} } diff --git a/server_test.go b/server_test.go deleted file mode 100644 index 9e10374d7..000000000 --- a/server_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package pilosa_test - -import ( - "context" - "testing" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -// TestMonitorAntiEntropy is a regression test which which caught a bug where -// pilosa.Server was not having its remoteClient field set by an option and so -// it was using a nil client in monitorAntiEntropy. -func TestMonitorAntiEntropy(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1)) - client := cluster[1].Client() - err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) - if err != nil { - t.Fatalf("creating frame: %v", err) - } - - time.Sleep(time.Millisecond * 2) - for _, m := range cluster { - err := m.Close() - if err != nil { - t.Fatal(err) - } - } - -} diff --git a/statik/.gitignore b/statik/.gitignore deleted file mode 100644 index 485c0c57d..000000000 --- a/statik/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/statik.go diff --git a/stats.go b/stats.go index 313708fbc..130a91a64 100644 --- a/stats.go +++ b/stats.go @@ -82,8 +82,8 @@ func (c *nopStatsClient) SetLogger(logger Logger) func (c *nopStatsClient) Open() {} func (c *nopStatsClient) Close() error { return nil } -// ExpvarStatsClient writes stats out to expvars. -type ExpvarStatsClient struct { +// expvarStatsClient writes stats out to expvars. +type expvarStatsClient struct { mu sync.Mutex m *expvar.Map tags []string @@ -91,41 +91,41 @@ type ExpvarStatsClient struct { // NewExpvarStatsClient returns a new instance of ExpvarStatsClient. // This client points at the root of the expvar index map. -func NewExpvarStatsClient() *ExpvarStatsClient { - return &ExpvarStatsClient{ +func NewExpvarStatsClient() *expvarStatsClient { + return &expvarStatsClient{ m: Expvar, } } // Tags returns a sorted list of tags on the client. -func (c *ExpvarStatsClient) Tags() []string { +func (c *expvarStatsClient) Tags() []string { return nil } // WithTags returns a new client with additional tags appended. -func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient { +func (c *expvarStatsClient) WithTags(tags ...string) StatsClient { m := &expvar.Map{} m.Init() c.m.Set(strings.Join(tags, ","), m) - return &ExpvarStatsClient{ + return &expvarStatsClient{ m: m, - tags: UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), } } // Count tracks the number of times something occurs. -func (c *ExpvarStatsClient) Count(name string, value int64, rate float64) { +func (c *expvarStatsClient) Count(name string, value int64, rate float64) { c.m.Add(name, value) } // CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (c *ExpvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { +func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { c.m.Add(name, value) } // Gauge sets the value of a metric. -func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) { +func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) { var f expvar.Float f.Set(value) c.m.Set(name, &f) @@ -133,19 +133,19 @@ func (c *ExpvarStatsClient) Gauge(name string, value float64, rate float64) { // Histogram tracks statistical distribution of a metric. // This works the same as gauge for this client. -func (c *ExpvarStatsClient) Histogram(name string, value float64, rate float64) { +func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) { c.Gauge(name, value, rate) } // Set tracks number of unique elements. -func (c *ExpvarStatsClient) Set(name string, value string, rate float64) { +func (c *expvarStatsClient) Set(name string, value string, rate float64) { var s expvar.String s.Set(value) c.m.Set(name, &s) } // Timing tracks timing information for a metric. -func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float64) { +func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) { c.mu.Lock() d, _ := c.m.Get(name).(time.Duration) c.m.Set(name, d+value) @@ -153,14 +153,14 @@ func (c *ExpvarStatsClient) Timing(name string, value time.Duration, rate float6 } // SetLogger has no logger. -func (c *ExpvarStatsClient) SetLogger(logger Logger) { +func (c *expvarStatsClient) SetLogger(logger Logger) { } // Open no-op. -func (c *ExpvarStatsClient) Open() {} +func (c *expvarStatsClient) Open() {} // Close no-op. -func (c *ExpvarStatsClient) Close() error { return nil } +func (c *expvarStatsClient) Close() error { return nil } // MultiStatsClient joins multiple stats clients together. type MultiStatsClient []StatsClient @@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error { return nil } -// UnionStringSlice returns a sorted set of tags which combine a & b. -func UnionStringSlice(a, b []string) []string { +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { // Sort both sets first. sort.Strings(a) sort.Strings(b) diff --git a/stats_test.go b/stats_test.go index 611b7b2b6..067cfa991 100644 --- a/stats_test.go +++ b/stats_test.go @@ -16,12 +16,13 @@ package pilosa_test import ( "context" - "net/http" + "net/http/httptest" "strings" "testing" "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/test" ) @@ -36,45 +37,45 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) + hldr.ClearBit("d", "f", 0, 1) - if pilosa.Expvar.String() != `{"index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Gauge creates a unique key, subsequent Gauge calls will overwrite hldr.Stats.Gauge("g", 5, 1.0) hldr.Stats.Gauge("g", 8, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Set creates a unique key, subsequent sets will overwrite hldr.Stats.Set("s", "4", 1.0) hldr.Stats.Set("s", "7", 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Record timing duration and a uniquely Set key/value dur, _ := time.ParseDuration("123us") hldr.Stats.Timing("tt", dur, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Expvar histogram is implemented as a gauge hldr.Stats.Histogram("hh", 3, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } @@ -85,18 +86,18 @@ func TestMultiStatClient_Expvar(t *testing.T) { } func TestStatsCount_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) // Execute query. called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - e.Holder.Stats = &MockStats{ + hldr.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { t.Errorf("Expected TopN, Results %s", name) @@ -109,7 +110,7 @@ func TestStatsCount_TopN(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { t.Fatal(err) } if !called { @@ -118,17 +119,17 @@ func TestStatsCount_TopN(t *testing.T) { } func TestStatsCount_Bitmap(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - e.Holder.Stats = &MockStats{ + hldr.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != "Bitmap" { - t.Errorf("Expected Bitmap, Results %s", name) + if name != "Row" { + t.Errorf("Expected Row, Results %s", name) } if tags[0] != "index:d" { @@ -138,7 +139,7 @@ func TestStatsCount_Bitmap(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil { t.Fatal(err) } if !called { @@ -147,20 +148,20 @@ func TestStatsCount_Bitmap(t *testing.T) { } func TestStatsCount_SetColumnAttrs(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - frame := e.Holder.Frame("d", "f") - if frame == nil { - t.Fatal("frame not found") + field := hldr.Field("d", "f") + if field == nil { + t.Fatal("field not found") } - frame.Stats = &MockStats{ + field.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "SetRowAttrs" { t.Errorf("Expected SetRowAttrs, Results %s", name) @@ -168,7 +169,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } if !called { @@ -177,15 +178,15 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { } func TestStatsCount_SetProfileAttrs(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - idx := e.Holder.Index("d") + idx := hldr.Holder.Index("d") if idx == nil { t.Fatal("idex not found") } @@ -199,7 +200,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil { t.Fatal(err) } if !called { @@ -207,116 +208,89 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } } -func TestStatsCount_CreateIndex(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != "createIndex" { - t.Errorf("Expected createIndex, Results %s", name) - } +func TestStatsCount_APICalls(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + h := cmd.Handler.(*http.Handler).Handler + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil)) - if !called { - t.Error("Count isn't called") - } -} + t.Run("create index", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCount: func(name string, value int64, rate float64) { + if name != "createIndex" { + t.Errorf("Expected createIndex, Results %s", name) + } + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) -func TestStatsCount_DeleteIndex(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("create field", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, rate float64, index []string) { + if name != "createField" { + t.Errorf("Expected createField, Results %s", name) + } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != "deleteIndex" { - t.Errorf("Expected deleteIndex, Results %s", name) - } + t.Run("delete field", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCountWithTags: func(name string, value int64, rate float64, index []string) { + if name != "deleteField" { + t.Errorf("Expected deleteField, Results %s", name) + } + if index[0] != "index:i" { + t.Errorf("Expected index:i, Results %s", index) + } - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } -} + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) -func TestStatsCount_CreateFrame(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("delete index", func(t *testing.T) { + called := false + hldr.Stats = &MockStats{ + mockCount: func(name string, value int64, rate float64) { + if name != "deleteIndex" { + t.Errorf("Expected deleteIndex, Results %s", name) + } - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() + called = true + }, + } + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) + if !called { + t.Error("Count isn't called") + } + }) - // Create index. - if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } - called := false - s.Handler.API.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "createFrame" { - t.Errorf("Expected createFrame, Results %s", name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) - if !called { - t.Error("Count isn't called") - } -} - -func TestStatsCount_DeleteFrame(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - called := false - // Create index. - indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - s.Handler.API.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "deleteFrame" { - t.Errorf("Expected deleteFrame, Results %s", name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } } type MockStats struct { diff --git a/statsd/statsd.go b/statsd/statsd.go index 0cec63718..7a9ae6c14 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,6 +15,7 @@ package statsd import ( + "sort" "time" "github.com/DataDog/datadog-go/statsd" @@ -25,102 +26,137 @@ import ( // statsD defailt host is "127.0.0.1:8125" const ( - // Prefix is appended to each metric event name - Prefix = "pilosa." + // prefix is appended to each metric event name + prefix = "pilosa." - // BufferLen Stats lient buffer size. - BufferLen = 1024 + // bufferLen Stats lient buffer size. + bufferLen = 1024 ) // Ensure client implements interface. -var _ pilosa.StatsClient = &StatsClient{} +var _ pilosa.StatsClient = &statsClient{} -// StatsClient represents a StatsD implementation of pilosa.StatsClient. -type StatsClient struct { +// statsClient represents a StatsD implementation of pilosa.statsClient. +type statsClient struct { client *statsd.Client tags []string logger pilosa.Logger } // NewStatsClient returns a new instance of StatsClient. -func NewStatsClient(host string) (*StatsClient, error) { - c, err := statsd.NewBuffered(host, BufferLen) +func NewStatsClient(host string) (*statsClient, error) { + c, err := statsd.NewBuffered(host, bufferLen) if err != nil { return nil, err } - return &StatsClient{ + return &statsClient{ client: c, logger: pilosa.NopLogger, }, nil } // Open no-op -func (c *StatsClient) Open() {} +func (c *statsClient) Open() {} // Close closes the connection to the agent. -func (c *StatsClient) Close() error { +func (c *statsClient) Close() error { return c.client.Close() } // Tags returns a sorted list of tags on the client. -func (c *StatsClient) Tags() []string { +func (c *statsClient) Tags() []string { return c.tags } // WithTags returns a new client with additional tags appended. -func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { - return &StatsClient{ +func (c *statsClient) WithTags(tags ...string) pilosa.StatsClient { + return &statsClient{ client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), logger: c.logger, } } // Count tracks the number of times something occurs per second. -func (c *StatsClient) Count(name string, value int64, rate float64) { - if err := c.client.Count(Prefix+name, value, c.tags, rate); err != nil { +func (c *statsClient) Count(name string, value int64, rate float64) { + if err := c.client.Count(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *StatsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { +func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { tags := append(c.tags, t...) - if err := c.client.Count(Prefix+name, value, tags, rate); err != nil { + if err := c.client.Count(prefix+name, value, tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Count error: %s", err) } } // Gauge sets the value of a metric. -func (c *StatsClient) Gauge(name string, value float64, rate float64) { - if err := c.client.Gauge(Prefix+name, value, c.tags, rate); err != nil { +func (c *statsClient) Gauge(name string, value float64, rate float64) { + if err := c.client.Gauge(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Gauge error: %s", err) } } // Histogram tracks statistical distribution of a metric. -func (c *StatsClient) Histogram(name string, value float64, rate float64) { - if err := c.client.Histogram(Prefix+name, value, c.tags, rate); err != nil { +func (c *statsClient) Histogram(name string, value float64, rate float64) { + if err := c.client.Histogram(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Histogram error: %s", err) } } // Set tracks number of unique elements. -func (c *StatsClient) Set(name string, value string, rate float64) { - if err := c.client.Set(Prefix+name, value, c.tags, rate); err != nil { +func (c *statsClient) Set(name string, value string, rate float64) { + if err := c.client.Set(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Set error: %s", err) } } // Timing tracks timing information for a metric. -func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { - if err := c.client.Timing(Prefix+name, value, c.tags, rate); err != nil { +func (c *statsClient) Timing(name string, value time.Duration, rate float64) { + if err := c.client.Timing(prefix+name, value, c.tags, rate); err != nil { c.logger.Printf("statsd.StatsClient.Timing error: %s", err) } } // SetLogger sets the logger for client. -func (c *StatsClient) SetLogger(logger pilosa.Logger) { +func (c *statsClient) SetLogger(logger pilosa.Logger) { c.logger = logger } + +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { + // Sort both sets first. + sort.Strings(a) + sort.Strings(b) + + // Find size of largest slice. + n := len(a) + if len(b) > n { + n = len(b) + } + + // Exit if both sets are empty. + if n == 0 { + return nil + } + + // Iterate over both in order and merge. + other := make([]string, 0, n) + for len(a) > 0 || len(b) > 0 { + if len(a) == 0 { + other, b = append(other, b[0]), b[1:] + } else if len(b) == 0 { + other, a = append(other, a[0]), a[1:] + } else if a[0] < b[0] { + other, a = append(other, a[0]), a[1:] + } else if b[0] < a[0] { + other, b = append(other, b[0]), b[1:] + } else { + other, a, b = append(other, a[0]), a[1:], b[1:] + } + } + return other +} diff --git a/test/attr.go b/test/attr.go deleted file mode 100644 index 16e8ff334..000000000 --- a/test/attr.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/boltdb" -) - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{boltdb.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - b.Fatal(err) - } - } - }() - } - wg.Wait() -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -} diff --git a/test/cluster.go b/test/cluster.go index 77cf496a8..ca08b700b 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -14,437 +14,7 @@ package test -import ( - "bufio" - "bytes" - "fmt" - "io/ioutil" - "path/filepath" - "sync" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" -) - -// NewCluster returns a cluster with n nodes and uses a mod-based hasher. -func NewCluster(n int) *pilosa.Cluster { - path, err := ioutil.TempDir("", "pilosa-cluster-") - if err != nil { - panic(err) - } - - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - c.Path = path - c.Topology = pilosa.NewTopology() - - for i := 0; i < n; i++ { - c.Nodes = append(c.Nodes, &pilosa.Node{ - ID: fmt.Sprintf("node%d", i), - URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), - }) - } - - c.Node = c.Nodes[0] - c.Coordinator = c.Nodes[0].ID - c.SetState(pilosa.ClusterStateNormal) - - return c -} - -// ModHasher represents a simple, mod-based hashing. +// modHasher represents a simple, mod-based hashing. type ModHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } - func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } - -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } - -// NewURI is a test URI creator that intentionally swallows errors. -func NewURI(scheme, host string, port uint16) pilosa.URI { - uri := pilosa.DefaultURI() - uri.SetScheme(scheme) - uri.SetHost(host) - uri.SetPort(port) - return *uri -} - -func NewURIFromHostPort(host string, port uint16) pilosa.URI { - uri := pilosa.DefaultURI() - uri.SetHost(host) - uri.SetPort(port) - return *uri -} - -// TestCluster represents a cluster of test nodes, each of which -// has a pilosa.Cluster. -type TestCluster struct { - Clusters []*pilosa.Cluster - - common *commonClusterSettings - - mu sync.RWMutex - resizing bool - resizeDone chan struct{} -} - -type commonClusterSettings struct { - Nodes []*pilosa.Node -} - -func (t *TestCluster) CreateIndex(name string) error { - for _, c := range t.Clusters { - if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { - return err - } - } - return nil -} - -func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error { - for _, c := range t.Clusters { - idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) - if err != nil { - return err - } - if _, err := idx.CreateFrame(frame, opt); err != nil { - return err - } - } - return nil -} -func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { - // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetBit(view, rowID, colID, x) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { - // Determine which node should receive the SetFieldValue. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := columnID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetFieldValue(columnID, name, value) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { - for _, c := range t.Clusters { - if c.Node.ID == id { - return c - } - } - return nil -} - -// AddNode adds a node to the cluster and (potentially) starts a resize job. -func (t *TestCluster) AddNode(saveTopology bool) error { - id := len(t.Clusters) - - c, err := t.addCluster(id, saveTopology) - if err != nil { - return err - } - - // Send NodeJoin event to coordinator. - if id > 0 { - coord := t.Clusters[0] - ev := &pilosa.NodeEvent{ - Event: pilosa.NodeJoin, - Node: c.Node, - } - - if err := coord.ReceiveEvent(ev); err != nil { - return err - } - - // Wait for the AddNode job to finish. - if c.State() != pilosa.ClusterStateNormal { - t.resizeDone = make(chan struct{}) - t.mu.Lock() - t.resizing = true - t.mu.Unlock() - <-t.resizeDone - } - } - - return nil -} - -// WriteTopology writes the given topology to disk. -func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { - return err - } - return nil -} - -func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { - - id := fmt.Sprintf("node%d", i) - uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) - - node := &pilosa.Node{ - ID: id, - URI: uri, - } - - // add URI to common - //t.common.NodeIDs = append(t.common.NodeIDs, id) - //sort.Sort(t.common.NodeIDs) - - // add node to common - t.common.Nodes = append(t.common.Nodes, node) - - // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) - if err != nil { - return nil, err - } - - // holder - h := pilosa.NewHolder() - h.Path = path - - // cluster - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - c.Path = path - c.Topology = pilosa.NewTopology() - c.Holder = h - c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) - c.Node = node - c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator - c.Broadcaster = t - - // add nodes - if saveTopology { - for _, n := range t.common.Nodes { - c.AddNode(n) - } - } - - // Add this node to the TestCluster. - t.Clusters = append(t.Clusters, c) - - return c, nil -} - -// NewTestCluster returns a new instance of test.Cluster. -func NewTestCluster(n int) *TestCluster { - - tc := &TestCluster{ - common: &commonClusterSettings{}, - } - - // add clusters - for i := 0; i < n; i++ { - _, err := tc.addCluster(i, true) - if err != nil { - panic(err) - } - } - return tc -} - -// SetState sets the state of the cluster on each node. -func (t *TestCluster) SetState(state string) { - for _, c := range t.Clusters { - c.SetState(state) - } -} - -// Open opens all clusters in the test cluster. -func (t *TestCluster) Open() error { - for _, c := range t.Clusters { - if err := c.Open(); err != nil { - return err - } - if err := c.Holder.Open(); err != nil { - return err - } - if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { - return err - } - } - - // Start the listener on the coordinator. - if len(t.Clusters) == 0 { - return nil - } - t.Clusters[0].ListenForJoins() - - return nil -} - -// Close closes all clusters in the test cluster. -func (t *TestCluster) Close() error { - for _, c := range t.Clusters { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// TestCluster implements Broadcaster interface. - -// SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *TestCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: - // Apply the send message to all nodes (except the coordinator). - for _, c := range t.Clusters { - c.MergeClusterStatus(obj) - } - t.mu.RLock() - if obj.State == pilosa.ClusterStateNormal && t.resizing { - close(t.resizeDone) - } - t.mu.RUnlock() - } - - return nil -} - -// SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *TestCluster) SendAsync(pb proto.Message) error { - return nil -} - -// SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: - err := t.FollowResizeInstruction(obj) - if err != nil { - return err - } - case *internal.ResizeInstructionComplete: - coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) - } - return nil -} - -// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - - // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ - JobID: instr.JobID, - Node: instr.Node, - Error: "", - } - - // Stop processing on any error. - if err := func() error { - - // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) - instrNode := pilosa.DecodeNode(instr.Node) - destCluster := t.clusterByID(instrNode.ID) - - // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { - return err - } - - for _, src := range instr.Sources { - srcNode := pilosa.DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) - - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - if destFragment == nil { - // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) - v := f.View(src.View) - var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) - if err != nil { - return err - } - } - - buf := bytes.NewBuffer(nil) - - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) - - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err - } - } - - return nil - }(); err != nil { - complete.Error = err.Error() - } - - node := pilosa.DecodeNode(instr.Coordinator) - if err := t.SendTo(node, complete); err != nil { - return err - } - - return nil -} diff --git a/test/executor.go b/test/executor.go deleted file mode 100644 index 8cd6391c3..000000000 --- a/test/executor.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "net/http" - "strings" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" -) - -// Executor represents a test wrapper for pilosa.Executor. -type Executor struct { - *pilosa.Executor -} - -var remoteClient *http.Client - -func init() { - remoteClient = server.GetHTTPClient(nil) -} - -// NewExecutor returns a new instance of Executor. -// The executor always matches the uri of the first cluster node. -func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(remoteClient) - e := &Executor{Executor: executor} - e.Holder = holder - e.Cluster = cluster - e.Node = cluster.Nodes[0] - return e -} - -// MustParse parses s into a PQL query. Panic on error. -func MustParse(s string) *pql.Query { - q, err := pql.NewParser(strings.NewReader(s)).Parse() - if err != nil { - panic(err) - } - return q -} diff --git a/test/field.go b/test/field.go new file mode 100644 index 000000000..bb5048575 --- /dev/null +++ b/test/field.go @@ -0,0 +1,96 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/pilosa/pilosa" +) + +// Field represents a test wrapper for pilosa.Field. +type Field struct { + *pilosa.Field +} + +// newField returns a new instance of Field d/0. +func newField(opts pilosa.FieldOption) *Field { + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + field, err := pilosa.NewField(path, "i", "f", opts) + if err != nil { + panic(err) + } + return &Field{Field: field} +} + +// mustOpenField returns a new, opened field at a temporary path. Panic on error. +func mustOpenField(opts pilosa.FieldOption) *Field { + f := newField(opts) + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// close closes the field and removes the underlying data. +func (f *Field) close() error { + defer os.RemoveAll(f.Path()) + return f.Field.Close() +} + +// reopen closes the index and reopens it. +func (f *Field) reopen() error { + var err error + if err := f.Field.Close(); err != nil { + return err + } + + path, index, name := f.Path(), f.Index(), f.Name() + f.Field, err = pilosa.NewField(path, index, name, pilosa.OptFieldTypeDefault()) + if err != nil { + return err + } + + if err := f.Open(); err != nil { + return err + } + return nil +} + +// Ensure field can set its cache +func TestField_SetCacheSize(t *testing.T) { + f := mustOpenField(pilosa.OptFieldTypeDefault()) + defer f.close() + cacheSize := uint32(100) + + // Set & retrieve field cache size. + if err := f.SetCacheSize(cacheSize); err != nil { + t.Fatal(err) + } else if q := f.CacheSize(); q != cacheSize { + t.Fatalf("unexpected field cache size: %d", q) + } + + // Reload field and verify that it is persisted. + if err := f.reopen(); err != nil { + t.Fatal(err) + } else if q := f.CacheSize(); q != cacheSize { + t.Fatalf("unexpected field cache size (reopen): %d", q) + } +} diff --git a/test/fragment.go b/test/fragment.go deleted file mode 100644 index cd1111750..000000000 --- a/test/fragment.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - - "github.com/pilosa/pilosa" -) - -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth - -// Fragment is a test wrapper for pilosa.Fragment. -type Fragment struct { - *pilosa.Fragment - RowAttrStore pilosa.AttrStore -} - -// NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") - if err != nil { - panic(err) - } - file.Close() - - f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), - RowAttrStore: MustOpenAttrStore(), - } - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - return f -} - -// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - if cacheType == "" { - cacheType = pilosa.DefaultCacheType - } - f := NewFragment(index, frame, view, slice, cacheType) - - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the fragment and removes all underlying data. -func (f *Fragment) Close() error { - defer os.Remove(f.Path()) - defer os.Remove(f.CachePath()) - defer f.RowAttrStore.Close() - return f.Fragment.Close() -} - -// Reopen closes the fragment and reopens it as a new instance. -func (f *Fragment) Reopen() error { - cacheType := f.Fragment.CacheType - path := f.Path() - if err := f.Fragment.Close(); err != nil { - return err - } - - f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - if err := f.Open(); err != nil { - return err - } - return nil -} - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearColumns clears columns on a row. Panic on error. -func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// RowAttrStore provides simple storage for attributes. -type RowAttrStore struct { - attrs map[uint64]map[string]interface{} -} - -// NewRowAttrStore returns a new instance of RowAttrStore. -func NewRowAttrStore() *RowAttrStore { - return &RowAttrStore{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// RowAttrs returns the attributes set to a row id. -func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { - return s.attrs[id], nil -} - -// SetRowAttrs assigns a set of attributes to a row id. -func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { - s.attrs[id] = m -} - -// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk. -func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { - ipct := int(pct * 100) - for i := 0; i < SliceWidth*rowN; i++ { - if i%100 >= ipct { - continue - } - - rowIDs = append(rowIDs, uint64(i%SliceWidth)) - columnIDs = append(columnIDs, uint64(i/SliceWidth)) - } - return -} diff --git a/test/frame.go b/test/frame.go deleted file mode 100644 index e107b7d85..000000000 --- a/test/frame.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - "testing" - "time" - - "github.com/pilosa/pilosa" -) - -// Frame represents a test wrapper for pilosa.Frame. -type Frame struct { - *pilosa.Frame -} - -// NewFrame returns a new instance of Frame d/0. -func NewFrame() *Frame { - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - frame, err := pilosa.NewFrame(path, "i", "f") - if err != nil { - panic(err) - } - return &Frame{Frame: frame} -} - -// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame() *Frame { - f := NewFrame() - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the frame and removes the underlying data. -func (f *Frame) Close() error { - defer os.RemoveAll(f.Path()) - return f.Frame.Close() -} - -// Reopen closes the index and reopens it. -func (f *Frame) Reopen() error { - var err error - if err := f.Frame.Close(); err != nil { - return err - } - - path, index, name := f.Path(), f.Index(), f.Name() - f.Frame, err = pilosa.NewFrame(path, index, name) - if err != nil { - return err - } - - if err := f.Open(); err != nil { - return err - } - return nil -} - -// MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { - changed, err := f.SetBit(view, rowID, columnID, t) - if err != nil { - panic(err) - } - return changed -} - -// Ensure frame can set its cache -func TestFrame_SetCacheSize(t *testing.T) { - f := MustOpenFrame() - defer f.Close() - cacheSize := uint32(100) - - // Set & retrieve frame cache size. - if err := f.SetCacheSize(cacheSize); err != nil { - t.Fatal(err) - } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size: %d", q) - } - - // Reload frame and verify that it is persisted. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size (reopen): %d", q) - } -} diff --git a/test/handler.go b/test/handler.go index 401fa64ab..375ae8c3f 100644 --- a/test/handler.go +++ b/test/handler.go @@ -15,138 +15,15 @@ package test import ( - "context" "encoding/json" "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" + gohttp "net/http" ) -// Handler represents a test wrapper for pilosa.Handler. -type Handler struct { - *pilosa.Handler - Executor HandlerExecutor -} - -// NewHandler returns a new instance of Handler. -func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { - handler, err := pilosa.NewHandler(opts...) - if err != nil { - return nil, err - } - h := &Handler{ - Handler: handler, - } - h.API = pilosa.NewAPI() - h.Handler.API = h.API - h.Handler.API.Executor = &h.Executor - - // Handler test messages can no-op. - h.API.Broadcaster = pilosa.NopBroadcaster - - return h, nil -} - -// MustNewHandler returns a new instance of Handler. -func MustNewHandler(opts ...pilosa.HandlerOption) *Handler { - h, err := NewHandler(opts...) - if err != nil { - panic(err) - } - return h -} - -// HandlerExecutor is a mock implementing pilosa.Handler.Executor. -type HandlerExecutor struct { - cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) -} - -func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } - -func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, index, query, slices, opt) -} - -// Server represents a test wrapper for httptest.Server. -type Server struct { - *httptest.Server - Handler *Handler -} - -// NewServer returns a test server running on a random port. -func NewServer() *Server { - handler, err := NewHandler() - if err != nil { - panic(err) - } - s := &Server{ - Handler: handler, - } - s.Server = httptest.NewServer(s.Handler.Handler) - - // Handler test messages can no-op. - s.Handler.API.Broadcaster = pilosa.NopBroadcaster - // Create a default cluster on the handler - s.Handler.API.Cluster = NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - - return s -} - -// LocalStatus exists so that test.Server implements StatusHandler. -func (s *Server) LocalStatus() (proto.Message, error) { - return nil, nil -} - -// ClusterStatus exists so that test.Server implements StatusHandler. -func (s *Server) ClusterStatus() (proto.Message, error) { - id := "test-node" - uri := pilosa.DefaultURI() - node := &pilosa.Node{ - ID: id, - URI: *uri, - } - return &internal.ClusterStatus{ - ClusterID: "", - State: pilosa.ClusterStateNormal, - Nodes: pilosa.EncodeNodes([]*pilosa.Node{node}), - }, nil -} - -// HandleRemoteStatus just need to implement a nop to complete the Interface -func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil } - -// Host returns the hostname of the running server. -func (s *Server) Host() string { return MustParseURLHost(s.URL) } - -func (s *Server) HostURI() pilosa.URI { - uri, err := pilosa.NewURIFromAddress(s.URL) - if err != nil { - panic(err) - } - return *uri -} - -// MustParseURLHost parses rawurl and returns the hostname. Panic on error. -func MustParseURLHost(rawurl string) string { - u, err := url.Parse(rawurl) - if err != nil { - panic(err) - } - return u.Host -} - // MustNewHTTPRequest creates a new HTTP request. Panic on error. -func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { - req, err := http.NewRequest(method, urlStr, body) +func MustNewHTTPRequest(method, urlStr string, body io.Reader) *gohttp.Request { + req, err := gohttp.NewRequest(method, urlStr, body) + req.Header.Add("Accept", "application/json") if err != nil { panic(err) } @@ -161,12 +38,3 @@ func MustMarshalJSON(v interface{}) []byte { } return buf } - -// MustReadAll reads a reader into a buffer and returns it. Panic on error. -func MustReadAll(r io.Reader) []byte { - buf, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return buf -} diff --git a/test/holder.go b/test/holder.go index 97b399bea..ff87ae02c 100644 --- a/test/holder.go +++ b/test/holder.go @@ -17,6 +17,7 @@ package test import ( "io/ioutil" "os" + "time" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" @@ -80,47 +81,72 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption return &Index{Index: idx} } -// MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) +// Row returns a Row for a given field. +func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } - return f + row, err := f.Row(rowID) + if err != nil { + panic(err) + } + return row } -// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } - v, err := f.CreateViewIfNotExists(view) - if err != nil { - panic(err) - } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} + return f.RowAttrStore() } -// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum string) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) if err != nil { panic(err) } - v, err := f.CreateViewIfNotExists(view) + row, err := f.RowTime(rowID, t, quantum) if err != nil { panic(err) } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} + return row +} + +// SetBit clears a bit on the given field. +func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) + if err != nil { + panic(err) + } + _, err = f.SetBit(rowID, columnID, nil) + if err != nil { + panic(err) + } +} + +// ClearBit clears a bit on the given field. +func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) + if err != nil { + panic(err) + } + _, err = f.ClearBit(rowID, columnID) + if err != nil { + panic(err) + } +} + +// MustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + h.SetBit(index, field, rowID, columnID) + } } diff --git a/test/index.go b/test/index.go index 5354c85f9..ae2b997a9 100644 --- a/test/index.go +++ b/test/index.go @@ -26,8 +26,8 @@ type Index struct { *pilosa.Index } -// NewIndex returns a new instance of Index. -func NewIndex() *Index { +// newIndex returns a new instance of Index. +func newIndex() *Index { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { panic(err) @@ -41,7 +41,7 @@ func NewIndex() *Index { // MustOpenIndex returns a new, opened index at a temporary path. Panic on error. func MustOpenIndex() *Index { - index := NewIndex() + index := newIndex() if err := index.Open(); err != nil { panic(err) } @@ -73,20 +73,20 @@ func (i *Index) Reopen() error { return nil } -// CreateFrame creates a frame with the given options. -func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrame(name, opt) +// CreateField creates a field with the given options. +func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, error) { + f, err := i.Index.CreateField(name, opts...) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Field{Field: f}, nil } -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrameIfNotExists(name, opt) +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) { + f, err := i.Index.CreateFieldIfNotExists(name, opts) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Field{Field: f}, nil } diff --git a/test/logger.go b/test/logger.go index b4a0079b1..60af32acb 100644 --- a/test/logger.go +++ b/test/logger.go @@ -20,20 +20,20 @@ import ( "io/ioutil" ) -// BufferLogger represents a test Logger that holds log messages +// bufferLogger represents a test Logger that holds log messages // in a buffer for review. -type BufferLogger struct { +type bufferLogger struct { buf *bytes.Buffer } // NewBufferLogger returns a new instance of BufferLogger. -func NewBufferLogger() *BufferLogger { - return &BufferLogger{ +func NewBufferLogger() *bufferLogger { + return &bufferLogger{ buf: &bytes.Buffer{}, } } -func (b *BufferLogger) Printf(format string, v ...interface{}) { +func (b *bufferLogger) Printf(format string, v ...interface{}) { s := fmt.Sprintf(format, v...) _, err := b.buf.WriteString(s) if err != nil { @@ -41,8 +41,8 @@ func (b *BufferLogger) Printf(format string, v ...interface{}) { } } -func (b *BufferLogger) Debugf(format string, v ...interface{}) {} +func (b *bufferLogger) Debugf(format string, v ...interface{}) {} -func (b *BufferLogger) ReadAll() ([]byte, error) { +func (b *bufferLogger) ReadAll() ([]byte, error) { return ioutil.ReadAll(b.buf) } diff --git a/test/pilosa.go b/test/pilosa.go index 4a741ea8b..e1022ce8b 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -19,64 +19,49 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "os" "strings" "testing" - "time" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/boltdb" - "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" - "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) //////////////////////////////////////////////////////////////////////////////////// -// Main represents a test wrapper for main.Main. -type Main struct { +// Command represents a test wrapper for server.Command. +type Command struct { *server.Command - Stdin bytes.Buffer - Stdout bytes.Buffer - Stderr bytes.Buffer + commandOptions []server.CommandOption + + stdin bytes.Buffer + stdout bytes.Buffer + stderr bytes.Buffer } -type MainOpt func(m *Main) error - -func OptAntiEntropyInterval(dur time.Duration) MainOpt { - return func(m *Main) error { - m.Command.Config.AntiEntropy.Interval = toml.Duration(dur) +func OptAllowedOrigins(origins []string) server.CommandOption { + return func(m *server.Command) error { + m.Config.Handler.AllowedOrigins = origins return nil } } -// NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain(opts ...MainOpt) *Main { +// newCommand returns a new instance of Main with a temporary data directory and random port. +func newCommand(opts ...server.CommandOption) *Command { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) } - m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} + m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts} m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true - m.Command.Stdin = &m.Stdin - m.Command.Stdout = &m.Stdout - m.Command.Stderr = &m.Stderr - for _, opt := range opts { - err := opt(m) - if err != nil { - panic(err) - } - - } - err = m.SetupServer() - if err != nil { - panic(err) - } + m.Command.Stdin = &m.stdin + m.Command.Stdout = &m.stdout + m.Command.Stderr = &m.stderr if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) @@ -86,56 +71,17 @@ func NewMain(opts ...MainOpt) *Main { return m } -// NewMainWithCluster returns a new instance of Main with clustering enabled. -func NewMainWithCluster(isCoordinator bool, opts ...MainOpt) *Main { - m := NewMain(opts...) +// NewCommandNode returns a new instance of Command with clustering enabled. +func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { + m := newCommand(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m } -// MustRunMainWithCluster ruturns a running array of *Main where -// all nodes are joined via memberlist (i.e. clustering enabled). -func MustRunMainWithCluster(t *testing.T, size int, opts ...MainOpt) []*Main { - ma, err := runMainWithCluster(size, opts...) - if err != nil { - t.Fatalf("new main array with cluster: %v", err) - } - return ma -} - -// runMainWithCluster runs an array of *Main where all nodes are -// joined via memberlist (i.e. clustering enabled). -func runMainWithCluster(size int, opts ...MainOpt) ([]*Main, error) { - if size == 0 { - return nil, errors.New("cluster must contain at least one node") - } - - mains := make([]*Main, size) - - gossipHost := "localhost" - gossipPort := 0 - var err error - var gossipSeeds = make([]string, size) - - for i := 0; i < size; i++ { - m := NewMainWithCluster(i == 0, opts...) - m.Config.Cluster.Disabled = false - - gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) - if err != nil { - return nil, errors.Wrap(err, "RunWithTransport") - } - - mains[i] = m - } - - return mains, nil -} - -// MustRunMain returns a new, running Main. Panic on error. -func MustRunMain() *Main { - m := NewMain() +// MustRunCommand returns a new, running Main. Panic on error. +func MustRunCommand() *Command { + m := newCommand() m.Config.Metric.Diagnostics = false // Disable diagnostics. if err := m.Start(); err != nil { panic(err) @@ -143,29 +89,29 @@ func MustRunMain() *Main { return m } +// GossipAddress returns the address on which gossip is listening after a Main +// has been setup. Useful to pass as a seed to other nodes when creating and +// testing clusters. +func (m *Command) GossipAddress() string { + return m.GossipTransport().URI.String() +} + // Close closes the program and removes the underlying data directory. -func (m *Main) Close() error { +func (m *Command) Close() error { defer os.RemoveAll(m.Config.DataDir) return m.Command.Close() } // Reopen closes the program and reopens it. -func (m *Main) Reopen() error { +func (m *Command) Reopen() error { if err := m.Command.Close(); err != nil { return err } // Create new main with the same config. config := m.Command.Config - m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) + m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...) m.Command.Config = config - err := m.SetupServer() - if err != nil { - return errors.Wrap(err, "setting up server") - } - - m.Server.NewAttrStore = boltdb.NewAttrStore - m.Server.Holder.NewAttrStore = m.Server.NewAttrStore // Run new program. if err := m.Start(); err != nil { @@ -174,72 +120,12 @@ func (m *Main) Reopen() error { return nil } -// RunWithTransport runs Main and returns the dynamically allocated gossip port. -func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (seed string, err error) { - defer close(m.Started) - - /* - TEST: - - SetupServer (just static settings from config) - - OpenListener (sets Server.Name to use in gossip) - - NewTransport (gossip) - - SetupNetworking (does the gossip or static stuff) - uses Server.Name - - Open server - - PRODUCTION: - - SetupServer (just static settings from config) - - SetupNetworking (does the gossip or static stuff) - calls NewTransport - - Open server - calls OpenListener - */ - - // SetupServer - err = m.SetupServer() - if err != nil { - return seed, err - } - - // Open gossip transport to use in SetupServer. - transport, err := gossip.NewTransport(host, bindPort, nil) - if err != nil { - return seed, err - } - m.GossipTransport = transport - - if len(joinSeeds) != 0 { - m.Config.Gossip.Seeds = joinSeeds - } else { - m.Config.Gossip.Seeds = []string{transport.URI.String()} - } - - seed = transport.URI.String() - - // SetupNetworking - err = m.SetupNetworking() - if err != nil { - return seed, err - } - - if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { - return seed, err - } - - m.Server.Cluster.Static = false - - // Initialize server. - err = m.Server.Open() - if err != nil { - return seed, err - } - - return seed, nil -} - // URL returns the base URL string for accessing the running program. -func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } +func (m *Command) URL() string { return m.API.Node().URI.String() } // Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) +func (m *Command) Client() *http.InternalClient { + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } @@ -247,15 +133,15 @@ func (m *Main) Client() *pilosa.InternalHTTPClient { } // Query executes a query against the program through the HTTP API. -func (m *Main) Query(index, rawQuery, query string) (string, error) { +func (m *Command) Query(index, rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) } return resp.Body, nil } -func (m *Main) RecalculateCaches() error { +func (m *Command) RecalculateCaches() error { resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") if resp.StatusCode != 204 { return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) @@ -263,15 +149,99 @@ func (m *Main) RecalculateCaches() error { return nil } +// Cluster represents a Pilosa cluster (multiple Command instances) +type Cluster []*Command + +// Start runs a Cluster +func (c Cluster) Start() error { + var gossipSeeds = make([]string, len(c)) + for i, cc := range c { + cc.Config.Gossip.Port = "0" + cc.Config.Gossip.Seeds = gossipSeeds[:i] + if err := cc.Start(); err != nil { + return errors.Wrapf(err, "starting server %d", i) + } + gossipSeeds[i] = cc.GossipAddress() + } + return nil +} + +// Stop stops a Cluster +func (c Cluster) Close() error { + for i, cc := range c { + if err := cc.Close(); err != nil { + return errors.Wrapf(err, "stopping server %d", i) + } + } + return nil +} + +// MustNewCluster creates a new cluster +func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { + c, err := newCluster(size, opts...) + if err != nil { + t.Fatalf("new cluster: %v", err) + } + return c +} + +// newCluster creates a new cluster +func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { + if size == 0 { + return nil, errors.New("cluster must contain at least one node") + } + if len(opts) != size && len(opts) != 0 && len(opts) != 1 { + return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes") + } + + cluster := make(Cluster, size) + for i := 0; i < size; i++ { + var commandOpts []server.CommandOption + if len(opts) > 0 { + commandOpts = opts[i%len(opts)] + } + m := NewCommandNode(i == 0, commandOpts...) + cluster[i] = m + } + + return cluster, nil +} + +// runCluster creates and starts a new cluster +func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { + cluster, err := newCluster(size, opts...) + if err != nil { + return nil, errors.Wrap(err, "new cluster") + } + if err = cluster.Start(); err != nil { + return nil, errors.Wrap(err, "starting cluster") + } + return cluster, nil +} + +// MustRunCluster creates and starts a new cluster +func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { + c, err := runCluster(size, opts...) + if err != nil { + t.Fatalf("run cluster: %v", err) + } + return c +} + //////////////////////////////////////////////////////////////////////////////////// // MustDo executes http.Do() with an http.NewRequest(). Panic on error. func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) - if err != nil { - panic(err) - } - resp, err := http.DefaultClient.Do(req) + req, err := gohttp.NewRequest( + method, + urlStr, + strings.NewReader(body), + ) + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := gohttp.DefaultClient.Do(req) if err != nil { panic(err) } @@ -287,6 +257,6 @@ func MustDo(method, urlStr string, body string) *httpResponse { // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { - *http.Response + *gohttp.Response Body string } diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 583a244c0..25bb808df 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -15,8 +15,10 @@ package test_test import ( + "context" "encoding/json" "net/http" + "strings" "testing" "github.com/pilosa/pilosa" @@ -25,19 +27,29 @@ import ( func TestNewCluster(t *testing.T) { numNodes := 3 - cluster := test.MustRunMainWithCluster(t, numNodes) - coordinator := cluster[0].Server.Cluster.Coordinator + cluster := test.MustRunCluster(t, numNodes) + + coordinator := getCoordinator(cluster[0]) for i := 1; i < numNodes; i++ { - if coordi := cluster[i].Server.Cluster.Coordinator; coordi != coordinator { + if coordi := getCoordinator(cluster[i]); coordi != coordinator { t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator) } } + req, err := http.NewRequest( + "GET", + cluster[0].URL()+"/status", + strings.NewReader(""), + ) - response, err := http.Get("http://" + cluster[0].Server.Addr().String() + "/status") + req.Header.Set("Accept", "application/json") + + resp, err := http.DefaultClient.Do(req) if err != nil { - t.Fatalf("getting schema: %v", err) + t.Fatalf("sending request: %v", err) } - dec := json.NewDecoder(response.Body) + defer resp.Body.Close() + + dec := json.NewDecoder(resp.Body) body := struct { State string Nodes []struct { @@ -65,3 +77,13 @@ func TestNewCluster(t *testing.T) { t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State) } } + +func getCoordinator(m *test.Command) string { + hosts := m.API.Hosts(context.Background()) + for _, host := range hosts { + if host.IsCoordinator { + return host.ID + } + } + panic("no coordinator in cluster") +} diff --git a/time.go b/time.go index 517292071..ecfdcad3d 100644 --- a/time.go +++ b/time.go @@ -70,17 +70,8 @@ func (q TimeQuantum) Type() string { return "TimeQuantum" } -// ParseTimeQuantum parses v into a time quantum. -func ParseTimeQuantum(v string) (TimeQuantum, error) { - q := TimeQuantum(strings.ToUpper(v)) - if !q.Valid() { - return "", ErrInvalidTimeQuantum - } - return q, nil -} - -// ViewByTimeUnit returns the view name for time with a given quantum unit. -func ViewByTimeUnit(name string, t time.Time, unit rune) string { +// viewByTimeUnit returns the view name for time with a given quantum unit. +func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -95,11 +86,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string { } } -// ViewsByTime returns a list of views for a given timestamp. -func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { +// viewsByTime returns a list of views for a given timestamp. +func viewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - view := ViewByTimeUnit(name, t, unit) + view := viewByTimeUnit(name, t, unit) if view == "" { continue } @@ -108,8 +99,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { return a } -// ViewsByTimeRange returns a list of views to traverse to query a time range. -func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// viewsByTimeRange returns a list of views to traverse to query a time range. +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -127,7 +118,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -138,7 +129,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -148,7 +139,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -164,16 +155,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'Y')) + results = append(results, viewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_internal_test.go similarity index 61% rename from time_test.go rename to time_internal_test.go index 233a2ae1a..bd4afae43 100644 --- a/time_test.go +++ b/time_internal_test.go @@ -12,28 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "reflect" + "strings" "testing" "time" - - "github.com/pilosa/pilosa" ) // Ensure string can be parsed into time quantum. func TestParseTimeQuantum(t *testing.T) { t.Run("OK", func(t *testing.T) { - if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + if q, err := parseTimeQuantum("YMDH"); err != nil { t.Fatalf("unexpected error: %s", err) - } else if q != pilosa.TimeQuantum("YMDH") { + } else if q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %#v", q) } }) t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { - if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + if _, err := parseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { t.Fatalf("unexpected error: %s", err) } }) @@ -44,127 +43,136 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) } -// Ensure all applicable frame names can be generated when mutating a time bit. +// Ensure all applicable field names can be generated when mutating a time bit. func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) + a := viewsByTime("F", ts, mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } }) } -// Ensure sets of frames can be returned for a given time range. +// Ensure sets of fields can be returned for a given time range. func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) } -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" +// defaultTimeLayout is the time layout used by the tests. +const defaultTimeLayout = "2006-01-02 15:04" -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) +// mustParseTime parses value using DefaultTimeLayout. Panic on error. +func mustParseTime(value string) time.Time { + v, err := time.Parse(defaultTimeLayout, value) if err != nil { panic(err) } return v } -// MustParseTimeQuantum parses v into a time quantum. Panic on error. -func MustParseTimeQuantum(v string) pilosa.TimeQuantum { - q, err := pilosa.ParseTimeQuantum(v) +// mustParseTimeQuantum parses v into a time quantum. Panic on error. +func mustParseTimeQuantum(v string) TimeQuantum { + q, err := parseTimeQuantum(v) if err != nil { panic(err) } return q } + +// parseTimeQuantum parses v into a time quantum. +func parseTimeQuantum(v string) (TimeQuantum, error) { + q := TimeQuantum(strings.ToUpper(v)) + if !q.Valid() { + return "", ErrInvalidTimeQuantum + } + return q, nil +} diff --git a/translate.go b/translate.go new file mode 100644 index 000000000..2341efe51 --- /dev/null +++ b/translate.go @@ -0,0 +1,999 @@ +package pilosa + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/cespare/xxhash" +) + +const ( + LogEntryTypeInsertColumn = 1 + LogEntryTypeInsertRow = 2 +) + +const ( + defaultReplicationRetryInterval = 1 * time.Second +) + +var ( + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") +) + +// TranslateStore is the storage for translation string-to-uint64 values. +type TranslateStore interface { + TranslateColumnsToUint64(index string, values []string) ([]uint64, error) + TranslateColumnToString(index string, values uint64) (string, error) + + TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) + TranslateRowToString(index, frame string, values uint64) (string, error) + + // Returns a reader from the given offset of the raw data file. + // The returned reader must be closed by the caller when done. + Reader(ctx context.Context, off int64) (io.ReadCloser, error) +} + +// Ensure type implements interface. +var _ TranslateStore = &TranslateFile{} + +// TranslateFile is an on-disk storage engine for translating string-to-uint64 values. +type TranslateFile struct { + mu sync.RWMutex + data []byte + file *os.File + w *bufio.Writer + n int64 + writeNotify chan struct{} + + once sync.Once + wg sync.WaitGroup + closing chan struct{} + + cols map[string]*index + rows map[frameKey]*index + + Path string + mapSize int + + // If non-nil, data is streamed from a primary and this is a read-only store. + PrimaryTranslateStore TranslateStore + + // Delay after attempting to connect to a primary that the store will retry. + replicationRetryInterval time.Duration +} + +// NewTranslateFile returns a new instance of TranslateFile. +func NewTranslateFile() *TranslateFile { + return &TranslateFile{ + writeNotify: make(chan struct{}), + closing: make(chan struct{}), + cols: make(map[string]*index), + rows: make(map[frameKey]*index), + + mapSize: defaultMapSize, + + replicationRetryInterval: defaultReplicationRetryInterval, + } +} + +func (s *TranslateFile) Open() (err error) { + // Open writer & buffered writer. + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + return err + } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { + return err + } + s.w = bufio.NewWriter(s.file) + + // Memory map data file. + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + return err + } + + // Replay the log. + if err := s.replayEntries(); err != nil { + return err + } + + // Stream from primary, if available. + if s.PrimaryTranslateStore != nil { + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorReplication() }() + } + + return nil +} + +func (s *TranslateFile) Close() (err error) { + s.once.Do(func() { + close(s.closing) + + if s.file != nil { + if e := s.file.Close(); e != nil && err == nil { + err = e + } + } + if s.data != nil { + if e := syscall.Munmap(s.data); e != nil && err == nil { + err = e + } + } + }) + s.wg.Wait() + return err +} + +// Closing returns a channel that is closed when the store is closed. +func (s *TranslateFile) Closing() <-chan struct{} { + return s.closing +} + +// size returns the number of bytes in use in the data file. +func (s *TranslateFile) size() int64 { + s.mu.RLock() + n := s.n + s.mu.RUnlock() + return n +} + +// isReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) isReadOnly() bool { + return s.PrimaryTranslateStore != nil +} + +// WriteNotify returns a channel that is closed when a new entry is written. +func (s *TranslateFile) WriteNotify() <-chan struct{} { + s.mu.RLock() + ch := s.writeNotify + s.mu.RUnlock() + return ch +} + +func (s *TranslateFile) appendEntry(entry *LogEntry) error { + offset := s.n + + // Append entry to the end of the WAL. + n, err := entry.WriteTo(s.w) + if err != nil { + return err + } else if err := s.w.Flush(); err != nil { + return err + } + + // Move position forward. + s.n += n + + // Apply the entry to the current state. + if err := s.applyEntry(entry, offset); err != nil { + return err + } else if err := s.file.Sync(); err != nil { + return err + } + + // Notify others of write update. + close(s.writeNotify) + s.writeNotify = make(chan struct{}) + + return nil +} + +func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { + // Move offset to the start of the id/key pairs. + offset += entry.headerSize() + + var idx *index + switch entry.Type { + case LogEntryTypeInsertColumn: + idx = s.col(string(entry.Index)) + + case LogEntryTypeInsertRow: + idx = s.row(string(entry.Index), string(entry.Frame)) + + default: + return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) + } + + // Insert id/key pairs into index. + for i, id := range entry.IDs { + key := entry.Keys[i] + + // Determine key offset based on ID size. + sz := int64(uVarintSize(id)) + idx.insert(id, offset+sz) + + // Move sequence forward. + if id > idx.seq { + idx.seq = id + } + + // Move offset forward. + offset += sz + int64(uVarintSize(uint64(len(key)))) + int64(len(key)) + } + + return nil +} + +func (s *TranslateFile) replayEntries() error { + // Build a reader from the memory-map data. + fi, err := os.Stat(s.Path) + if err != nil { + return err + } + r := bytes.NewReader(s.data[:fi.Size()]) + + // Iterate over each entry and reapply. + for { + offset := s.n + + var entry LogEntry + if n, err := entry.ReadFrom(r); err == io.EOF { + return nil + } else if err != nil { + return err + } else { + s.n += n + } + + if err := s.applyEntry(&entry, offset); err != nil { + return err + } + } +} + +// monitorReplication is executed in a separate goroutine and continually streams +// from the primary store until this store is closed. +func (s *TranslateFile) monitorReplication() { + // Create context that will cancel on close. + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-s.closing; cancel() }() + + // Keep attempting to replicate until the store closes. + for { + if err := s.replicate(ctx); err != nil { + log.Printf("pilosa: replication error: %s", err) + } + + select { + case <-s.closing: + return + case <-time.After(s.replicationRetryInterval): + log.Printf("pilosa: reconnecting to primary replica") + } + } +} + +func (s *TranslateFile) replicate(ctx context.Context) error { + off := s.size() + + // Connect to remote primary. + log.Printf("pilosa: replicating from offset %d", off) + rc, err := s.PrimaryTranslateStore.Reader(ctx, off) + if err != nil { + return err + } + defer rc.Close() + + // Wrap in bufferred I/O so it implements io.ByteReader. + bufr := bufio.NewReader(rc) + + // Continually read new entries from primary and append to local store. + for { + // Read next available entry. + var entry LogEntry + if _, err := entry.ReadFrom(bufr); err == io.EOF { + return nil + } else if err != nil { + return err + } + s.mu.Lock() + // Write to local store. + if err := s.appendEntry(&entry); err != nil { + s.mu.Unlock() + return err + } + s.mu.Unlock() + } +} + +func (s *TranslateFile) col(index string) *index { + idx := s.cols[index] + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + return idx +} + +func (s *TranslateFile) row(index, frame string) *index { + idx := s.rows[frameKey{index, frame}] + if idx == nil { + idx = newIndex(s.data) + s.rows[frameKey{index, frame}] = idx + } + return idx +} + +// TranslateColumnsToUint64 converts values to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.isReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertColumn, + Index: []byte(index), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + + ret[i] = v + + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.keyByID(value); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.isReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.rows[key] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertRow, + Index: []byte(index), + Frame: []byte(frame), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + ret[i] = v + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.keyByID(id); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +// Reader returns a reader that streams the underlying data file. +func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + rc := newTranslateFileReader(ctx, s, offset) + if err := rc.Open(); err != nil { + return nil, err + } + return rc, nil +} + +type LogEntry struct { + Type uint8 + Index []byte + Frame []byte + + IDs []uint64 + Keys [][]byte + + // Length of the entry, in bytes. + // This is only populated after ReadFrom() or WriteTo(). + Length uint64 +} + +// headerSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) headerSize() int64 { + sz := uVarintSize(e.Length) + // total entry length + 1 + // type + uVarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data + uVarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + uVarintSize(uint64(len(e.IDs))) // ID/Key pair count + return int64(sz) +} + +// ReadFrom deserializes a LogEntry from r. r must be a ByteReader. +func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { + br := r.(io.ByteReader) + + // Read the entry length. + if e.Length, err = binary.ReadUvarint(br); err != nil { + return int64(uVarintSize(e.Length)), err + } + + // Slurp entire entry and replace reader. + buf := make([]byte, e.Length) + n, err := io.ReadFull(r, buf) + n64 := int64(n + uVarintSize(e.Length)) + if err != nil { + return n64, err + } + bufr := bytes.NewReader(buf) + br, r = bufr, bufr + + // Read the entry type. + if err := binary.Read(r, binary.BigEndian, &e.Type); err != nil { + return n64, err + } + + // Read index name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Index = nil + } else { + e.Index = make([]byte, sz) + if _, err := io.ReadFull(r, e.Index); err != nil { + return n64, err + } + } + + // Read frame name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Frame = nil + } else { + e.Frame = make([]byte, sz) + if _, err := io.ReadFull(r, e.Frame); err != nil { + return n64, err + } + } + + // Read key count. + if n, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if n == 0 { + e.IDs, e.Keys = nil, nil + } else { + e.IDs, e.Keys = make([]uint64, n), make([][]byte, n) + } + + // Read each id/key pairs. + for i := range e.Keys { + // Read identifier. + if e.IDs[i], err = binary.ReadUvarint(br); err != nil { + return n64, err + } + + // Read key. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz > 0 { + e.Keys[i] = make([]byte, sz) + if _, err := io.ReadFull(r, e.Keys[i]); err != nil { + return n64, err + } + } + } + return n64, nil +} + +// WriteTo serializes a LogEntry to w. +func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { + var buf bytes.Buffer + b := make([]byte, binary.MaxVarintLen64) + + // Write the entry type. + if err := binary.Write(&buf, binary.BigEndian, e.Type); err != nil { + return 0, err + } + + // Write the index name. + sz := binary.PutUvarint(b, uint64(len(e.Index))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Index); err != nil { + return 0, err + } + + // Write frame name. + sz = binary.PutUvarint(b, uint64(len(e.Frame))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Frame); err != nil { + return 0, err + } + + // Write key count. + sz = binary.PutUvarint(b, uint64(len(e.IDs))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write each id/key pairs. + for i := range e.Keys { + // Write identifier. + sz = binary.PutUvarint(b, e.IDs[i]) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write key. + sz = binary.PutUvarint(b, uint64(len(e.Keys[i]))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Keys[i]); err != nil { + return 0, err + } + } + + // Write buffer size. + e.Length = uint64(buf.Len()) + sz = binary.PutUvarint(b, e.Length) + if n, err := w.Write(b[:sz]); err != nil { + return int64(n), err + } + + // Write buffer. + n, err := buf.WriteTo(w) + return int64(sz) + n, err +} + +// validLogEntriesLen returns the maximum length of p that contains valid entries. +func validLogEntriesLen(p []byte) (n int) { + r := bytes.NewReader(p) + for { + if sz, err := binary.ReadUvarint(r); err != nil { + return n + } else if off, err := r.Seek(int64(sz), io.SeekCurrent); err != nil { + return n + } else if off > int64(len(p)) { + return n + } else { + n = int(off) + } + } +} + +type frameKey struct { + index string + frame string +} + +const defaultLoadFactor = 90 + +// index represents a two-way index between IDs and keys. +type index struct { + seq uint64 // autoincrement sequence + data []byte // memory-mapped file containing key data + + // RHH hashmap for id-to-offset mapping. + // This is required so we don't need to store key data on the heap. + // https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf + elems []elem // id/offset key pairs + n uint64 // number of inuse elements + mask uint64 // mask applied for modulus + threshold uint64 // threshold when capacity doubles + loadFactor int // factor used to calculate threshold + + // Builtin hashmap for offset-to-id mapping. + offsetsByID map[uint64]int64 +} + +func newIndex(data []byte) *index { + idx := &index{ + data: data, + offsetsByID: make(map[uint64]int64), + + loadFactor: defaultLoadFactor, + } + idx.alloc(pow2(uint64(256))) + return idx +} + +// keyByID returns the key for a given ID, if it exists. +func (idx *index) keyByID(id uint64) ([]byte, bool) { + offset, ok := idx.offsetsByID[id] + if !ok { + return nil, false + } + return idx.lookupKey(offset), true +} + +// idByKey returns the ID for a given key, if it exists. +func (idx *index) idByKey(key []byte) (uint64, bool) { + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + if e := &idx.elems[pos]; e.hash == 0 { + return 0, false + } else if dist > idx.dist(e.hash, pos) { + return 0, false + } else if e.hash == hash && bytes.Equal(idx.lookupKey(e.offset), key) { + return e.id, true + } + + pos = (pos + 1) & idx.mask + dist++ + } +} + +// insert adds the id/offset pair to the index. +// This function will resize the map if it crosses the threshold. +func (idx *index) insert(id uint64, offset int64) { + idx.n++ + + // Add to reverse lookup. + idx.offsetsByID[id] = offset + + // Grow the map if we've run out of slots. + if idx.n > idx.threshold { + elems, capacity := idx.elems, uint64(len(idx.elems)) + idx.alloc(uint64(len(idx.elems) * 2)) + + for i := uint64(0); i < capacity; i++ { + e := &elems[i] + if e.hash == 0 { + continue + } + idx.insertIDbyOffset(e.offset, e.id) + } + } + + // If the key was overwritten then decrement the size. + if overwritten := idx.insertIDbyOffset(offset, id); overwritten { + idx.n-- + } +} + +// insertIDbyOffset writes to the RHH id-by-offset map. +func (idx *index) insertIDbyOffset(offset int64, id uint64) (overwritten bool) { + key := idx.lookupKey(offset) + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + e := &idx.elems[pos] + + // Exit if a matching or empty slot exists. + if e.hash == 0 { + e.hash, e.offset, e.id = hash, offset, id + return false + } else if bytes.Equal(idx.lookupKey(e.offset), key) { + e.hash, e.offset, e.id = hash, offset, id + return true + } + + // Swap if current element has a lower probe distance. + d := idx.dist(e.hash, pos) + if d < dist { + hash, e.hash = e.hash, hash + offset, e.offset = e.offset, offset + id, e.id = e.id, id + dist = d + } + + // Move position forward. + pos = (pos + 1) & idx.mask + dist++ + } +} + +// lookupKey returns the key at the given offset in the memory-mapped file. +func (idx *index) lookupKey(offset int64) []byte { + data := idx.data[offset:] + n, sz := binary.Uvarint(data) + if sz == 0 { + return nil + } + return data[sz : sz+int(n)] +} + +func (idx *index) alloc(capacity uint64) { + idx.elems = make([]elem, capacity) + idx.threshold = (capacity * uint64(idx.loadFactor)) / 100 + idx.mask = uint64(capacity - 1) +} + +func (idx *index) dist(hash, i uint64) uint64 { + return (i + uint64(len(idx.elems)) - (hash & idx.mask)) & idx.mask +} + +type elem struct { + offset int64 + id uint64 + hash uint64 +} + +func (e *elem) reset() { + e.offset = 0 + e.id = 0 + e.hash = 0 +} + +func hashKey(key []byte) uint64 { + h := xxhash.Sum64(key) + if h == 0 { + h = 1 + } + return h +} + +func pow2(v uint64) uint64 { + for i := uint64(2); i < 1<<62; i *= 2 { + if i >= v { + return i + } + } + panic("unreachable") +} + +// translateFileReader implements a reader that continuously streams data from a store. +type translateFileReader struct { + ctx context.Context + store *TranslateFile + file *os.File + offset int64 + notify <-chan struct{} + + once sync.Once + closing chan struct{} +} + +// newTranslateFileReader returns a new instance of TranslateFileReader. +func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *translateFileReader { + return &translateFileReader{ + ctx: ctx, + store: store, + offset: offset, + notify: store.WriteNotify(), + closing: make(chan struct{}), + } +} + +// Open initializes the reader. +func (r *translateFileReader) Open() (err error) { + if r.file, err = os.Open(r.store.Path); err != nil { + return err + } + return nil +} + +// Close closes the underlying file reader. +func (r *translateFileReader) Close() error { + r.once.Do(func() { close(r.closing) }) + + if r.file != nil { + return r.file.Close() + } + return nil +} + +// Read reads the next section of the available data to p. This should always +// read from the start of an entry and read n bytes to the end of another entry. +func (r *translateFileReader) Read(p []byte) (n int, err error) { + for { + // Obtain notification channel before we check for new data. + notify := r.store.WriteNotify() + + // Exit if we can read one or more valid entries or we receive an error. + if n, err = r.read(p); n > 0 || err != nil { + return n, err + } + + // Wait for new data or close. + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.closing: + return 0, ErrTranslateStoreReaderClosed + case <-r.store.Closing(): + return 0, ErrTranslateStoreClosed + case <-notify: + continue + } + } +} + +// read writes the bytes for zero or more valid entries to p. +func (r *translateFileReader) read(p []byte) (n int, err error) { + sz := r.store.size() + + // Exit if there is no new data. + if sz < r.offset { + return 0, fmt.Errorf("pilosa: translate store reader past file size: sz=%d off=%d", sz, r.offset) + } else if sz == r.offset { + return 0, nil + } + + // Shorten buffer to maximum read size. + if max := sz - r.offset; int64(len(p)) > max { + p = p[:max] + } + + // Read data from file at offset. + // Limit the number of bytes read to only whole entries. + n, err = r.file.ReadAt(p, r.offset) + n = validLogEntriesLen(p[:n]) + r.offset += int64(n) + return n, err +} + +// Copied & modified from encoding/binary. +func uVarintSize(x uint64) (i int) { + for x >= 0x80 { + x >>= 7 + i++ + } + return i + 1 +} diff --git a/translate_mapsize_386.go b/translate_mapsize_386.go new file mode 100644 index 000000000..b8beafa13 --- /dev/null +++ b/translate_mapsize_386.go @@ -0,0 +1,5 @@ +package pilosa + +// DefaultMapSize is the default size of mapped memory for the translate store. +// It is passed as an int to syscall.Mmap and so must be < 2^31 +const DefaultMapSize = (1 << 31) - 1 // 2GB diff --git a/translate_mapsize_all64bitsystems.go b/translate_mapsize_all64bitsystems.go new file mode 100644 index 000000000..605d6270a --- /dev/null +++ b/translate_mapsize_all64bitsystems.go @@ -0,0 +1,8 @@ +// +build !386 + +package pilosa + +// defaultMapSize is the default size of mapped memory for the translate store. +// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on +// 64bit systems. +const defaultMapSize = 10 * (1 << 30) // 10GB diff --git a/translate_test.go b/translate_test.go new file mode 100644 index 000000000..c73853d69 --- /dev/null +++ b/translate_test.go @@ -0,0 +1,565 @@ +package pilosa_test + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pilosa/pilosa" +) + +func TestTranslateFile_TranslateColumn(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateColumn_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateColumnsToUint64("IDX0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_TranslateRow(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateRow_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_Reader(t *testing.T) { + t.Run("NoOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + rc, err := s.Reader(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b. + var entry pilosa.LogEntry + if n, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if n != 14 { + t.Fatalf("unexpected n: %d", n) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{1}, + Keys: [][]byte{[]byte("foo")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Read second entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + + // Write new entry. + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil { + t.Fatal(err) + } + + // Read new entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{2}, + Keys: [][]byte{[]byte("xyz")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Close reader and ensure it returns EOF. + if err := rc.Close(); err != nil { + t.Fatal(err) + } else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("WithOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Start offset after the first entry. + rc, err := s.Reader(context.Background(), 14) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // This should be the second entry. + var entry pilosa.LogEntry + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { + // Create a primary store that accepts writes. + primary := MustOpenTranslateFile() + defer primary.MustClose() + + // Create a replica that accepts writes from primary. + replica := NewTranslateFile() + replica.PrimaryTranslateStore = primary + if err := replica.Open(); err != nil { + t.Fatal(err) + } + defer replica.MustClose() + + // Write to the primary. + if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until writes appear. + if err := retryFor(2*time.Second, func() error { + // Verify that replica have received writes. + if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil { + return err + } else if value != "foo" { + return fmt.Errorf("unexpected column 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + return err + } else if value != "bar" { + return fmt.Errorf("unexpected row 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected row 2 value: %s", value) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect primary store & write more values. + if err := primary.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected column 2 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect replica store & write more values. + if err := replica.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil { + return err + } else if value != "foobar" { + return fmt.Errorf("unexpected column 3 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/batchSize) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} + +type TranslateFile struct { + *pilosa.TranslateFile +} + +func NewTranslateFile() *TranslateFile { + f, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } + f.Close() + + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s.Path = f.Name() + return s +} + +func MustOpenTranslateFile() *TranslateFile { + s := NewTranslateFile() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +func (s *TranslateFile) Close() error { + defer os.Remove(s.Path) + return s.TranslateFile.Close() +} + +func (s *TranslateFile) MustClose() { + if err := s.Close(); err != nil { + panic(err) + } +} + +// Reopen closes the store and opens a new instance of it for the same path. +func (s *TranslateFile) Reopen() error { + prev := s.TranslateFile + if err := s.TranslateFile.Close(); err != nil { + return err + } + + s.TranslateFile = pilosa.NewTranslateFile() + s.Path = prev.Path + s.PrimaryTranslateStore = prev.PrimaryTranslateStore + if err := s.Open(); err != nil { + return err + } + return nil +} + +// retryFor executes fn every 100ms until d time passes or until fn return nil. +func retryFor(d time.Duration, fn func() error) (err error) { + timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond) + defer timer.Stop() + defer ticker.Stop() + + for { + if err = fn(); err == nil { + return nil + } + + select { + case <-timer.C: + return err + case <-ticker.C: + } + } +} diff --git a/uri.go b/uri.go index 5d823d8b6..8f9df2b72 100644 --- a/uri.go +++ b/uri.go @@ -21,7 +21,6 @@ import ( "strconv" "strings" - "github.com/pilosa/pilosa/internal" "github.com/pkg/errors" ) @@ -43,17 +42,17 @@ var addressRegexp = regexp.MustCompile(`^(([+a-z]+):\/\/)?([0-9a-z.-]+|\[[:0-9a- // localhost // :10101 type URI struct { - scheme string `json:"scheme"` - host string `json:"host"` - port uint16 `json:"port"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` } -// DefaultURI creates and returns the default URI. -func DefaultURI() *URI { +// defaultURI creates and returns the default URI. +func defaultURI() *URI { return &URI{ - scheme: "http", - host: "localhost", - port: 10101, + Scheme: "http", + Host: "localhost", + Port: 10101, } } @@ -69,8 +68,8 @@ func (u URIs) HostPortStrings() []string { // NewURIFromHostPort returns a URI with specified host and port. func NewURIFromHostPort(host string, port uint16) (*URI, error) { - uri := DefaultURI() - err := uri.SetHost(host) + uri := defaultURI() + err := uri.setHost(host) if err != nil { return nil, errors.Wrap(err, "setting uri host") } @@ -83,44 +82,29 @@ func NewURIFromAddress(address string) (*URI, error) { return parseAddress(address) } -// Scheme returns the scheme of this URI. -func (u *URI) Scheme() string { - return u.scheme -} - -// SetScheme sets the scheme of this URI. -func (u *URI) SetScheme(scheme string) error { +// setScheme sets the scheme of this URI. +func (u *URI) setScheme(scheme string) error { m := schemeRegexp.FindStringSubmatch(scheme) if m == nil { return errors.New("invalid scheme") } - u.scheme = scheme + u.Scheme = scheme return nil } -// Host returns the host of this URI. -func (u *URI) Host() string { - return u.host -} - -// SetHost sets the host of this URI. -func (u *URI) SetHost(host string) error { +// setHost sets the host of this URI. +func (u *URI) setHost(host string) error { m := hostRegexp.FindStringSubmatch(host) if m == nil { return errors.New("invalid host") } - u.host = host + u.Host = host return nil } -// Port returns the port of this URI. -func (u *URI) Port() uint16 { - return u.port -} - // SetPort sets the port of this URI. func (u *URI) SetPort(port uint16) { - u.port = port + u.Port = port } // HostPort returns `Host:Port` @@ -129,41 +113,33 @@ func (u *URI) HostPort() string { if u == nil { return "" } - s := fmt.Sprintf("%s:%d", u.host, u.port) + s := fmt.Sprintf("%s:%d", u.Host, u.Port) return s } -// Normalize returns the address in a form usable by a HTTP client. -func (u *URI) Normalize() string { - scheme := u.scheme +// normalize returns the address in a form usable by a HTTP client. +func (u *URI) normalize() string { + scheme := u.Scheme index := strings.Index(scheme, "+") if index >= 0 { scheme = scheme[:index] } - return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port) + return fmt.Sprintf("%s://%s:%d", scheme, u.Host, u.Port) } // String returns the address as a string. func (u URI) String() string { - return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port) -} - -// Equals returns true if the checked URI is equivalent to this URI. -func (u URI) Equals(other *URI) bool { - if other == nil { - return false - } - return u == *other + return fmt.Sprintf("%s://%s:%d", u.Scheme, u.Host, u.Port) } // Path returns URI with path func (u *URI) Path(path string) string { - return fmt.Sprintf("%s%s", u.Normalize(), path) + return fmt.Sprintf("%s%s", u.normalize(), path) } // The following methods are required to implement pflag Value interface. -// Set sets the time quantum value. +// Set sets the uri value. func (u *URI) Set(value string) error { uri, err := NewURIFromAddress(value) if err != nil { @@ -173,7 +149,7 @@ func (u *URI) Set(value string) error { return nil } -// Type returns the type of a time quantum value. +// Type returns the type of a uri. func (u URI) Type() string { return "URI" } @@ -199,41 +175,13 @@ func parseAddress(address string) (uri *URI, err error) { } } uri = &URI{ - scheme: scheme, - host: host, - port: uint16(port), + Scheme: scheme, + Host: host, + Port: uint16(port), } return uri, nil } -// Encode converts o into its internal representation. -func (u URI) Encode() *internal.URI { - return encodeURI(u) -} - -func encodeURI(u URI) *internal.URI { - return &internal.URI{ - Scheme: u.scheme, - Host: u.host, - Port: uint32(u.port), - } -} - -func DecodeURI(i *internal.URI) URI { - return decodeURI(i) -} - -func decodeURI(i *internal.URI) URI { - if i == nil { - return URI{} - } - return URI{ - scheme: i.Scheme, - host: i.Host, - port: uint16(i.Port), - } -} - // MarshalJSON marshals URI into a JSON-encoded byte slice. func (u *URI) MarshalJSON() ([]byte, error) { var output struct { @@ -241,9 +189,9 @@ func (u *URI) MarshalJSON() ([]byte, error) { Host string `json:"host,omitempty"` Port uint16 `json:"port,omitempty"` } - output.Scheme = u.scheme - output.Host = u.host - output.Port = u.port + output.Scheme = u.Scheme + output.Host = u.Host + output.Port = u.Port return json.Marshal(output) } @@ -257,8 +205,8 @@ func (u *URI) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &input); err != nil { return err } - u.scheme = input.Scheme - u.host = input.Host - u.port = input.Port + u.Scheme = input.Scheme + u.Host = input.Host + u.Port = input.Port return nil } diff --git a/uri_test.go b/uri_internal_test.go similarity index 56% rename from uri_test.go rename to uri_internal_test.go index 70ea3f273..9bc6403d7 100644 --- a/uri_test.go +++ b/uri_internal_test.go @@ -1,41 +1,23 @@ // Copyright 2017 Pilosa Corp. // -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. +// http://www.apache.org/licenses/LICENSE-2.0 // -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND -// CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -// DAMAGE. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package pilosa import "testing" func TestDefaultURI(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() compare(t, uri, "http", "localhost", 10101) } @@ -78,7 +60,7 @@ func TestNormalizedAddress(t *testing.T) { if err != nil { t.Fatalf("Can't parse address") } - if uri.Normalize() != "http://big-data.pilosa.com:6888" { + if uri.normalize() != "http://big-data.pilosa.com:6888" { t.Fatalf("Normalized address is not normal") } } @@ -94,60 +76,50 @@ func TestURIPath(t *testing.T) { } } -func TestEquals(t *testing.T) { - uri1 := DefaultURI() - if uri1.Equals(nil) { - t.Fatalf("URI should not be equal to nil") - } - if !uri1.Equals(DefaultURI()) { - t.Fatalf("URI should be equal to another URI with the same scheme, host and port") - } -} - func TestSetScheme(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := "fun" - err := uri.SetScheme(target) + err := uri.setScheme(target) if err != nil { t.Fatal(err) } - if uri.Scheme() != target { - t.Fatalf("%s != %s", uri.Scheme(), target) + if uri.Scheme != target { + t.Fatalf("%s != %s", uri.Scheme, target) } } func TestSetHost(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := "10.20.30.40" - err := uri.SetHost(target) + err := uri.setHost(target) if err != nil { t.Fatal(err) } - if uri.Host() != target { - t.Fatalf("%s != %s", uri.host, target) + if uri.Host != target { + t.Fatalf("%s != %s", uri.Host, target) } } func TestSetPort(t *testing.T) { - uri := DefaultURI() + uri := defaultURI() target := uint16(9999) uri.SetPort(target) - if uri.Port() != target { - t.Fatalf("%d != %d", uri.port, target) + if uri.Port != target { + t.Fatalf("%d != %d", uri.Port, target) } } func TestSetInvalidScheme(t *testing.T) { - uri := DefaultURI() - err := uri.SetScheme("?invalid") + uri := defaultURI() + err := uri.setScheme("?invalid") if err == nil { t.Fatalf("Should have failed") } } func TestSetInvalidHost(t *testing.T) { - uri := DefaultURI() - err := uri.SetHost("index?.pilosa.com") + uri := defaultURI() + err := uri.setHost("index?.pilosa.com") if err == nil { t.Fatalf("Should have failed") } @@ -165,14 +137,14 @@ func TestHostPort(t *testing.T) { } func compare(t *testing.T, uri *URI, scheme string, host string, port uint16) { - if uri.Scheme() != scheme { - t.Fatalf("Scheme does not match: %s != %s", uri.scheme, scheme) + if uri.Scheme != scheme { + t.Fatalf("Scheme does not match: %s != %s", uri.Scheme, scheme) } - if uri.Host() != host { - t.Fatalf("Host does not match: %s != %s", uri.host, host) + if uri.Host != host { + t.Fatalf("Host does not match: %s != %s", uri.Host, host) } - if uri.Port() != port { - t.Fatalf("Port does not match: %d != %d", uri.port, port) + if uri.Port != port { + t.Fatalf("Port does not match: %d != %d", uri.Port, port) } } diff --git a/utils_test.go b/utils_internal_test.go similarity index 67% rename from utils_test.go rename to utils_internal_test.go index 1732522d4..bf2c4f01f 100644 --- a/utils_test.go +++ b/utils_internal_test.go @@ -24,21 +24,20 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. -func NewTestCluster(n int) *Cluster { +func NewTestCluster(n int) *cluster { path, err := ioutil.TempDir("", "pilosa-cluster-") if err != nil { panic(err) } - c := NewCluster() + c := newCluster() c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = NewTopology() + c.Topology = newTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &Node{ @@ -56,16 +55,16 @@ func NewTestCluster(n int) *Cluster { // NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { - uri := DefaultURI() - uri.SetScheme(scheme) - uri.SetHost(host) + uri := defaultURI() + uri.setScheme(scheme) + uri.setHost(host) uri.SetPort(port) return *uri } func NewTestURIFromHostPort(host string, port uint16) URI { - uri := DefaultURI() - uri.SetHost(host) + uri := defaultURI() + uri.setHost(host) uri.SetPort(port) return *uri } @@ -82,7 +81,7 @@ func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } // has a Cluster. // ClusterCluster implements Broadcaster interface. type ClusterCluster struct { - Clusters []*Cluster + Clusters []*cluster common *commonClusterSettings @@ -97,42 +96,42 @@ type commonClusterSettings struct { func (t *ClusterCluster) CreateIndex(name string) error { for _, c := range t.Clusters { - if _, err := c.Holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil { + if _, err := c.holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil { return err } } return nil } -func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { +func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error { for _, c := range t.Clusters { - idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) + idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { return err } - if _, err := idx.CreateFrame(frame, opt); err != nil { + if _, err := idx.CreateField(field, opts); err != nil { return err } } return nil } -func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { +func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / SliceWidth - nodes := c0.SliceNodes(index, slice) + c0 := t.Clusters[0] // use the first node's cluster to determine shard location. + shard := colID / ShardWidth + nodes := c0.shardNodes(index, shard) for _, node := range nodes { c := t.clusterByID(node.ID) if c == nil { continue } - f := c.Holder.Frame(index, frame) + f := c.holder.Field(index, field) if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + return fmt.Errorf("index/field does not exist: %s/%s", index, field) } - _, err := f.SetBit(view, rowID, colID, x) + _, err := f.SetBit(rowID, colID, x) if err != nil { return err } @@ -141,31 +140,7 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, return nil } -func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { - // Determine which node should receive the SetFieldValue. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := columnID / SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetFieldValue(columnID, name, value) - if err != nil { - return err - } - } - - return nil -} - -func (t *ClusterCluster) clusterByID(id string) *Cluster { +func (t *ClusterCluster) clusterByID(id string) *cluster { for _, c := range t.Clusters { if c.Node.ID == id { return c @@ -174,11 +149,11 @@ func (t *ClusterCluster) clusterByID(id string) *Cluster { return nil } -// AddNode adds a node to the cluster and (potentially) starts a resize job. -func (t *ClusterCluster) AddNode(saveTopology bool) error { +// addNode adds a node to the cluster and (potentially) starts a resize job. +func (t *ClusterCluster) addNode() error { id := len(t.Clusters) - c, err := t.addCluster(id, saveTopology) + c, err := t.addCluster(id, false) if err != nil { return err } @@ -210,7 +185,7 @@ func (t *ClusterCluster) AddNode(saveTopology bool) error { // WriteTopology writes the given topology to disk. func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { + if buf, err := proto.Marshal(top.encode()); err != nil { return err } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { return err @@ -218,7 +193,7 @@ func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { return nil } -func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) { +func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) { id := fmt.Sprintf("node%d", i) uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)) @@ -246,21 +221,20 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) h.Path = path // cluster - c := NewCluster() + c := newCluster() c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = NewTopology() - c.Holder = h - c.MemberSet = NewStaticMemberSet(c.Nodes) + c.Topology = newTopology() + c.holder = h c.Node = node c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator - c.Broadcaster = t + c.broadcaster = t // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.AddNode(n) + c.addNode(n) } } @@ -297,13 +271,13 @@ func (t *ClusterCluster) SetState(state string) { // Open opens all clusters in the test cluster. func (t *ClusterCluster) Open() error { for _, c := range t.Clusters { - if err := c.Open(); err != nil { + if err := c.open(); err != nil { return err } - if err := c.Holder.Open(); err != nil { + if err := c.holder.Open(); err != nil { return err } - if err := c.SetNodeState(NodeStateReady); err != nil { + if err := c.setNodeState(nodeStateReady); err != nil { return err } } @@ -312,7 +286,7 @@ func (t *ClusterCluster) Open() error { if len(t.Clusters) == 0 { return nil } - t.Clusters[0].ListenForJoins() + t.Clusters[0].listenForJoins() return nil } @@ -320,7 +294,7 @@ func (t *ClusterCluster) Open() error { // Close closes all clusters in the test cluster. func (t *ClusterCluster) Close() error { for _, c := range t.Clusters { - err := c.Close() + err := c.close() if err != nil { return err } @@ -329,12 +303,12 @@ func (t *ClusterCluster) Close() error { } // SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *ClusterCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: +func (t *ClusterCluster) SendSync(m Message) error { + switch obj := m.(type) { + case *ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { - c.MergeClusterStatus(obj) + c.mergeClusterStatus(obj) } t.mu.RLock() if obj.State == ClusterStateNormal && t.resizing { @@ -347,30 +321,30 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { } // SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *ClusterCluster) SendAsync(pb proto.Message) error { +func (t *ClusterCluster) SendAsync(Message) error { return nil } // SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: +func (t *ClusterCluster) SendTo(to *Node, m Message) error { + switch obj := m.(type) { + case *ResizeInstruction: err := t.FollowResizeInstruction(obj) if err != nil { return err } - case *internal.ResizeInstructionComplete: + case *ResizeInstructionComplete: coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) + go coord.markResizeInstructionComplete(obj) } return nil } // FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error { // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ + complete := &ResizeInstructionComplete{ JobID: instr.JobID, Node: instr.Node, Error: "", @@ -380,27 +354,26 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) - instrNode := DecodeNode(instr.Node) + // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) + instrNode := instr.Node destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + if err := destCluster.holder.applySchema(instr.Schema); err != nil { return err } for _, src := range instr.Sources { - srcNode := DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) + srcCluster := t.clusterByID(src.Node.ID) - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + srcFragment := srcCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) + destFragment := destCluster.holder.fragment(src.Index, src.Field, src.View, src.Shard) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) - v := f.View(src.View) + f := destCluster.holder.Field(src.Index, src.Field) + v := f.view(src.View) var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + destFragment, err = v.CreateFragmentIfNotExists(src.Shard) if err != nil { return err } @@ -430,6 +403,6 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi complete.Error = err.Error() } - node := DecodeNode(instr.Coordinator) + node := instr.Coordinator return t.SendTo(node, complete) } diff --git a/view.go b/view.go index f90bf64e9..609664304 100644 --- a/view.go +++ b/view.go @@ -22,83 +22,64 @@ import ( "strings" "sync" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) // View layout modes. const ( - ViewStandard = "standard" + viewStandard = "standard" - ViewFieldPrefix = "field_" + viewBSIGroupPrefix = "bsig_" ) -// IsValidView returns true if name is valid. -func IsValidView(name string) bool { - return name == ViewStandard -} - -// View represents a container for frame data. -type View struct { +// view represents a container for field data. +type view struct { mu sync.RWMutex path string index string - frame string + field string name string cacheSize uint32 - // Fragments by slice. - cacheType string // passed in by frame - fragments map[uint64]*Fragment + // Fragments by shard. + cacheType string // passed in by field + fragments map[uint64]*fragment - // maxSlice maintains this view's max slice in order to - // prevent sending multiple `CreateSliceMessage` messages - maxSlice uint64 + // maxShard maintains this view's max shard in order to + // prevent sending multiple `CreateShardMessage` messages + maxShard uint64 - broadcaster Broadcaster - stats StatsClient - - RowAttrStore AttrStore - Logger Logger + broadcaster broadcaster + stats StatsClient + rowAttrStore AttrStore + logger Logger } -// NewView returns a new instance of View. -func NewView(path, index, frame, name string, cacheSize uint32) *View { - return &View{ +// newView returns a new instance of View. +func newView(path, index, field, name string, cacheSize uint32) *view { + return &view{ path: path, index: index, - frame: frame, + field: field, name: name, cacheSize: cacheSize, cacheType: DefaultCacheType, - fragments: make(map[uint64]*Fragment), + fragments: make(map[uint64]*fragment), broadcaster: NopBroadcaster, stats: NopStatsClient, - Logger: NopLogger, + logger: NopLogger, } } -// Name returns the name the view was initialized with. -func (v *View) Name() string { return v.name } - -// Index returns the index name the view was initialized with. -func (v *View) Index() string { return v.index } - -// Frame returns the frame name the view was initialized with. -func (v *View) Frame() string { return v.frame } - -// Path returns the path the view was initialized with. -func (v *View) Path() string { return v.path } - -// Open opens and initializes the view. -func (v *View) Open() error { +// open opens and initializes the view. +func (v *view) open() error { // Never keep a cache for field views. - if strings.HasPrefix(v.name, ViewFieldPrefix) { + if strings.HasPrefix(v.name, viewBSIGroupPrefix) { v.cacheType = CacheTypeNone } @@ -116,7 +97,7 @@ func (v *View) Open() error { return nil }(); err != nil { - v.Close() + v.close() return err } @@ -124,7 +105,7 @@ func (v *View) Open() error { } // openFragments opens and initializes the fragments inside the view. -func (v *View) openFragments() error { +func (v *view) openFragments() error { file, err := os.Open(filepath.Join(v.path, "fragments")) if os.IsNotExist(err) { return nil @@ -144,24 +125,24 @@ func (v *View) openFragments() error { } // Parse filename into integer. - slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { continue } - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } - frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.Slice()] = frag + frag.RowAttrStore = v.rowAttrStore + v.fragments[frag.shard] = frag } return nil } -// Close closes the view and its fragments. -func (v *View) Close() error { +// close closes the view and its fragments. +func (v *view) close() error { v.mu.Lock() defer v.mu.Unlock() @@ -171,117 +152,117 @@ func (v *View) Close() error { return errors.Wrap(err, "closing fragment") } } - v.fragments = make(map[uint64]*Fragment) + v.fragments = make(map[uint64]*fragment) return nil } -// MaxSlice returns the max slice in the view. -func (v *View) MaxSlice() uint64 { +// calculateMaxShard returns the max shard in the view. +func (v *view) calculateMaxShard() uint64 { v.mu.RLock() defer v.mu.RUnlock() var max uint64 - for slice := range v.fragments { - if slice > max { - max = slice + for shard := range v.fragments { + if shard > max { + max = shard } } return max } -// FragmentPath returns the path to a fragment in the view. -func (v *View) FragmentPath(slice uint64) string { - return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) +// fragmentPath returns the path to a fragment in the view. +func (v *view) fragmentPath(shard uint64) string { + return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10)) } -// Fragment returns a fragment in the view by slice. -func (v *View) Fragment(slice uint64) *Fragment { +// Fragment returns a fragment in the view by shard. +func (v *view) Fragment(shard uint64) *fragment { v.mu.RLock() defer v.mu.RUnlock() - return v.fragment(slice) + return v.fragment(shard) } -func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } +func (v *view) fragment(shard uint64) *fragment { return v.fragments[shard] } -// Fragments returns a list of all fragments in the view. -func (v *View) Fragments() []*Fragment { +// allFragments returns a list of all fragments in the view. +func (v *view) allFragments() []*fragment { v.mu.Lock() defer v.mu.Unlock() - other := make([]*Fragment, 0, len(v.fragments)) + other := make([]*fragment, 0, len(v.fragments)) for _, fragment := range v.fragments { other = append(other, fragment) } return other } -// RecalculateCaches recalculates the cache on every fragment in the view. -func (v *View) RecalculateCaches() { - for _, fragment := range v.Fragments() { +// recalculateCaches recalculates the cache on every fragment in the view. +func (v *view) recalculateCaches() { + for _, fragment := range v.allFragments() { fragment.RecalculateCache() } } -// CreateFragmentIfNotExists returns a fragment in the view by slice. -func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { +// CreateFragmentIfNotExists returns a fragment in the view by shard. +func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { v.mu.Lock() defer v.mu.Unlock() - return v.createFragmentIfNotExists(slice) + return v.createFragmentIfNotExists(shard) } -func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { +func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { // Find fragment in cache first. - if frag := v.fragments[slice]; frag != nil { + if frag := v.fragments[shard]; frag != nil { return frag, nil } // Initialize and open fragment. - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } - frag.RowAttrStore = v.RowAttrStore + frag.RowAttrStore = v.rowAttrStore - // Broadcast a message that a new max slice was just created. - if slice > v.maxSlice { - v.maxSlice = slice + // Broadcast a message that a new max shard was just created. + if shard > v.maxShard { + v.maxShard = shard - // Send the create slice message to all nodes. - err := v.broadcaster.SendAsync( - &internal.CreateSliceMessage{ + // Send the create shard message to all nodes. + err := v.broadcaster.SendSync( + &CreateShardMessage{ Index: v.index, - Slice: slice, + Shard: shard, }) if err != nil { - return nil, errors.Wrap(err, "sending message") + return nil, errors.Wrap(err, "sending createshard message") } } // Save to lookup. - v.fragments[slice] = frag + v.fragments[shard] = frag return frag, nil } -func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.index, v.frame, v.name, slice) +func (v *view) newFragment(path string, shard uint64) *fragment { + frag := newFragment(path, v.index, v.field, v.name, shard) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.Logger = v.Logger - frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) + frag.Logger = v.logger + frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) return frag } -// DeleteFragment removes the fragment from the view. -func (v *View) DeleteFragment(slice uint64) error { +// deleteFragment removes the fragment from the view. +func (v *view) deleteFragment(shard uint64) error { - fragment := v.fragments[slice] + fragment := v.fragments[shard] if fragment == nil { return ErrFragmentNotFound } - v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + v.logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -289,64 +270,78 @@ func (v *View) DeleteFragment(slice uint64) error { } // Delete fragment file. - if err := os.Remove(fragment.Path()); err != nil { + if err := os.Remove(fragment.path); err != nil { return errors.Wrap(err, "deleting fragment file") } // Delete fragment cache file. - if err := os.Remove(fragment.CachePath()); err != nil { - v.Logger.Printf("no cache file to delete for slice %d", slice) + if err := os.Remove(fragment.cachePath()); err != nil { + v.logger.Printf("no cache file to delete for shard %d", shard) } - delete(v.fragments, slice) + delete(v.fragments, shard) return nil } -// SetBit sets a bit within the view. -func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) +// row returns a row for a shard of the view. +func (v *view) row(rowID uint64) *Row { + row := NewRow() + for _, frag := range v.allFragments() { + fr := frag.row(rowID) + if fr == nil { + continue + } + row.Merge(fr) + } + return row + +} + +// setBit sets a bit within the view. +func (v *view) setBit(rowID, columnID uint64) (changed bool, err error) { + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - return frag.SetBit(rowID, columnID) + return frag.setBit(rowID, columnID) } -// ClearBit clears a bit within the view. -func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return changed, err +// clearBit clears a bit within the view. +func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) { + shard := columnID / ShardWidth + frag, found := v.fragments[shard] + if !found { + return false, nil } - return frag.ClearBit(rowID, columnID) + return frag.clearBit(rowID, columnID) } -// FieldValue uses a column of bits to read a multi-bit value. -func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) +// value uses a column of bits to read a multi-bit value. +func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return value, exists, err } - return frag.FieldValue(columnID, bitDepth) + return frag.value(columnID, bitDepth) } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) +// setValue uses a column of bits to set a multi-bit value. +func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } - return frag.SetFieldValue(columnID, bitDepth, value) + return frag.setValue(columnID, bitDepth, value) } -// FieldSum returns the sum & count of a field. -func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - for _, f := range v.Fragments() { - fsum, fcount, err := f.FieldSum(filter, bitDepth) +// sum returns the sum & count of a field. +func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { + for _, f := range v.allFragments() { + fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { return sum, count, err } @@ -356,11 +351,11 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro return sum, count, nil } -// FieldMin returns the min and count of a field. -func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { +// min returns the min and count of a field. +func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool - for _, f := range v.Fragments() { - fmin, fcount, err := f.FieldMin(filter, bitDepth) + for _, f := range v.allFragments() { + fmin, fcount, err := f.min(filter, bitDepth) if err != nil { return min, count, err } @@ -384,10 +379,10 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro return min, count, nil } -// FieldMax returns the max and count of a field. -func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { - for _, f := range v.Fragments() { - fmax, fcount, err := f.FieldMax(filter, bitDepth) +// max returns the max and count of a field. +func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.allFragments() { + fmax, fcount, err := f.max(filter, bitDepth) if err != nil { return max, count, err } @@ -399,11 +394,11 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro return max, count, nil } -// FieldRange returns rows with a field value encoding matching the predicate. -func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns rows with a field value encoding matching the predicate. +func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.FieldRange(op, bitDepth, predicate) + for _, frag := range v.allFragments() { + other, err := frag.rangeOp(op, bitDepth, predicate) if err != nil { return nil, err } @@ -412,12 +407,12 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, return r, nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any +// rangeBetween returns bitmaps with a field value encoding matching any // value between predicateMin and predicateMax. -func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (v *view) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) + for _, frag := range v.allFragments() { + other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } diff --git a/view_internal_test.go b/view_internal_test.go new file mode 100644 index 000000000..6fe0b0e33 --- /dev/null +++ b/view_internal_test.go @@ -0,0 +1,68 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "testing" +) + +// mustOpenView returns a new instance of View with a temporary path. +func mustOpenView(index, field, name string) *view { + path, err := ioutil.TempDir("", "pilosa-view-") + if err != nil { + panic(err) + } + + v := newView(path, index, field, name, DefaultCacheSize) + if err := v.open(); err != nil { + panic(err) + } + v.rowAttrStore = newMemAttrStore() + return v +} + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := mustOpenView("i", "f", "v") + defer v.close() + + shard := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(shard) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.deleteFragment(shard) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(shard) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same shard, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(shard) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} diff --git a/view_test.go b/view_test.go deleted file mode 100644 index 21834113d..000000000 --- a/view_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -// View is a test wrapper for pilosa.View. -type View struct { - *pilosa.View - RowAttrStore pilosa.AttrStore -} - -// NewView returns a new instance of View with a temporary path. -func NewView(index, frame, name string) *View { - path, err := ioutil.TempDir("", "pilosa-view-") - if err != nil { - panic(err) - } - - v := &View{ - View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), - RowAttrStore: test.MustOpenAttrStore(), - } - v.View.RowAttrStore = v.RowAttrStore - return v -} - -// MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(index, frame, name string) *View { - v := NewView(index, frame, name) - if err := v.Open(); err != nil { - panic(err) - } - return v -} - -// Close closes the view and removes all underlying data. -func (v *View) Close() error { - defer os.Remove(v.Path()) - defer v.RowAttrStore.Close() - return v.View.Close() -} - -// Reopen closes the view and reopens it as a new instance. -func (v *View) Reopen() error { - path := v.Path() - if err := v.View.Close(); err != nil { - return err - } - - v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore - return v.Open() -} - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearColumns clears columns on a row. Panic on error. -func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// Ensure view can open and retrieve a fragment. -func TestView_DeleteFragment(t *testing.T) { - v := MustOpenView("i", "f", "v") - defer v.Close() - - slice := uint64(9) - - // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == nil { - t.Fatal("expected fragment") - } - - err = v.DeleteFragment(slice) - if err != nil { - t.Fatal(err) - } - - if v.Fragment(slice) != nil { - t.Fatal("fragment still exists in view") - } - - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == fragment2 { - t.Fatal("failed to create new fragment") - } -} diff --git a/webui/assets/chevron-down.png b/webui/assets/chevron-down.png deleted file mode 100644 index 3312489a2..000000000 Binary files a/webui/assets/chevron-down.png and /dev/null differ diff --git a/webui/assets/main.js b/webui/assets/main.js deleted file mode 100644 index 265f51997..000000000 --- a/webui/assets/main.js +++ /dev/null @@ -1,594 +0,0 @@ -class REPL { - constructor(input, output, button, completer) { - this.input = input - this.output = output - this.button = button - this.completer = completer - this.history = [] - this.history_index = 0 - this.history_buffer = '' - this.result_number = 0 - } - bind_events() { - var repl = this - var keys = { - TAB: 9, - ENTER: 13, - UP_ARROW: 38, - DOWN_ARROW: 40 - } - - this.input.addEventListener("keydown", function(e) { - if (e.keyCode == keys.UP_ARROW) { - e.preventDefault() - if (repl.input.value.substring(0, repl.input.selectionStart).indexOf('\n') == '-1') { - if (repl.history_index == 0) { - return - } else { - if (repl.history_index == repl.history.length) { - repl.history_buffer = repl.input.value - } - repl.history_index-- - repl.input.value = repl.history[repl.history_index] - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.DOWN_ARROW) { - e.preventDefault() - if (repl.input.value.substring(repl.input.selectionEnd, repl.input.length).indexOf('\n') == '-1') { - if (repl.history_index == repl.history.length) { - return - } else { - repl.history_index++ - if (repl.history_index == repl.history.length) { - repl.input.value = repl.history_buffer - } else { - repl.input.value = repl.history[repl.history_index] - } - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.ENTER && !e.shiftKey) { - e.preventDefault() - repl.submit(); - } - if (e.keyCode == keys.TAB) { - e.preventDefault() - repl.completer.complete() - } - }) - repl.button.onclick = function() { - repl.submit(); - }; - } - - submit() { - this.history_buffer = '' - this.history_index = this.history.length - this.history[this.history_index] = this.input.value - this.history_index++ - this.process_query(this.input.value) - this.input.value = "" - } - - process_query(query) { - var xhr = new XMLHttpRequest(); - var url, data, request, command_name; - var e = document.getElementById("index-dropdown"); - var indexname = e.options[e.selectedIndex].text; - var repl = this; - if (query.startsWith(":")) { - var parsed_query = parse_query(query, indexname); - if (Object.keys(parsed_query).length === 0) { - repl.create_single_output({ - "input": query, - "output": "invalid query", - "status": 400, - "indexname": indexname, - }); - return; - } else { - // set selectedIndex from dropdown list - if (parsed_query.command === "use") { - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === parsed_query.command_name) { - e.selectedIndex = i; - break; - } - } - return; - } - url = parsed_query.url; - data = parsed_query.data; - request = parsed_query.request; - command_name = parsed_query.command_name; - } - } - else { - url = '/index/' + indexname + '/query' - request = "POST" - data = query - } - xhr.open(request, url); - xhr.setRequestHeader('Content-Type', 'application/text'); - - var start_time = new Date().getTime(); - xhr.onload = function () { - var end_time = new Date().getTime(); - repl.result_number++ - repl.create_single_output({ - "input": query, - "output": xhr.responseText, - "status": xhr.status, - "indexname": indexname, - "querytime_ms": end_time - start_time, - }); - }; - - xhr.send(data); - // Remove index from dropdown with delete index command - if (request === 'DELETE' && url === '/index/' + command_name){ - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === command_name) { - e.remove(i); - break; - } - } - } - } - - create_single_output(res) { - var node = document.createElement("div"); - node.classList.add('output'); - var output_string = res['output'] - var result_class = "result-output" - var getting_started_errors = [ - 'index not found', - 'frame not found', - ] - var output_json; - if (isJSON(output_string)) { - output_json = JSON.parse(output_string) - } - // handle output formatting - if (res["status"] != 200) { - result_class = "result-error"; - if (output_json) { - if ("error" in output_json) { - if (getting_started_errors.indexOf(output_json['error']) >= 0) { - output_string += `
-
- Just getting started? Try this:
- :create index test
- :use test
- :create frame foo
- SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit - ` - } - } - } - } - - - var markup =` -
-
-
-
-
Input
-        - Source: ${res.indexname} -
-
- ${res.input} -
-
-
-
-
output
-        - ${res.querytime_ms} ms -
-
- ${output_string} -
-
Expand
 - -
-
-
-
- - Export raw .csv -
-
-
- ` - node.innerHTML = markup; - this.output.insertBefore(node, this.output.firstChild); - - // Expand when overflow - var element = this.output.firstChild.getElementsByClassName(result_class)[0]; - var expand = this.output.firstChild.getElementsByClassName("expand")[0]; - if (element.clientHeight < element.scrollHeight) { - expand.style.display = 'block'; - } else { - expand.style.display = 'none'; - } - expand.onclick = function () { - element.style.height = element.scrollHeight + "px"; - expand.style.display = 'none'; - return false; - }; - } - - populate_index_dropdown() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/schema') - var select = document.getElementById('index-dropdown') - - xhr.onload = function() { - var schema = JSON.parse(xhr.responseText) - for(var i=0; i 0) { - select.value = 1; - } - } - xhr.send(null) - } - -} - -function populate_version() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/version') - var node = document.getElementById('server-version') - - xhr.onload = function() { - var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] - var doc_link = document.getElementById('nav-documentation') - doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') - } - node.innerHTML = "Pilosa v" + version - } - xhr.send(null) -} - -function handle_nav_click(e) { - // e.id = "nav-xxx" - name = e.id.substring(4) - set_active_pane_by_name(name) - window.location.hash = name -} - -function set_active_pane_by_name(name) { - // toggle the nav buttons - document.getElementsByClassName("nav-active")[0].classList.remove("nav-active") - document.getElementById("nav-" + name).classList.add("nav-active") - - // toggle the main interface content divs - document.getElementsByClassName("interface-active")[0].classList.remove("interface-active") - document.getElementById('interface-' + name).classList.add("interface-active") - - // hack hack - switch(name) { - case "cluster": - update_cluster_status() - break - case "documentation": - open_external_docs() - break - } -} - - -function update_cluster_status() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/status') - xhr.onload = function() { - var status = JSON.parse(xhr.responseText) - render_status(status) - } - xhr.send(null) - - var xhrSchema = new XMLHttpRequest(); - xhrSchema.open('GET', '/schema') - xhrSchema.onload = function() { - var schema = JSON.parse(xhrSchema.responseText) - render_schema(schema) - } - xhrSchema.send(null) -} - -function render_status(status) { - // render node table - var nodes_div = document.getElementById("status-nodes") - while (nodes_div.firstChild) { - nodes_div.removeChild(nodes_div.firstChild); - } - - var nodes = status["nodes"] - table = document.createElement("table") - tbody = document.createElement("tbody") - table.appendChild(tbody) - var caption = document.createElement("caption") - caption.innerHTML = "(" + nodes.length + ")" - table.appendChild(caption) - - var header = document.createElement('tr') - markup = `Host - ID - Coordinator` - header.innerHTML = markup - tbody.appendChild(header) - for(var n=0; n${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} - ${nodes[n]["id"]} - ${nodes[n]["isCoordinator"]}` - row.innerHTML = markup - tbody.appendChild(row) - } - nodes_div.appendChild(table) -} - -function render_schema(schema) { - // render index tables - var indexes_div = document.getElementById("status-indexes") - while (indexes_div.firstChild) { - indexes_div.removeChild(indexes_div.firstChild); - } - - var indexes = schema["indexes"] // TODO currently comes from only node 0 - for(var n=0; nName - Cache Type - Cache Size` - header.innerHTML = markup - tbody.appendChild(header) - - var frames = indexes[n]["frames"] - if(frames) { - for(var m=0; m${frames[m]["name"]} - ${frames[m]["options"]["cacheType"]} - ${frames[m]["options"]["cacheSize"]}` - tbody.appendChild(row) - } - } - indexes_div.appendChild(table) - } - - // render slice tables - // TODO enable when Slices element is present in status response - /* - var slices_div = document.getElementById("status-slices") - data = "" - for(var n=0; n" - } - } - slices_div.innerHTML = data - */ - -} - -function open_external_docs() { - window.open("https://www.pilosa.com/docs"); -} - -function check_anchor_uri() { - var pane_names = {"console": 0, "cluster": 0, "documentation": 0} - var anchor = window.location.hash.substr(1); - if(anchor in pane_names) { - set_active_pane_by_name(anchor) - } -} - -Date.prototype.today = function () { - return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate(); -} - -Date.prototype.timeNow = function () { - return ((this.getHours() < 10)?"0":"") + this.getHours() +":"+ ((this.getMinutes() < 10)?"0":"") + this.getMinutes() +":"+ ((this.getSeconds() < 10)?"0":"") + this.getSeconds(); -} - -populate_version() - - -class Autocompleter { - constructor(input, output) { - this.input = input - this.output = output - this.keyword_map = this.static_keywords - this.init_dynamic_keywords() - } - - get static_keywords() { - return { - // keyword: length of substring that comes after cursor - "SetBit()": 1, - "ClearBit()": 1, - "SetRowAttrs()": 1, - "SetColumnAttrs()": 1, - "Bitmap()": 1, - "Union()": 1, - "Intersect()": 1, - "Difference()": 1, - "Count()": 1, - "Range()": 1, - "TopN()": 1, - "frame=": 0, - } - } - - complete() { - var completer = this - // extract word fragment ending at cursor. a word fragment: - // - starts with last nonalpha character before cursor (or beginning of string) - // - ends at cursor - var word_start = completer.input.selectionEnd-1 - while(word_start>0) { - var c = completer.input.value.charCodeAt(word_start) - if(!((c>64 && c<91) || (c>96 && c<123))) { - word_start++ - break - } - word_start-- - } - var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd) - - // check for keyword match and insert if exactly one match - var matches = [] - for(var keyword in this.keyword_map) { - if(keyword.startsWith(input_word)){ - matches.push(keyword) - } - } - if(matches.length > 1) { - // completer.output.innerHTML = whatever - } - - if(matches.length == 1) { - // completer.output.innerHTML = "" - var cursor_pos = completer.input.selectionEnd - var completion = matches[0].substring(input_word.length) - var before = completer.input.value.substring(0, cursor_pos) - var after = completer.input.value.substring(cursor_pos) - completer.input.value = before + completion + after - var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]] - completer.input.setSelectionRange(new_pos, new_pos) - } - } - - init_dynamic_keywords() { - // hit /schema, parse indexes, frames, add to list - } - - add_keyword() { - // call when index or frame created in webui - } - - remove_keyword() { - // call when index or frame deleted in webui - // issue: if e.g. multiple indexes have same frame, removing one removes all. - // solution: maintain count. requires more elaborate representation of keywords. - } -} - -var input = document.getElementById('query') -var output = document.getElementById('outputs') -var button = document.getElementById('query-btn') -var autocomplete_output = document.getElementById('autocomplete-container') - -autocompleter = new Autocompleter(input, autocomplete_output) -repl = new REPL(input, output, button, autocompleter) -repl.populate_index_dropdown() -repl.bind_events() - -input.focus() - -check_anchor_uri() - -function isJSON(str) { - try { - JSON.parse(str) - } catch (e) { - return false - } - return true -} - -function parse_query(query, indexname) { - var keys = query.replace(/\s+/g, " ").split(" "); - var command = keys[0]; - var command_type = keys[1]; - var command_name = keys[2]; - var option_str = keys.slice(3, keys.length) - var options = parse_options(option_str); - if (command !== ":use") { - if (!command_name){ - return {} - } - } - - var parsed_query = {}; - parsed_query["command"] = command.substr(1, command.length); - parsed_query["command_name"] = command_name; - switch (command) { - case ":create": - parsed_query["request"] = "POST"; - if(Object.keys(options).length === 0) { - parsed_query["data"] = ""; - } else { - var opts = {"options":{}}; - for (var o in options) { - opts.options[o] = options[o] - } - parsed_query["data"] = JSON.stringify(opts); - } - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - break - } - break; - case ":delete": - parsed_query["request"] = "DELETE"; - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - parsed_query["data"] = ""; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - parsed_query["data"] = ""; - break; - } - break; - case ":use": - parsed_query["command_name"] = keys[1]; - break; - default: - return {} - } - return parsed_query; -} - -function parse_options(option_str) { - var int_keys = ["cacheSize"]; - var options = {}; - for (var i = 0; i < option_str.length; i++) { - var parts = option_str[i].split('='); - if (int_keys.indexOf(parts[0]) !== -1 ){ - options[parts[0]] = Number(parts[1]) - } else { - options[parts[0]] = parts[1] - } - } - return options; -} diff --git a/webui/assets/nav-cluster-active.svg b/webui/assets/nav-cluster-active.svg deleted file mode 100644 index 871d6f2af..000000000 --- a/webui/assets/nav-cluster-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 diff --git a/webui/assets/nav-cluster.svg b/webui/assets/nav-cluster.svg deleted file mode 100644 index baf6310ea..000000000 --- a/webui/assets/nav-cluster.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 \ No newline at end of file diff --git a/webui/assets/nav-console-active.svg b/webui/assets/nav-console-active.svg deleted file mode 100644 index 2263c2a69..000000000 --- a/webui/assets/nav-console-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console \ No newline at end of file diff --git a/webui/assets/nav-console.svg b/webui/assets/nav-console.svg deleted file mode 100644 index e9d2a9569..000000000 --- a/webui/assets/nav-console.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console diff --git a/webui/assets/nav-documentation-active.svg b/webui/assets/nav-documentation-active.svg deleted file mode 100644 index b6e994980..000000000 --- a/webui/assets/nav-documentation-active.svg +++ /dev/null @@ -1 +0,0 @@ -documentation diff --git a/webui/assets/nav-documentation.svg b/webui/assets/nav-documentation.svg deleted file mode 100644 index 12cd2108c..000000000 --- a/webui/assets/nav-documentation.svg +++ /dev/null @@ -1 +0,0 @@ -documentation \ No newline at end of file diff --git a/webui/assets/nav_item1.svg b/webui/assets/nav_item1.svg deleted file mode 100644 index d6d7f779e..000000000 --- a/webui/assets/nav_item1.svg +++ /dev/null @@ -1 +0,0 @@ -nav_item1 \ No newline at end of file diff --git a/webui/assets/style.css b/webui/assets/style.css deleted file mode 100644 index 2cf65d086..000000000 --- a/webui/assets/style.css +++ /dev/null @@ -1,336 +0,0 @@ -*{ - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} -body{ - font-family: sans-serif; - background-color: #fbfcfd; - margin: 0; - color: #102445; -} -h2{ - margin-bottom: 30px; -} - -h5{ - text-transform: uppercase; - letter-spacing: 2px; - line-height: 1.21; - margin: 0; -} -a{ - line-height: 1.38; - letter-spacing: 0.2px; - text-decoration: none; - color: #102445; -} - - -a:hover{ - color: #1db598; -} - -textarea{ - width: 100%; - margin-bottom: 10px; - border-radius: 2px; - background-color: #fbfcfd; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - outline: none; - padding: 30px; -} - - -select{ - /*-webkit-appearance: none; - -moz-appearance: none; - appearance: none; - background: url("img/chevron-down.png") no-repeat calc(100% - 10px) !important;*/ - border-radius: 3px; - background-color: #fbfcfd; - width: 187px; - height: 50px; - border: solid 1.5px #e4eff4; - font-size: 18px; - font-weight: bold; - line-height: 1.39; - letter-spacing: 0.2px; - color: #102445; - padding: 10.5px; - -} - -button{ - width: 165px; - height: 50px; - border-radius: 3px; - background-color: #1db598; - outline: none; - border: none; - font-size: 16px; - color: white; -} - -em{ - font-style: normal; - opacity: 0.5; - font-size: 14px; - font-weight: 500; - letter-spacing: 0.2px; - color: #102445; -} - -.header{ - height: 92px; - display: flex; - align-items: center; - justify-content: space-between; - width: 90%; - margin: auto; -} - -.container{ - display: flex; - height:100%; - min-height: 100vh; -} -.nav{ - color: white; - display: flex; - flex-direction: column; - width: 150px; - background: #3c5f8d; -} - -.nav-item{ - height:150px; - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; - border-bottom: 3px solid #2a4871; - cursor: pointer; -} - -.nav-active{ - background: #f2f7f9; - font-weight: bold; - color: #1db598; -} - -.nav-item > .nav-image { - display: flex; -} - -.nav-item > .nav-image-active { - display: none; -} - -.nav-active > .nav-image { - display: none; -} - -.nav-active > .nav-image-active { - display: flex; -} - - -.interface{ - display: none; - flex: 1; - flex-direction: column; - align-items: center; - background: #f2f7f9; -} - -.interface-active{ - display: flex; -} - -.query{ - margin-bottom: 30px; -} -.query, -.output-container, -.status-container{ - width: 75%; -} - -.output{ - margin-bottom: 30px; -} - -.input-controls{ - display: flex; - justify-content: flex-end; -} - -.tabs{ - display: flex; - background: #eaf2f6; -} -.active-tab{ - background: white; - font-weight: bold; - color: #1db598; - -} - -.tab{ - height:60px; - width: 100px; - border-top-right-radius: 5px; - display: flex; - align-items: center; - justify-content: center; - visibility: visible; - cursor: pointer; - -} - -.pane{ - background: white; - padding: 30px; - display: none; -} - -.active{ - display: block; -} - -.result-io-header{ - display: flex; - align-items: center; - margin-bottom: 15px; -} - -.result-input, -.result-output, -.result-error{ - height: 60px; - border-radius: 2px; - background-color: #fafafa; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - color: #102445; - padding: 15px; - margin-bottom: 15px; - word-break: break-all; - overflow-wrap: break-word; - overflow:hidden; -} - - -.result-output{ - background-color: #edf9f7; - border-left: solid 4px #1db598; -} - -.result-error{ - background-color: #fbf1f0; - border-left: solid 4px #fa3035; - color: #fa3035; -} - -.raw{ - height: 253px; - display: flex; - align-items: center; - justify-content: center; -} - - -.result-table > table { - border-left: solid 4px #1db598; -} - -table{ - border: solid 0.5px #e0e0e0; - width: 100%; - margin-bottom: 30px; - /*color:#3c5f8d;*/ -} -caption{ - text-align:left; - font-size: 16px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - text-align: left; -} -th{ - font-size: 14px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - color: #102445; - text-transform: uppercase; - text-align: left; - padding: 21px 30px; - background-color: white; -} -tr{ - border: solid 0.5px #e0e0e0; - background-color: white; -} -tr:nth-child(even) { - background-color: #f2f7f9; -} -td{ - padding: 21px 30px; -} - -.expand { - text-align: center; -} - -.query h2 { - display: inline-block; -} - -.query-tooltip { - position: relative; - display: inline; - color: #000; - margin-left: 5px; -} - -.query-tooltip:hover { - color: #000; -} - -.query-tooltip-content { - background-color: rgb(250, 250, 250); - border: solid 1.5px #e4eff4; - color: #102445; - border-radius: 2px; - padding: 15px; - margin-bottom: 15px; - - position: absolute; - left: 80px; - top: -30px; - z-index: 1; -} - -.query-tooltip-container { - position: relative; - visibility: hidden; -} - -.query-tooltip:hover+.query-tooltip-container{ - visibility: visible; -} - -.code{ - font-family: monospace; -} - diff --git a/webui/index.html b/webui/index.html deleted file mode 100644 index f6bc369de..000000000 --- a/webui/index.html +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - - - Pilosa WebUI - - - -
- -
-
-
-
- - - -
-
- -
-

Query

- ? -
-
-
PQL
-
- SetBit(frame=foo, row=0, col=0)
- ClearBit(frame=foo, row=0, col=0)
- SetRowAttrs(frame=foo, row=0, color="blue")
- SetColumnAttrs(frame=foo, col=0, shape="circle")
- SetFieldValue(frame=foo, col=0, age=30)
- Bitmap(frame=foo, row=0)
- Range(frame=foo, row=0, start="2010-01", end="2017-03")
- Count(<BITMAP_CALL>)
- TopN([BITMAP_CALL], frame=foo, n=20)
- Union([BITMAP_CALL, ...])
- Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
- Difference(<BITMAP_CALL>, <BITMAP_CALL>)
- Xor([BITMAP_CALL, ...])
- Min([BITMAP_CALL], frame=foo, field=age)
- Max([BITMAP_CALL], frame=foo, field=age)
- Sum([BITMAP_CALL], frame=foo, field=age) -
-
-
Special commands
-
- :create index test
- :use test
- :create frame foo
- :delete index test
- :delete frame foo -
-
- <tab>: autocomplete
- <up>/<down>: history
-
-
- -
-
- -     - -
-
-
- -
-

Output

-
- -
-
- -
- -
-
-

Nodes

-
-
-
-
-

Indexes

-
-
-
-
- -
- -
- -
- docs! -
- -
- - -