scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into seebs/bench

Browse source
This commit is contained in:
seebs 2019-01-17 13:48:19 -06:00 • committed by GitHub
commit 7857730b7f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
59 changed files with 5527 additions and 2485 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
.circleci/config.yml
View file

@ -7,7 +7,7 @@ fast-checkout: &fast-checkout
attach_workspace:
at: .
jobs:
build:
setup:
<<: *defaults
steps:
- checkout
@ -30,6 +30,14 @@ jobs:
- run: gometalinter --install
- run: go get github.com/remyoudompheng/go-misc/deadcode
- run: make gometalinter
test-build-arm:
<<: *defaults
steps:
- *fast-checkout
- run: make build GOOS=linux GOARCH=arm GOARM=5
- run: make build GOOS=linux GOARCH=arm GOARM=6
- run: make build GOOS=linux GOARCH=arm GOARM=7
- run: make build GOOS=linux GOARCH=arm64
test-golang-1.11: &base-test
<<: *defaults
steps:
@ -42,7 +50,7 @@ jobs:
- *fast-checkout
- run: sudo apt-get install lsof
- run:
command: make test TESTFLAGS="-race -timeout=30m"
command: make test TESTFLAGS="-race -v -timeout=30m"
no_output_timeout: 30m
test-golang-1.11-386:
<<: *base-test
@ -100,25 +108,28 @@ workflows:
version: 2
test:
jobs:
- build
- setup
- linter:
requires:
- build
- setup
- test-build-arm:
requires:
- setup
- test-golang-1.11:
requires:
- build
- setup
- test-golang-1.11-race:
requires:
- build
- setup
- test-golang-1.11-386:
requires:
- build
- setup
- test-golang-1.10:
requires:
- build
- setup
- cluster-tests:
requires:
- build
- setup
- prerelease:
requires:
- linter

4
.github/ISSUE_TEMPLATE.md vendored
View file

@ -1,8 +1,8 @@
For bugs, please provide the following:
### Expected behavior
### What's going wrong?
### Actual behavior
### What was expected?
### Steps to reproduce the behavior

104
CHANGELOG.md
View file

@ -5,7 +5,96 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## [v1.1.0] - 2018-08-21
## [1.2.0] - 2018-12-20
This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions.
### Added
- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773))
- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774))
- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771))
- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753))
- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751))
- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756))
- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684))
- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733))
- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743))
- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717))
- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713))
- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707))
- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647))
- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699))
- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663))
- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683))
- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666))
- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671))
- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673))
- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645))
- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658))
- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653))
- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646))
- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738))
- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635))
- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631))
- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625))
- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628))
### Changed
- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780))
- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755))
- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748))
- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740))
- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725))
- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724))
- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677))
- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664))
### Performance
- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619))
- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741))
### Fixed
- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795))
- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787))
- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790))
- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785))
- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782))
- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777))
- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769))
- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764))
- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750))
- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749))
- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732))
- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744))
- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742))
- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737))
- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729))
- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719))
- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727))
- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712))
- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711))
- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674))
- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675))
- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696))
- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686))
- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679))
- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676))
- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665))
- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662))
- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655))
- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656))
- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651))
- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612))
- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621))
- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620))
- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614))
## [1.1.0] - 2018-08-21
This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions.
@ -29,7 +118,7 @@ This version contains 32 contributions from 5 contributors. There are 89 files c
- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552))
- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558))
## [v1.0.2] - 2018-08-01
## [1.0.2] - 2018-08-01
This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions.
@ -44,7 +133,7 @@ This version contains 11 contributions from 3 contributors. There are 30 files c
- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502))
- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555))
## [v1.0.1] - 2018-07-11
## [1.0.1] - 2018-07-11
This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions.
@ -56,7 +145,7 @@ This version contains 12 contributions from 4 contributors. There are 11 files c
- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483))
- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493))
## [v1.0.0] - 2018-07-09
## [1.0.0] - 2018-07-09
This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions.
@ -104,7 +193,7 @@ This version contains 218 contributions from 7 contributors. There are 184 files
- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415))
- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350))
## [v0.10.0] - 2018-05-15
## [0.10.0] - 2018-05-15
This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions.
@ -135,7 +224,7 @@ This version contains 93 contributions from 8 contributors. There are 93 files c
- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234))
- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253))
## [v0.9.0] - 2018-05-04
## [0.9.0] - 2018-05-04
This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions.
@ -489,7 +578,7 @@ This version contains 53 contributions from 13 contributors (including 4 volunte
- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577))
- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562))
[Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD
[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD
[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
@ -499,3 +588,4 @@ This version contains 53 contributions from 13 contributors (including 4 volunte
[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10
[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0
[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1
[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2

2
Dockerfile
View file

@ -1,4 +1,4 @@
FROM golang:1.10.3 as builder
FROM golang:1.11.4 as builder
COPY . /go/src/github.com/pilosa/pilosa/

16
Makefile
View file

@ -23,7 +23,7 @@ clean:
rm -rf vendor build
# Set up vendor directory using `dep`
vendor: Gopkg.toml
vendor: Gopkg.toml Gopkg.lock
$(MAKE) require-dep
dep ensure -vendor-only
touch vendor
@ -68,20 +68,24 @@ release: check-clean
$(MAKE) release-build GOOS=linux GOARCH=386
$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
# Run cluster integration tests using docker. Requires docker daemon to be
# running. This will catch changes to internal/clustertests/*.go, but if you
# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
# pilosa image.
clustertests:
docker-compose -f internal/clustertests/docker-compose.yml down
docker-compose -f internal/clustertests/docker-compose.yml build client1
docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build client1
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
# Like clustertests, but rebuilds all images.
clustertests-build:
docker-compose -f internal/clustertests/docker-compose.yml down
docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Create prerelease builds
prerelease: vendor

53
api.go
View file

@ -333,8 +333,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
}
return err
})
go func(node *Node) {
}(node)
} else if !remote { // if remote == true we don't forward to other nodes
// forward it on
eg.Go(func() error {
@ -561,6 +559,23 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa
return blocks, nil
}
// FragmentData returns all data in the specified fragment.
func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentData")
defer span.Finish()
if err := api.validate(apiFragmentData); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.holder.fragment(indexName, fieldName, viewName, shard)
if f == nil {
return nil, ErrFragmentNotFound
}
return f, nil
}
// Hosts returns a list of the hosts in the cluster including their ID,
// URL, and which is the coordinator.
func (api *API) Hosts(ctx context.Context) []*Node {
@ -1157,6 +1172,36 @@ func (api *API) Info() serverInfo {
}
}
func (api *API) TranslateKeys(body io.Reader) ([]byte, error) {
reqBytes, err := ioutil.ReadAll(body)
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read body error"))
}
var req TranslateKeysRequest
if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error"))
}
var ids []uint64
if req.Field == "" {
ids, err = api.holder.translateFile.TranslateColumnsToUint64(req.Index, req.Keys)
} else {
ids, err = api.holder.translateFile.TranslateRowsToUint64(req.Index, req.Field, req.Keys)
}
if err != nil {
return nil, err
}
resp := TranslateKeysResponse{
IDs: ids,
}
// Encode response.
buf, err := api.Serializer.Marshal(&resp)
if err != nil {
return nil, errors.Wrap(err, "translate keys response encoding error")
}
return buf, nil
}
type serverInfo struct {
ShardWidth uint64 `json:"shardWidth"`
}
@ -1175,6 +1220,7 @@ const (
apiExportCSV
apiFragmentBlockData
apiFragmentBlocks
apiFragmentData
apiField
apiFieldAttrDiff
//apiHosts // not implemented
@ -1204,7 +1250,8 @@ var methodsCommon = map[apiMethod]struct{}{
}
var methodsResizing = map[apiMethod]struct{}{
apiResizeAbort: {},
apiFragmentData: {},
apiResizeAbort: {},
}
var methodsNormal = map[apiMethod]struct{}{

21
api_test.go
View file

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa"
@ -48,7 +49,7 @@ func TestAPI_Import(t *testing.T) {
index := "rick"
field := "f"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
@ -99,6 +100,22 @@ func TestAPI_Import(t *testing.T) {
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("unexpected column keys: %+v", keys)
}
})
// Relies on the previous test creating an index with TrackExistence and
// adding some data.
t.Run("SchemaHasNoExists", func(t *testing.T) {
schema := m1.API.Schema(context.Background())
for _, f := range schema[0].Fields {
if f.Name == "_exists" {
t.Fatalf("found _exists field in schema")
}
if strings.HasPrefix(f.Name, "_") {
t.Fatalf("found internal field '%s' in schema output", f.Name)
}
}
})
t.Run("RowKeyColumnID", func(t *testing.T) {
@ -206,7 +223,7 @@ func TestAPI_ImportValue(t *testing.T) {
t.Fatal(err)
}
pql := fmt.Sprintf("Range(%s>0)", field)
pql := fmt.Sprintf("Row(%s>0)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {

4
client.go
View file

@ -52,7 +52,7 @@ type InternalClient interface {
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
}
@ -149,6 +149,6 @@ func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fie
func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
return nil
}
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) {
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
return nil, nil
}

72
cluster.go
View file

@ -70,7 +70,7 @@ type Node struct {
}
func (n Node) String() string {
return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6])
return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID)
}
// Nodes represents a list of nodes.
@ -364,6 +364,7 @@ func (c *cluster) addNode(node *Node) error {
if !c.Topology.addID(node.ID) {
return nil
}
c.Topology.nodeStates[node.ID] = node.State
// save topology
return c.saveTopology()
@ -588,6 +589,12 @@ func (c *cluster) nodePositionByID(nodeID string) int {
func (c *cluster) addNodeBasicSorted(node *Node) bool {
n := c.unprotectedNodeByID(node.ID)
if n != nil {
if n.State != node.State || n.IsCoordinator != node.IsCoordinator || n.URI != node.URI {
n.State = node.State
n.IsCoordinator = node.IsCoordinator
n.URI = node.URI
return true
}
return false
}
@ -829,6 +836,13 @@ func (c *cluster) partition(index string, shard uint64) int {
return int(h.Sum64() % uint64(c.partitionN))
}
// ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use.
func (c *cluster) ShardNodes(index string, shard uint64) []*Node {
c.mu.RLock()
defer c.mu.RUnlock()
return c.shardNodes(index, shard)
}
// shardNodes returns a list of nodes that own a fragment. unprotected
func (c *cluster) shardNodes(index string, shard uint64) []*Node {
return c.partitionNodes(c.partition(index, shard))
@ -976,7 +990,7 @@ func (c *cluster) markAsJoined() {
// needTopologyAgreement is unprotected.
func (c *cluster) needTopologyAgreement() bool {
return c.state == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
return (c.state == ClusterStateStarting || c.state == ClusterStateDegraded) && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs())
}
// haveTopologyAgreement is unprotected.
@ -1204,7 +1218,7 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (*
Node: toCluster.unprotectedNodeByID(id),
Coordinator: c.unprotectedCoordinatorNode(),
Sources: sources,
Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node.
NodeStatus: c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node.
ClusterStatus: c.unprotectedStatus(),
}
j.Instructions = append(j.Instructions, instr)
@ -1263,12 +1277,30 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "Cluster.followResizeInstruction")
defer span.Finish()
// Sync the schema received in the resize instruction.
// Sync the NodeStatus received in the resize instruction.
// Sync schema.
c.logger.Debugf("holder applySchema")
if err := c.holder.applySchema(instr.Schema); err != nil {
if err := c.holder.applySchema(instr.NodeStatus.Schema); err != nil {
return errors.Wrap(err, "applying schema")
}
// Sync available shards.
for _, is := range instr.NodeStatus.Indexes {
for _, fs := range is.Fields {
f := c.holder.Field(is.Name, fs.Name)
// if we don't know about a field locally, log an error because
// fields should be created and synced prior to shard creation
if f == nil {
c.logger.Printf("local field not found: %s/%s", is.Name, fs.Name)
continue
}
if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
}
}
// Request each source file in ResizeSources.
for _, src := range instr.Sources {
c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
@ -1295,7 +1327,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
// Stream shard from remote node.
c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI)
rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.Shard, srcURI)
rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.View, src.Shard, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
// on the remote node. This occurs when a shard has been skipped and
@ -1304,7 +1336,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
// TODO: figure out a way to distinguish from "fragment not found" errors
// which are true errors and which simply mean the fragment doesn't have data.
if err == ErrFragmentNotFound {
return nil
continue
}
return errors.Wrap(err, "retrieving shard")
} else if rd == nil {
@ -1803,6 +1835,30 @@ func (c *cluster) nodeLeave(nodeID string) error {
return nil
}
func (c *cluster) nodeStatus() *NodeStatus {
ns := &NodeStatus{
Node: c.Node,
Schema: &Schema{Indexes: c.holder.Schema()},
}
var availableShards *roaring.Bitmap
for _, idx := range ns.Schema.Indexes {
is := &IndexStatus{Name: idx.Name}
for _, f := range idx.Fields {
if field := c.holder.Field(idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards()
} else {
availableShards = roaring.NewBitmap()
}
is.Fields = append(is.Fields, &FieldStatus{
Name: f.Name,
AvailableShards: availableShards,
})
}
ns.Indexes = append(ns.Indexes, is)
}
return ns
}
func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
@ -1904,7 +1960,7 @@ type ResizeInstruction struct {
Node *Node
Coordinator *Node
Sources []*ResizeSource
Schema *Schema
NodeStatus *NodeStatus
ClusterStatus *ClusterStatus
}

65
ctl/import_test.go
View file

@ -18,9 +18,11 @@ import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"reflect"
"strings"
"testing"
@ -184,6 +186,69 @@ func TestImportCommand_RunKeys(t *testing.T) {
}
}
// Ensure that import with keys runs with key replication.
func TestImportCommand_KeyReplication(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import-key.csv")
// create a large import file in order to test the
// translateStoreBufferSize growth logic.
keyBytes := []byte{}
for row := 0; row < 100; row++ {
for col := 0; col < 100; col++ {
x := fmt.Sprintf("foo%d,bar%d\n", row, col)
keyBytes = append(keyBytes, x...)
}
}
x := "fooEND,barEND"
keyBytes = append(keyBytes, x...)
file.Write(keyBytes)
ctx := context.Background()
if err != nil {
t.Fatal(err)
}
c := test.MustRunCluster(t, 2)
cmd0 := c[0]
cmd1 := c[1]
host0 := cmd0.API.Node().URI.HostPort()
host1 := cmd1.API.Node().URI.HostPort()
cm.Host = host0
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`)))
cm.Index = "i"
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with key replication doesn't work: %s", err)
}
// Verify that the data is available on both nodes.
for _, host := range []string{host0, host1} {
qry := "Count(Row(f=foo0))"
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry)))
if err != nil {
t.Fatalf("Querying data for validation: %s", err)
}
// Read body and unmarshal response.
exp := `{"results":[100]}` + "\n"
if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatalf("reading: %s", err)
} else if !reflect.DeepEqual(body, []byte(exp)) {
t.Fatalf("expected: %s, but got: %s", exp, body)
}
}
}
// Ensure that integer import with keys runs.
func TestImportCommand_RunValueKeys(t *testing.T) {
buf := bytes.Buffer{}

2
docs/administration.md
View file

@ -323,7 +323,7 @@ We currently track the following events
- **Xor:** Count of Xor queries.
- **Not:** Count of Not queries.
- **Count:** Count of Count queries.
- **Range:** Count of Range queries.
- **Range:** Count of ranged Row queries.
- **Snapshot:** Event count when the snapshot process is triggered.
- **BlockRepair:** Count of data blocks that were out of sync and repaired.
- **GarbageCollection:** Event count when garbage collection occurs.

6
docs/data-model.md
View file

@ -64,7 +64,7 @@ Simple queries:
Relational | Pilosa
-----------------------------------------------|------------------------------------
`select ID from People where Name = 'Bob'` | `Row(Name="Bob")`
`select ID from People where Age > 30` | `Range(Age > 30)`
`select ID from People where Age > 30` | `Row(Age > 30)`
`select ID from People where Member = true` | `Row(Member=0)`
Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join:
@ -100,7 +100,7 @@ The LRU cache maintains the most recently accessed Rows.
### Time Quantum
Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example, if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported.
Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to `YMD`, ranged Row queries down to the granularity of a day are supported.
### Attribute
@ -145,7 +145,7 @@ curl localhost:10101/index/repository/field/quantity \
##### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Row`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.

10
docs/glossary.md
View file

@ -46,16 +46,16 @@ nav = []
<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="range">[Range](../query-language/#range-queries):</strong>: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
<strong id="range-bsi">[Range (BSI)](../query-language/#range-bsi):</strong>: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="range">[Row (Ranged)](../query-language/#range-queries):</strong>: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
<strong id="range-bsi">[Row (BSI)](../query-language/#range-bsi):</strong>: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
<strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
<strong id="shard">[Shard](../data-model/#shard):</strong> [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
@ -64,7 +64,7 @@ nav = []
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [Range](#range) queries on time [fields](#field).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [ranged Row](#range) queries on time [fields](#field).
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).

22
docs/installation.md
View file

@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.1.0
Version: v1.2.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
1. Download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-darwin-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-darwin-amd64.tar.gz
```
Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v1.1.0-darwin-amd64.tar.gz
tar xfz pilosa-v1.2.0-darwin-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v1.1.0-darwin-amd64/pilosa /usr/local/bin
cp -i pilosa-v1.2.0-darwin-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.1.0
Version: v1.2.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.1.0
Version: v1.2.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
@ -222,19 +222,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
1. To install the latest version of Pilosa, download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-linux-amd64.tar.gz
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-linux-amd64.tar.gz
```
Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v1.1.0-linux-amd64.tar.gz
tar xfz pilosa-v1.2.0-linux-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v1.1.0-linux-amd64/pilosa /usr/local/bin
cp -i pilosa-v1.2.0-linux-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
@ -251,7 +251,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.1.0
Version: v1.2.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
@ -314,7 +314,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.1.0
Version: v1.2.0
Build Time: 2018-05-14T22:14:01+0000
Usage:

275
docs/query-language.md
View file

@ -50,7 +50,7 @@ curl localhost:10101/index/repository/query \
* `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*`
* `ATTR_VALUE` Can be a string, float, integer, or bool.
* `CALL` Any query
* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`, `Not`
* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not`
* `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
### Write Operations
@ -335,6 +335,88 @@ Row(stargazer=1)
* attrs are the attributes for user 1
* columns are the repositories which user 1 has starred.
#### Row (Range)
**Spec:**
```
Row(<FIELD>=<ROW>, from=<TIMESTAMP>, to=<TIMESTAMP>)
```
**Description:**
Similar to `Row`, but only returns bits which were set with timestamps between the given `from` (inclusive) and `to` (exclusive) timestamps. Both `from` and `to` parameters are optional. The default for `to` timestamp is current time + 1 day. If a later end timestamp is required, specify it explicitly.
**Result Type:** object with attrs and bits
**Examples:**
Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
```request
Row(stargazer=1, from='2010-01-01T00:00', to='2017-03-02T03:00')
```
```response
{{"attrs":{},"columns":[10]}
```
This example assumes timestamps have been set on some bits.
* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
#### Row (BSI)
**Spec:**
```
Row([<COMPARISON_VALUE> <COMPARISON_OPERATOR>] <FIELD> <COMPARISON_OPERATOR> <COMPARISON_VALUE>)
```
**Description:**
The `Row` query is overloaded to work on `integer` values as well as `timestamp` values.
Returns bits that are true for the comparison operator.
**Result Type:** object with attrs and columns
**Examples:**
In our source data, commitactivity was counted over the last year.
The following greater-than `Row` query returns all columns with a field value greater than 100 (repositories having more than 100 commits):
```request
Row(commitactivity > 100)
```
```response
{{"attrs":{},"columns":[10]}
```
* columns are repositories which had at least 100 commits in the last year.
BSI range queries support the following operators:
Operator | Name | Value
----------|-------------------------------|--------------------
`>` | greater-than, GT | integer
`<` | less-than, LT | integer
`<=` | less-than-or-equal-to, LTE | integer
`>=` | greater-than-or-equal-to, GTE | integer
`==` | equal-to, EQ | integer
`!=` | not-equal-to, NEQ | integer or `null`
`<`, and `<=` can be chained together to represent a bounded interval. For example:
```request
Row(50 < commitactivity < 150)
```
```response
{{"attrs":{},"columns":[10]}
```
As of Pilosa 1.0, the "between" syntax `Row(frame=stats, commitactivity >< [50, 150])` is no longer supported.
#### Union
**Spec:**
@ -584,87 +666,6 @@ TopN(stargazer, n=2, attrName=active, attrValues=[true])
* Results are the top two users (rows) which have the "active" attribute set to "true", sorted by the number of bits set (repositories that they've starred).
#### Range Queries
**Spec:**
```
Range(<FIELD>=<ROW>, <TIMESTAMP>, <TIMESTAMP>)
```
**Description:**
Similar to `Row`, but only returns bits which were set with timestamps
between the given `start` (first) and `end` (second) timestamps.
**Result Type:** object with attrs and bits
**Examples:**
Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range:
```request
Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00)
```
```response
{{"attrs":{},"columns":[10]}
```
This example assumes timestamps have been set on some bits.
* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02.
#### Range (BSI)
**Spec:**
```
Range([<COMPARISON_VALUE> <COMPARISON_OPERATOR>] <FIELD> <COMPARISON_OPERATOR> <COMPARISON_VALUE>)
```
**Description:**
The `Range` query is overloaded to work on `integer` values as well as `timestamp` values.
Returns bits that are true for the comparison operator.
**Result Type:** object with attrs and columns
**Examples:**
In our source data, commitactivity was counted over the last year.
The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits):
```request
Range(commitactivity > 100)
```
```response
{{"attrs":{},"columns":[10]}
```
* columns are repositories which had at least 100 commits in the last year.
BSI range queries support the following operators:
Operator | Name | Value
----------|-------------------------------|--------------------
`>` | greater-than, GT | integer
`<` | less-than, LT | integer
`<=` | less-than-or-equal-to, LTE | integer
`>=` | greater-than-or-equal-to, GTE | integer
`==` | equal-to, EQ | integer
`!=` | not-equal-to, NEQ | integer or `null`
`<`, and `<=` can be chained together to represent a bounded interval. For example:
```request
Range(50 < commitactivity < 150)
```
```response
{{"attrs":{},"columns":[10]}
```
As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported.
#### Min
@ -762,7 +763,7 @@ Modifies the given query as follows:
* `excludeRowAttrs`: Exclude row attributes from the result (Default: `false`).
* `shards`: Run the query using only the data from the given shards. By default, the entire data set (i.e. data from all shards) is used.
**Result Type:** Same result type as <CALL>.
**Result Type:** Same result type as `<CALL>`.
**Examples:**
@ -781,3 +782,111 @@ Options(Row(f1=10), shards=[0, 2])
```response
{"attrs":{},"columns":[100, 2097152]}
```
**Spec:**
```
Rows(field=<STRING>, previous=<UINT|STRING>, limit=<UINT>, column=<UINT|STRING>)
```
**Description:**
Rows returns a list of row IDs in the given field which have at least one bit
set. The field argument is mandatory, the others are optional.
If `previous` is given, rows prior to and including the specified row ID or
key will not be returned. If `column` is given, only rows which have a set bit
in the given column will be returned. `previous` or `column` must be strings if
and only if the field or index respectively is using key translation. If `limit`
is given, the number of rowIDs returned will be less than or equal to
`limit`. The combination of `limit` and `previous` allows for paging over large
result sets. Results are always ordered, so setting `previous` as the last
result of the previous request will start from the next available row.
**Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.`
**Examples:**
Without keys:
```request
Rows(field=blah)
```
```response
{"rows":[1,9,39]}
```
With keys:
```request
Rows(field=blahk)
```
```response
{"rows":null,"keys":["haha","zaaa","traa"]}
```
**Spec:**
```
GroupBy(<RowsCall>, [RowsCall...], limit=<UINT>, filter=<CALL>)
```
**Description:**
GroupBy returns the count of the intersection of every combination of rows
taking one row each from the specified `Rows` calls. It returns only those
combinations for which the count is greater than 0.
The optional `filter` argument takes any type of `Row` query (e.g. Row, Union,
Intersect, etc.) which will be intersected with each result prior to returning
the count. This is analagous to a WHERE clause applied to a relational GROUP BY
query.
The optional `limit` argument limits the number of results returned. The results
are ordered, so as long as the data isn't changing, the same query will return
the same result set.
Paging through results is supported by passing the `previous` argument to each
of the `Rows` calls in the GroupBy. Take the last result from your previous
`GroupBy` query, and pass each row ID in that result as the `previous` argument
to each of the respective `Rows` queries in your next `GroupBy` query.
**Result Type:** Array of "groups". Each group is an object with a group key and
a count key. The count is an integer, and the group is an array of objects which
specify the field and row for each row that was intersected to get that result.
**Examples:**
A single `Rows` query.
```request
GroupBy(Rows(field=blah))
```
```response
[{"group":[{"field":"blah","rowID":1}],"count":1},
{"group":[{"field":"blah","rowID":9}],"count":1},
{"group":[{"field":"blah","rowID":39}],"count":1}]
```
With two `Rows` queries - one with IDs and one with keys.
```request
GroupBy(Rows(field=blah), Rows(field=blahk), limit=7)
```
```response
[{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1},
{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"zaaa"}],"count":1},
{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"traa"}],"count":1},
{"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"haha"}],"count":1},
{"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"zaaa"}],"count":1},
{"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"traa"}],"count":1},
{"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"haha"}],"count":1}]
```
Getting the rest of the results from the previous example (paging).
```request
GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7)
```
```response
[{"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"zaaa"}],"count":1},
{"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"traa"}],"count":1}]
```

18
docs/tutorials.md
View file

@ -444,7 +444,7 @@ Refer to the [Docker documentation](https://docs.docker.com) to see your options
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Row`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
``` request
@ -455,7 +455,7 @@ curl localhost:10101/index/patients \
{"success":true}
```
In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` field.
In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `patients` index.
``` request
curl localhost:10101/index/patients/field/age \
-X POST \
@ -529,22 +529,22 @@ Assuming we have a file called `ages.csv` that is structured like this:
```
where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command:
```
pilosa import -i patients -f measurements --field age ages.csv
pilosa import -i patients --field age ages.csv
```
Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field.
In order to find all patients over the age of 40, then simply run a `Row` query against the `age` field.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Range(age > 40)'
-d 'Row(age > 40)'
```
``` response
{"results":[{"attrs":{},"columns":[2,6,9]}]}
```
You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#range-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
``` request
@ -561,7 +561,7 @@ You can also provide a filter to the `Sum()` function to find the average age of
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Range(age > 40), field="age")'
-d 'Sum(Row(age > 40), field="age")'
```
``` response
{"results":[{"value":191,"count":3}]}
@ -583,7 +583,7 @@ You can also provide a filter to the `Min()` function to find the minimum age of
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Range(age > 40), field="age")'
-d 'Min(Row(age > 40), field="age")'
```
``` response
{"results":[{"value":57,"count":1}]}
@ -604,7 +604,7 @@ You can also provide a filter to the `Max()` function to find the maximum age of
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Range(age < 40), field="age")'
-d 'Max(Row(age < 40), field="age")'
```
``` response
{"results":[{"value":34,"count":1}]}

73
encoding/proto/proto.go
View file

@ -249,6 +249,22 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
decodeBlockDataResponse(msg, mt)
return nil
case *pilosa.TranslateKeysRequest:
msg := &internal.TranslateKeysRequest{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling TranslateKeysRequest")
}
decodeTranslateKeysRequest(msg, mt)
return nil
case *pilosa.TranslateKeysResponse:
msg := &internal.TranslateKeysResponse{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling TranslateKeysResponse")
}
decodeTranslateKeysResponse(msg, mt)
return nil
default:
panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
}
@ -308,6 +324,10 @@ func encodeToProto(m pilosa.Message) proto.Message {
return encodeBlockDataRequest(mt)
case *pilosa.BlockDataResponse:
return encodeBlockDataResponse(mt)
case *pilosa.TranslateKeysRequest:
return encodeTranslateKeysRequest(mt)
case *pilosa.TranslateKeysResponse:
return encodeTranslateKeysResponse(mt)
}
return nil
}
@ -437,7 +457,7 @@ func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstru
Node: encodeNode(m.Node),
Coordinator: encodeNode(m.Coordinator),
Sources: encodeResizeSources(m.Sources),
Schema: encodeSchema(m.Schema),
NodeStatus: encodeNodeStatus(m.NodeStatus),
ClusterStatus: encodeClusterStatus(m.ClusterStatus),
}
}
@ -695,6 +715,20 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac
return &internal.RecalculateCaches{}
}
func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse {
return &internal.TranslateKeysResponse{
IDs: response.IDs,
}
}
func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest {
return &internal.TranslateKeysRequest{
Index: request.Index,
Field: request.Field,
Keys: request.Keys,
}
}
func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) {
m.JobID = ri.JobID
m.Node = &pilosa.Node{}
@ -703,8 +737,8 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeIns
decodeNode(ri.Coordinator, m.Coordinator)
m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources))
decodeResizeSources(ri.Sources, m.Sources)
m.Schema = &pilosa.Schema{}
decodeSchema(ri.Schema, m.Schema)
m.NodeStatus = &pilosa.NodeStatus{}
decodeNodeStatus(ri.NodeStatus, m.NodeStatus)
m.ClusterStatus = &pilosa.ClusterStatus{}
decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus)
}
@ -999,6 +1033,16 @@ func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) {
}
}
func decodeTranslateKeysRequest(pb *internal.TranslateKeysRequest, m *pilosa.TranslateKeysRequest) {
m.Index = pb.Index
m.Field = pb.Field
m.Keys = pb.Keys
}
func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.TranslateKeysResponse) {
m.IDs = pb.IDs
}
// QueryResult types.
const (
queryResultTypeNil uint32 = iota
@ -1103,8 +1147,13 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount {
func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow {
other := make([]pilosa.FieldRow, len(a))
for i := range a {
other[i].Field = a[i].Field
other[i].RowID = a[i].RowID
fr := a[i]
other[i].Field = fr.Field
if fr.RowKey == "" {
other[i].RowID = fr.RowID
} else {
other[i].RowKey = fr.RowKey
}
}
return other
}
@ -1182,9 +1231,17 @@ func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount {
func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow {
other := make([]*internal.FieldRow, len(a))
for i := range a {
other[i] = &internal.FieldRow{
Field: a[i].Field,
RowID: a[i].RowID,
fr := a[i]
if fr.RowKey == "" {
other[i] = &internal.FieldRow{
Field: fr.Field,
RowID: fr.RowID,
}
} else {
other[i] = &internal.FieldRow{
Field: fr.Field,
RowKey: fr.RowKey,
}
}
}
return other

9
enterprise/b/containers_btree.go
View file

@ -177,6 +177,15 @@ func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterato
}, found
}
func (btc *bTreeContainers) Repair() {
e, _ := btc.tree.Seek(0)
_, c, err := e.Next()
for err != io.EOF {
c.Repair()
_, c, err = e.Next()
}
}
type btcIterator struct {
e *enumerator
key uint64

341
executor.go
View file

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"sort"
"time"
@ -87,6 +88,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
resp := QueryResponse{}
// Check for query cancellation.
if err := validateQueryContext(ctx); err != nil {
return resp, err
}
// Verify that an index is set.
if index == "" {
return resp, ErrIndexRequired
@ -112,12 +118,16 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
if !opt.Remote {
if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil {
return resp, err
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
}
results, err := e.execute(ctx, index, q, shards, opt)
if err != nil {
return resp, err
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
resp.Results = results
@ -159,6 +169,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
if !opt.Remote {
if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil {
return resp, err
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
}
@ -217,6 +229,10 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
v, err := e.executeCall(ctx, index, call, shards, opt)
if err != nil {
return nil, err
@ -231,7 +247,9 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall")
defer span.Finish()
if err := e.validateCallArgs(c); err != nil {
if err := validateQueryContext(ctx); err != nil {
return nil, err
} else if err := e.validateCallArgs(c); err != nil {
return nil, errors.Wrap(err, "validating args")
}
indexTag := fmt.Sprintf("index:%s", index)
@ -475,11 +493,11 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
return nil, errors.Wrap(err, "map reduce")
}
// Attach attributes for Row() calls.
// Attach attributes for non-BSI Row() calls.
// If the column label is used then return column attributes.
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Row" {
if c.Name == "Row" && !c.HasConditionArg() {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
@ -521,18 +539,20 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// executeBitmapCallShard executes a bitmap call for a single shard.
func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
switch c.Name {
case "Row":
return e.executeBitmapShard(ctx, index, c, shard)
case "Row", "Range":
return e.executeRowShard(ctx, index, c, shard)
case "Difference":
return e.executeDifferenceShard(ctx, index, c, shard)
case "Intersect":
return e.executeIntersectShard(ctx, index, c, shard)
case "Range":
return e.executeRangeShard(ctx, index, c, shard)
case "Union":
return e.executeUnionShard(ctx, index, c, shard)
case "Xor":
@ -884,6 +904,10 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
} else if hasLimit {
limit = int(lim)
}
filter, _, err := c.CallArg("filter")
if err != nil {
return nil, err
}
// perform necessary Rows queries (any that have limit or columns args) -
// TODO, call async? would only help if multiple Rows queries had a column
@ -892,7 +916,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
childRows := make([]RowIDs, len(c.Children))
for i, child := range c.Children {
if child.Name != "Rows" {
return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", c.Name)
return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name)
}
_, hasLimit, err := child.UintArg("limit")
if err != nil {
@ -915,7 +939,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeGroupByShard(ctx, index, c, shard, childRows)
return e.executeGroupByShard(ctx, index, c, filter, shard, childRows)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
@ -975,7 +999,7 @@ func (fr FieldRow) MarshalJSON() ([]byte, error) {
}
func (fr FieldRow) String() string {
return fmt.Sprintf("%s.%d", fr.Field, fr.RowID)
return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey)
}
type GroupCount struct {
@ -1028,8 +1052,15 @@ func (g GroupCount) Compare(o GroupCount) int {
return 0
}
func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) {
iter, err := newGroupByIterator(childRows, c.Children, index, shard, e.Holder)
func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) {
var filterRow *Row
if filter != nil {
if filterRow, err = e.executeBitmapCallShard(ctx, index, filter, shard); err != nil {
return nil, errors.Wrapf(err, "executing group by filter for shard %d", shard)
}
}
iter, err := newGroupByIterator(childRows, c.Children, filterRow, index, shard, e.Holder)
if err != nil {
return nil, errors.Wrapf(err, "getting group by iterator for shard %d", shard)
}
@ -1138,10 +1169,19 @@ func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call
return frag.rows(start, filters...), nil
}
func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapShard")
func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard")
defer span.Finish()
if c.Name == "Range" {
log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.")
}
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
return e.executeRowBSIGroupShard(ctx, index, c, shard)
}
// Fetch column label from index.
idx := e.Holder.Index(index)
if idx == nil {
@ -1165,93 +1205,43 @@ func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
if frag == nil {
return NewRow(), nil
}
return frag.row(rowID), nil
}
// executeIntersectShard executes a intersect() call for a local shard.
func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard")
defer span.Finish()
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Intersect query is currently not supported")
}
for i, input := range c.Children {
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
if i == 0 {
other = row
} else {
other = other.Intersect(row)
// Parse "from" time, if set.
var fromTime time.Time
if _, ok := c.Args["from"]; ok {
switch v := c.Args["from"].(type) {
case string:
if fromTime, err = time.Parse(TimeFormat, v); err != nil {
return nil, errors.New("cannot parse Row() 'from' time")
}
case int64:
fromTime = time.Unix(v, 0).UTC()
default:
return nil, errors.New("Row() 'from' arg must be a timestamp")
}
}
other.invalidateCount()
return other, nil
}
// executeRangeShard executes a range() call for a local shard.
func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeRangeShard")
defer span.Finish()
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
return e.executeBSIGroupRangeShard(ctx, index, c, shard)
// Parse "to" time, if set.
var toTime time.Time
if _, ok := c.Args["to"]; ok {
switch v := c.Args["to"].(type) {
case string:
if toTime, err = time.Parse(TimeFormat, v); err != nil {
return nil, errors.New("cannot parse Row() 'to' time")
}
case int64:
toTime = time.Unix(v, 0).UTC()
default:
return nil, errors.New("Row() 'to' arg must be a timestamp")
}
}
// Parse field.
fieldName, err := c.FieldArg()
if err != nil {
return nil, errors.New("Range() argument required: field")
}
// Retrieve column label.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
// Retrieve base field.
f := idx.Field(fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
// Read row & column id.
rowID, rowOK, err := c.UintArg(fieldName)
if err != nil {
return nil, fmt.Errorf("executeRangeShard - reading row: %v", err)
}
if !rowOK {
return nil, fmt.Errorf("Range() must specify %q", rowLabel)
}
// Parse start time.
startTimeStr, ok := c.Args["_start"].(string)
if !ok {
return nil, errors.New("Range() start time required")
}
startTime, err := time.Parse(TimeFormat, startTimeStr)
if err != nil {
return nil, errors.New("cannot parse Range() start time")
}
// Parse end time.
endTimeStr, ok := c.Args["_end"].(string)
if !ok {
return nil, errors.New("Range() end time required")
}
endTime, err := time.Parse(TimeFormat, endTimeStr)
if err != nil {
return nil, errors.New("cannot parse Range() end time")
// Simply return row if times are not set.
if c.Name == "Row" && fromTime.IsZero() && toTime.IsZero() {
frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
if frag == nil {
return NewRow(), nil
}
return frag.row(rowID), nil
}
// If no quantum exists then return an empty bitmap.
@ -1260,9 +1250,16 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C
return &Row{}, nil
}
// Set maximum "to" value if only "from" is set. We don't need to worry
// about setting the minimum "from" since it is the zero value if omitted.
if toTime.IsZero() {
// Set the end timestamp to current time + 1 day, in order to account for timezone differences.
toTime = time.Now().AddDate(0, 0, 1)
}
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range viewsByTimeRange(viewStandard, startTime, endTime, q) {
for _, view := range viewsByTimeRange(viewStandard, fromTime, toTime, q) {
f := e.Holder.fragment(index, fieldName, view, shard)
if f == nil {
continue
@ -1271,18 +1268,19 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C
}
f.Stats.Count("range", 1, 1.0)
return row, nil
}
// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard.
func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBSIGroupRangeShard")
// executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard.
func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard")
defer span.Finish()
// Only one conditional should be present.
if len(c.Args) == 0 {
return nil, errors.New("Range(): condition required")
return nil, errors.New("Row(): condition required")
} else if len(c.Args) > 1 {
return nil, errors.New("Range(): too many arguments")
return nil, errors.New("Row(): too many arguments")
}
// Extract conditional.
@ -1291,7 +1289,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string,
for k, v := range c.Args {
vv, ok := v.(*pql.Condition)
if !ok {
return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v)
return nil, fmt.Errorf("Row(): %q: expected condition argument, got %v", k, v)
}
fieldName, cond = k, vv
}
@ -1303,7 +1301,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string,
// EQ null (not implemented: flip frag.NotNull with max ColumnID)
// NEQ null frag.NotNull()
// BETWEEN a,b(in) BETWEEN/frag.RangeBetween()
// BETWEEN a,b(in) BETWEEN/frag.RowBetween()
// BETWEEN a,b(out) BETWEEN/frag.NotNull()
// EQ <int> frag.RangeOp
// NEQ <int> frag.RangeOp
@ -1333,11 +1331,11 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string,
// Only support two integers for the between operation.
if len(predicates) != 2 {
return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values")
return nil, errors.New("Row(): BETWEEN condition requires exactly two integer values")
}
// The reason we don't just call:
// return f.RangeBetween(fieldName, predicates[0], predicates[1])
// return f.RowBetween(fieldName, predicates[0], predicates[1])
// here is because we need the call to be shard-specific.
// Find bsiGroup.
@ -1370,7 +1368,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string,
// Only support integers for now.
value, ok := cond.Value.(int64)
if !ok {
return nil, errors.New("Range(): conditions only support integer values")
return nil, errors.New("Row(): conditions only support integer values")
}
// Find bsiGroup.
@ -1406,6 +1404,31 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string,
}
}
// executeIntersectShard executes a intersect() call for a local shard.
func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard")
defer span.Finish()
var other *Row
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Intersect query is currently not supported")
}
for i, input := range c.Children {
row, err := e.executeBitmapCallShard(ctx, index, input, shard)
if err != nil {
return nil, err
}
if i == 0 {
other = row
} else {
other = other.Intersect(row)
}
}
other.invalidateCount()
return other, nil
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@ -1966,7 +1989,13 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
for i, c := range calls {
if i%10 == 0 {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
}
field, ok := c.Args["_field"].(string)
if !ok {
return nil, errors.New("SetRowAttrs() field required")
@ -2127,7 +2156,7 @@ func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (m
loop:
for _, shard := range shards {
for _, node := range e.Cluster.shardNodes(index, shard) {
for _, node := range e.Cluster.ShardNodes(index, shard) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], shard)
continue loop
@ -2549,6 +2578,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res
return result, nil
}
// validateQueryContext returns a query-appropriate error if the context is done.
func validateQueryContext(ctx context.Context) error {
select {
case <-ctx.Done():
switch err := ctx.Err(); err {
case context.Canceled:
return ErrQueryCancelled
case context.DeadlineExceeded:
return ErrQueryTimeout
default:
return err
}
default:
return nil
}
}
// errShardUnavailable is a marker error if no nodes are available.
var errShardUnavailable = errors.New("shard unavailable")
@ -2687,22 +2733,32 @@ type groupByIterator struct {
// fields and then sets the row ids.
fields []FieldRow
done bool
// Optional filter row to intersect against first level of values.
filter *Row
}
// newGroupByIterator initializes a new groupByIterator.
func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) (*groupByIterator, error) {
func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, index string, shard uint64, holder *Holder) (*groupByIterator, error) {
gbi := &groupByIterator{
rowIters: make([]*rowIterator, len(children)),
rows: make([]struct {
row *Row
id uint64
}, len(children)),
filter: filter,
fields: make([]FieldRow, len(children)),
}
ignorePrev := false
for i, call := range children {
fieldName := call.Args["field"].(string) // this has already been validated by this point
fieldName, ok := call.Args["field"].(string)
if !ok {
return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"])
}
if holder.Field(index, fieldName) == nil {
return nil, ErrFieldNotFound
}
gbi.fields[i].Field = fieldName
// Fetch fragment.
frag := holder.fragment(index, fieldName, viewStandard, shard)
@ -2756,6 +2812,11 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha
}
}
// Apply filter to first level, if available.
if gbi.filter != nil && len(gbi.rows) > 0 {
gbi.rows[0].row = gbi.rows[0].row.Intersect(gbi.filter)
}
for i := 1; i < len(gbi.rows)-1; i++ {
gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row)
}
@ -2766,32 +2827,49 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha
// nextAtIdx is a recursive helper method for getting the next row for the field
// at index i, and then updating the rows in the "higher" fields if it wraps.
func (gbi *groupByIterator) nextAtIdx(i int) {
nr, rowID, wrapped := gbi.rowIters[i].Next()
if nr == nil {
gbi.done = true
return
// loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed.
for {
nr, rowID, wrapped := gbi.rowIters[i].Next()
if nr == nil {
gbi.done = true
return
}
if wrapped && i != 0 {
gbi.nextAtIdx(i - 1)
}
if i == 0 && gbi.filter != nil {
gbi.rows[i].row = nr.Intersect(gbi.filter)
} else if i == 0 || i == len(gbi.rows)-1 {
gbi.rows[i].row = nr
} else {
gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row)
}
gbi.rows[i].id = rowID
if !gbi.rows[i].row.IsEmpty() {
break
}
}
if wrapped && i != 0 {
gbi.nextAtIdx(i - 1)
}
if i != 0 && i != len(gbi.rows)-1 {
gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row)
} else {
gbi.rows[i].row = nr
}
gbi.rows[i].id = rowID
}
// Next returns a GroupCount representing the next group by record. When there
// are no more records it will return an empty GroupCount and done==true.
func (gbi *groupByIterator) Next() (ret GroupCount, done bool) {
if gbi.done {
return ret, true
}
if len(gbi.rows) == 1 {
ret.Count = gbi.rows[len(gbi.rows)-1].row.Count()
} else {
ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row)
// loop until we find a result with count > 0
for {
if gbi.done {
return ret, true
}
if len(gbi.rows) == 1 {
ret.Count = gbi.rows[len(gbi.rows)-1].row.Count()
} else {
ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row)
}
if ret.Count == 0 {
gbi.nextAtIdx(len(gbi.rows) - 1)
continue
}
break
}
ret.Group = make([]FieldRow, len(gbi.rows))
@ -2801,6 +2879,7 @@ func (gbi *groupByIterator) Next() (ret GroupCount, done bool) {
}
// set up for next call
gbi.nextAtIdx(len(gbi.rows) - 1)
return ret, false

579
executor_test.go
View file

@ -1478,23 +1478,29 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
// Ensure a range query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
func TestExecutor_Execute_Row_Range(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {
writeQuery := `
// Create a timestamp just out of the current date + 1 day timestamp (default end timestamp).
nextDayExclusive := time.Now().AddDate(0, 0, 2)
writeQuery := fmt.Sprintf(`
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
Set(8, f=1, %s)
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)`
Set(2, f=10, 2001-01-01T00:00)`, nextDayExclusive.Format("2006-01-02T15:04"))
readQueries := []string{
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00)`,
`Row(f=1, to=2002-01-01T02:00)`,
`Clear( 2, f=1)`,
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
@ -1505,8 +1511,20 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
})
t.Run("From", func(t *testing.T) {
if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("To", func(t *testing.T) {
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
if columns := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
@ -1525,9 +1543,9 @@ func TestExecutor_Execute_Range(t *testing.T) {
Set("two", f=1, 2002-02-01T00:00)
Set("two", f=10, 2001-01-01T00:00)`
readQueries := []string{
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear("two", f=1)`,
`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
@ -1559,9 +1577,9 @@ func TestExecutor_Execute_Range(t *testing.T) {
Set(2, f="foo", 2002-02-01T00:00)
Set(2, f="bar", 2001-01-01T00:00)`
readQueries := []string{
`Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear( 2, f="foo")`,
`Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
nil,
@ -1594,9 +1612,182 @@ func TestExecutor_Execute_Range(t *testing.T) {
Set("two", f="foo", 2002-02-01T00:00)
Set("two", f="bar", 2001-01-01T00:00)`
readQueries := []string{
`Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear("two", f="foo")`,
`Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`,
`Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) {
t.Fatalf("unexpected keys: %+v", keys)
}
})
t.Run("Clear", func(t *testing.T) {
if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) {
t.Fatalf("unexpected keys: %+v", keys)
}
})
})
t.Run("UnixTimestamp", func(t *testing.T) {
writeQuery := `
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)`
readQueries := []string{
`Row(f=1, from=946598400, to=1009854000)`,
`Clear( 2, f=1)`,
`Row(f=1, from=946598400, to=1009854000)`,
}
responses := runCallTest(t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
})
}
// Ensure a range query can be executed.
func TestExecutor_Execute_Range_Deprecated(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {
writeQuery := `
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)`
readQueries := []string{
`Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear( 2, f=1)`,
`Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
})
t.Run("RowIDColumnKey", func(t *testing.T) {
writeQuery := `
Set("two", f=1, 1999-12-31T00:00)
Set("three", f=1, 2000-01-01T00:00)
Set("four", f=1, 2000-01-02T00:00)
Set("five", f=1, 2000-02-01T00:00)
Set("six", f=1, 2001-01-01T00:00)
Set("seven", f=1, 2002-01-01T02:00)
Set("two", f=1, 1999-12-30T00:00)
Set("two", f=1, 2002-02-01T00:00)
Set("two", f=10, 2001-01-01T00:00)`
readQueries := []string{
`Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear("two", f=1)`,
`Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")))
t.Run("Standard", func(t *testing.T) {
if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) {
t.Fatalf("unexpected keys: %+v", keys)
}
})
t.Run("Clear", func(t *testing.T) {
if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) {
t.Fatalf("unexpected keys: %+v", keys)
}
})
})
t.Run("RowKeyColumnID", func(t *testing.T) {
writeQuery := `
Set(2, f="foo", 1999-12-31T00:00)
Set(3, f="foo", 2000-01-01T00:00)
Set(4, f="foo", 2000-01-02T00:00)
Set(5, f="foo", 2000-02-01T00:00)
Set(6, f="foo", 2001-01-01T00:00)
Set(7, f="foo", 2002-01-01T02:00)
Set(2, f="foo", 1999-12-30T00:00)
Set(2, f="foo", 2002-02-01T00:00)
Set(2, f="bar", 2001-01-01T00:00)`
readQueries := []string{
`Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear( 2, f="foo")`,
`Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
nil,
pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")),
pilosa.OptFieldKeys())
t.Run("Standard", func(t *testing.T) {
if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Clear", func(t *testing.T) {
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
})
t.Run("RowKeyColumnKey", func(t *testing.T) {
writeQuery := `
Set("two", f="foo", 1999-12-31T00:00)
Set("three", f="foo", 2000-01-01T00:00)
Set("four", f="foo", 2000-01-02T00:00)
Set("five", f="foo", 2000-02-01T00:00)
Set("six", f="foo", 2001-01-01T00:00)
Set("seven", f="foo", 2002-01-01T02:00)
Set("two", f="foo", 1999-12-30T00:00)
Set("two", f="foo", 2002-02-01T00:00)
Set("two", f="bar", 2001-01-01T00:00)`
readQueries := []string{
`Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
`Clear("two", f="foo")`,
`Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
@ -1617,8 +1808,202 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
}
// Ensure a Range(bsiGroup) query can be executed.
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
// Ensure a Row(bsiGroup) query can be executed.
func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil {
t.Fatal(err)
}
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set(0, f=0)
Set(` + strconv.Itoa(ShardWidth+1) + `, f=0)
Set(50, foo=20)
Set(50, bar=2000)
Set(` + strconv.Itoa(ShardWidth) + `, foo=30)
Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10)
Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20)
Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60)
Set(0, other=1000)
Set(0, edge=100)
Set(1, edge=-100)
`}); err != nil {
t.Fatal(err)
}
t.Run("EQ", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("NEQ", func(t *testing.T) {
// NEQ null
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns())
}
})
t.Run("LT", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo < 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("LTE", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("GT", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("GTE", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("BETWEEN", func(t *testing.T) {
tests := []struct {
q string
exp bool
}{
{q: `Row(0 < other < 1000)`, exp: false},
{q: `Row(0 <= other < 1000)`, exp: false},
{q: `Row(0 <= other <= 1000)`, exp: true},
{q: `Row(0 < other <= 1000)`, exp: true},
{q: `Row(1000 < other < 1000)`, exp: false},
{q: `Row(1000 <= other < 1000)`, exp: false},
{q: `Row(1000 <= other <= 1000)`, exp: true},
{q: `Row(1000 < other <= 1000)`, exp: false},
{q: `Row(1000 < other < 2000)`, exp: false},
{q: `Row(1000 <= other < 2000)`, exp: true},
{q: `Row(1000 <= other <= 2000)`, exp: true},
{q: `Row(1000 < other <= 2000)`, exp: false},
}
for i, test := range tests {
t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) {
var expected = []uint64{}
if test.exp {
expected = []uint64{0}
}
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result for query: %s", test.q)
}
})
}
})
// Ensure that the NotNull code path gets run.
t.Run("NotNull", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("BelowMin", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 0)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("AboveMax", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 200)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("LTAboveMax", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge < 200)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns()))
}
})
t.Run("GTBelowMin", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -200)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns()))
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound {
t.Fatal(err)
}
})
}
// Ensure a Range(bsiGroup) query can be executed. (Deprecated)
func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
@ -1730,14 +2115,14 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
} else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
// Ensure that the NotNull code path gets run.
t.Run("NotNull", func(t *testing.T) {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil {
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -1907,7 +2292,7 @@ Set(4500001, fn=4)
{Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4},
}
results := res.Results[0].([]pilosa.GroupCount)
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
}
})
}
@ -2010,7 +2395,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) {
Set(2, f=10, 2001-01-01T00:00)
`
clearColumn := `Clear( 2, f=1)`
rangeCheckQuery := `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`
rangeCheckQuery := `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`
for i, tt := range rangeTests {
t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) {
@ -2163,10 +2548,10 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatalf("unexpected columns: %+v", bits)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
t.Fatalf("unexpected existence columns: %+v", bits)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after Not: %+v", bits)
}
// Reopen cluster to ensure existence field is reloaded.
@ -2174,10 +2559,10 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatal(err)
}
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
t.Fatalf("unexpected existence columns after reopen: %+v", bits)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after reopen: %+v", bits)
}
})
}
@ -2346,10 +2731,10 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)`
readQueries := []string{
`Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`,
`ClearRow(f=1)`,
`Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`,
`Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`,
`Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`,
`Row(f=10, from=1999-12-31T00:00, to=2003-01-01T03:00)`,
}
responses := runCallTest(t, writeQuery, readQueries,
&pilosa.IndexOptions{TrackExistence: true},
@ -2674,6 +3059,66 @@ func TestExecutor_Execute_Rows(t *testing.T) {
}
}
func TestExecutor_Execute_Query_Error(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
c.CreateField(t, "i", pilosa.IndexOptions{}, "general")
tests := []struct {
query string
error string
}{
{
query: "GroupBy(Rows())",
error: "Rows call must have 'field' argument",
},
{
query: "GroupBy(Rows(field=true))",
error: "Rows call must have 'field' argument",
},
{
query: "GroupBy(Rows(field=\"true\"))",
error: "field not found",
},
{
query: "GroupBy(Rows(field=1))",
error: "Rows call must have 'field' argument",
},
{
query: "GroupBy(Rows(field))",
error: "parse error",
},
{
query: "GroupBy(Rows(field=general, limit=-1))",
error: "must be positive, but got",
},
{
query: "GroupBy(Rows(field=general), limit=-1)",
error: "must be positive, but got",
},
{
query: "GroupBy(Rows(field=general), filter=Rows(field=general))",
error: "unknown call: Rows",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i",
Query: test.query,
})
if err == nil {
t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r)
}
if !strings.Contains(err.Error(), test.error) {
t.Fatalf("unexpected error message: %s", err.Error())
}
})
}
}
func TestExecutor_Execute_Rows_Keys(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -2842,7 +3287,17 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
}
results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount)
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
t.Run("Filter", func(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3},
{Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount)
test.CheckGroupBy(t, expected, results)
})
t.Run("check field offset no limit", func(t *testing.T) {
@ -2852,7 +3307,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
}
results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount)
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
t.Run("check field offset limit", func(t *testing.T) {
@ -2861,7 +3316,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
}
results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount)
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
@ -2882,7 +3337,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
}
results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount)
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
// set the same bits in a single shard in three fields
@ -2915,7 +3370,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
t.Run("test previous is last result", func(t *testing.T) {
@ -2930,7 +3385,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
// test multiple shards with distinct results (different rows) and same
@ -2958,7 +3413,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
t.Run("distinct rows in different shards with row limit", func(t *testing.T) {
@ -2969,7 +3424,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
t.Run("distinct rows in different shards with column arg", func(t *testing.T) {
@ -2980,7 +3435,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 3}}, Count: 1},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
c.CreateField(t, "i", pilosa.IndexOptions{}, "na")
@ -3005,7 +3460,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 0}}, Count: 2},
{Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 1}}, Count: 2},
}
checkGroupBy(t, expected, results)
test.CheckGroupBy(t, expected, results)
})
@ -3050,8 +3505,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
}
expected[63].Count = 5
checkGroupBy(t, expected, totalResults)
test.CheckGroupBy(t, expected, totalResults)
})
// test row keys
c.CreateField(t, "i", pilosa.IndexOptions{}, "generalk", pilosa.OptFieldKeys())
c.CreateField(t, "i", pilosa.IndexOptions{}, "subk", pilosa.OptFieldKeys())
c.Query(t, "i", `
Set(0, generalk="ten")
Set(1, generalk="ten")
Set(1001, generalk="ten")
Set(2, generalk="eleven")
Set(1002, generalk="eleven")
Set(2, generalk="twelve")
Set(1002, generalk="twelve")
Set(0, subk="one-hundred")
Set(1, subk="one-hundred")
Set(3, subk="one-hundred")
Set(1001, subk="one-hundred")
Set(2, subk="one-hundred-ten")
Set(0, subk="one-hundred-ten")
`)
t.Run("test row keys", func(t *testing.T) {
// the execututor returns row IDs when the field has keys, so they should be included in the target.
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 1, RowKey: "one-hundred"}}, Count: 3},
{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 2, RowKey: "eleven"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1},
}
results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount)
test.CheckGroupBy(t, expected, results)
})
}
for size := range []int{1, 3} {
t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) {
@ -3114,18 +3604,9 @@ func BenchmarkGroupBy(b *testing.B) {
}
func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) {
if len(results) != len(expected) {
t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected)
}
for i, result := range results {
if !reflect.DeepEqual(expected[i], result) {
t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i])
}
}
}
func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse {
t.Helper()
if indexOptions == nil {
indexOptions = &pilosa.IndexOptions{}
}

2
field.go
View file

@ -1124,7 +1124,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts
frag, err := view.CreateFragmentIfNotExists(key.Shard)
if err != nil {
return errors.Wrap(err, "creating view")
return errors.Wrap(err, "creating fragment")
}
if err := frag.bulkImport(data.RowIDs, data.ColumnIDs, options); err != nil {

102
fragment.go
View file

@ -28,6 +28,7 @@ import (
"math"
"os"
"sort"
"strings"
"sync"
"syscall"
"time"
@ -2315,46 +2316,38 @@ func (s *fragmentSyncer) syncBlock(id int) error {
// Write updates to remote blocks.
for i := 0; i < len(uris); i++ {
set, clear := sets[i], clears[i]
count := 0
// Ignore if there are no differences.
if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 {
continue
}
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.columnIDs) + len(clear.columnIDs)
maxWrites := s.Cluster.maxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
}
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j])
count++
}
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j])
count++
}
// Iterate over the buffers.
for k := 0; k < len(buffers); k++ {
// Verify sync is not prematurely closing.
if s.isClosing() {
return nil
}
// Execute query.
queryRequest := &QueryRequest{
Query: buffers[k].String(),
Remote: true,
}
_, err := s.Cluster.InternalClient.QueryNode(ctx, uris[i], f.index, queryRequest)
// Handle Sets.
if len(set.columnIDs) > 0 {
setData, err := bitsToRoaringData(set)
if err != nil {
return errors.Wrap(err, "executing")
return errors.Wrap(err, "converting bits to roaring data (set)")
}
setReq := &ImportRoaringRequest{
Clear: false,
Views: map[string][]byte{cleanViewName(f.view): setData},
}
if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, setReq); err != nil {
return errors.Wrap(err, "sending roaring data (set)")
}
}
// Handle Clears.
if len(clear.columnIDs) > 0 {
clearData, err := bitsToRoaringData(clear)
if err != nil {
return errors.Wrap(err, "converting bits to roaring data (clear)")
}
clearReq := &ImportRoaringRequest{
Clear: true,
Views: map[string][]byte{"": clearData},
}
if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, clearReq); err != nil {
return errors.Wrap(err, "sending roaring data (clear)")
}
}
}
@ -2362,6 +2355,39 @@ func (s *fragmentSyncer) syncBlock(id int) error {
return nil
}
// cleanViewName converts a viewname into the equivalent
// string required by the external api. Because views are
// not exposed externally, the conversion looks like this:
// "standard" -> ""
// "standard_YYYYMMDD" -> "YYYYMMDD"
// "other" -> "other" (there is currently not a use for this)
func cleanViewName(v string) string {
viewPrefix := viewStandard + "_"
if strings.HasPrefix(v, viewPrefix) {
return v[len(viewPrefix):]
} else if v == viewStandard {
return ""
}
return v
}
// bitsToRoaringData converts a pairSet into a roaring.Bitmap
// which represents the data within a single shard.
func bitsToRoaringData(ps pairSet) ([]byte, error) {
bmp := roaring.NewBitmap()
for j := 0; j < len(ps.columnIDs); j++ {
bmp.DirectAdd(ps.rowIDs[j]*ShardWidth + (ps.columnIDs[j] % ShardWidth))
}
var buf bytes.Buffer
_, err := bmp.WriteTo(&buf)
if err != nil {
return nil, errors.Wrap(err, "writing to buffer")
}
return buf.Bytes(), nil
}
func madvise(b []byte, advice int) error { // nolint: unparam
_, _, err := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice))
if err != 0 {

482
fragment_internal_test.go
View file

@ -20,6 +20,8 @@ import (
"fmt"
"io/ioutil"
"math"
"math/rand"
"os"
"reflect"
"sort"
"testing"
@ -37,12 +39,17 @@ var (
// In order to generate the sample fragment file,
// run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view
FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path")
TempDir = ""
)
func init() { // nolint: gochecknoinits
flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.")
}
// Ensure a fragment can set a bit and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(120, 1); err != nil {
@ -73,7 +80,7 @@ func TestFragment_SetBit(t *testing.T) {
// Ensure a fragment can clear a set bit.
func TestFragment_ClearBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -100,7 +107,7 @@ func TestFragment_ClearBit(t *testing.T) {
// Ensure a fragment can clear a row.
func TestFragment_ClearRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -127,7 +134,7 @@ func TestFragment_ClearRow(t *testing.T) {
// Ensure a fragment can set a row.
func TestFragment_SetRow(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 7, "")
defer f.Close()
defer f.Clean(t)
rowID := uint64(1000)
@ -176,7 +183,7 @@ func TestFragment_SetRow(t *testing.T) {
func TestFragment_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -204,7 +211,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("Overwrite", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -232,7 +239,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("Clear", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 16, 3829); err != nil {
@ -260,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) {
t.Run("NotExists", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set value.
if changed, err := f.setValue(100, 10, 20); err != nil {
@ -290,7 +297,7 @@ func TestFragment_SetValue(t *testing.T) {
}
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
m := make(map[uint64]int64)
@ -328,7 +335,7 @@ func TestFragment_Sum(t *testing.T) {
const bitDepth = 16
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -360,6 +367,20 @@ func TestFragment_Sum(t *testing.T) {
t.Fatalf("unexpected sum: %d", sum)
}
})
// verify that clearValue clears values
if _, err := f.clearValue(1000, bitDepth, 23); err != nil {
t.Fatal(err)
}
t.Run("ClearValue", func(t *testing.T) {
if sum, n, err := f.sum(nil, bitDepth); err != nil {
t.Fatal(err)
} else if n != 3 {
t.Fatalf("unexpected count: %d", n)
} else if sum != (3800 - 382) {
t.Fatalf("unexpected sum: got %d, expecting %d", sum, 3800-382)
}
})
}
// Ensure a fragment can find the min and max of values.
@ -367,7 +388,7 @@ func TestFragment_MinMax(t *testing.T) {
const bitDepth = 16
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -441,7 +462,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -464,7 +485,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("NEQ", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -487,7 +508,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("LT", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -535,7 +556,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("GT", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -583,7 +604,7 @@ func TestFragment_Range(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set values.
if _, err := f.setValue(1000, bitDepth, 382); err != nil {
@ -630,10 +651,75 @@ func TestFragment_Range(t *testing.T) {
})
}
// benchmarkSetValues is a helper function to explore, very roughly, the cost
// of setting values.
func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
for i := 0; i < b.N; i++ {
f.setValue(column, bitDepth, uint64(i))
column = cfunc(column)
}
}
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_SetValue(b *testing.B) {
depths := []uint{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none")
b.Run(name+"_Sparse", func(b *testing.B) {
benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) })
})
f.Clean(b)
f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none")
b.Run(name+"_Dense", func(b *testing.B) {
benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) })
})
f.Clean(b)
}
}
// benchmarkImportValues is a helper function to explore, very roughly, the cost
// of setting values using the special setter used for imports.
func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
b.StopTimer()
columns := make([]uint64, b.N)
values := make([]uint64, b.N)
for i := 0; i < b.N; i++ {
values[i] = uint64(i)
columns[i] = column
column = cfunc(column)
}
b.StartTimer()
err := f.importValue(columns, values, bitDepth, false)
if err != nil {
b.Fatalf("error importing values: %s", err)
}
}
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_ImportValue(b *testing.B) {
depths := []uint{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none")
b.Run(name+"_Sparse", func(b *testing.B) {
benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) })
})
f.Clean(b)
f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none")
b.Run(name+"_Dense", func(b *testing.B) {
benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) })
})
f.Clean(b)
}
}
// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f.setBit(1000, 1); err != nil {
@ -662,7 +748,7 @@ func TestFragment_Snapshot(t *testing.T) {
// Ensure a fragment can iterate over all bits in order.
func TestFragment_ForEachBit(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set bits on the fragment.
if _, err := f.setBit(100, 20); err != nil {
@ -691,7 +777,7 @@ func TestFragment_ForEachBit(t *testing.T) {
// Ensure a fragment can return the top n results.
func TestFragment_Top(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
// Set bits on the rows 100, 101, & 102.
f.mustSetBits(100, 1, 3, 200)
f.mustSetBits(101, 1)
@ -713,7 +799,7 @@ func TestFragment_Top(t *testing.T) {
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
// Set bits on the rows 100, 101, & 102.
f.mustSetBits(100, 1, 3, 200)
@ -743,7 +829,7 @@ func TestFragment_Top_Filter(t *testing.T) {
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
// Create an intersecting input row.
src := NewRow(1, 2, 3)
@ -774,7 +860,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
}
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
// Create an intersecting input row.
src := NewRow(
@ -812,7 +898,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
// Ensure a fragment can return top rows when specified by ID.
func TestFragment_TopN_IDs(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -833,7 +919,7 @@ func TestFragment_TopN_IDs(t *testing.T) {
// Ensure a fragment return none if CacheTypeNone is set
func TestFragment_TopN_NopCache(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone)
defer f.Close()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -881,7 +967,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
if err := f.Open(); err != nil {
panic(err)
}
defer f.Close()
defer f.Clean(t)
// Set bits on various rows.
f.mustSetBits(100, 1, 2, 3)
@ -914,7 +1000,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
// Ensure fragment can return a checksum for its blocks.
func TestFragment_Checksum(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Retrieve checksum and set bits.
orig := f.Checksum()
@ -933,7 +1019,7 @@ func TestFragment_Checksum(t *testing.T) {
// Ensure fragment can return a checksum for a given block.
func TestFragment_Blocks(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Retrieve initial checksum.
var prev []FragmentBlock
@ -971,7 +1057,7 @@ func TestFragment_Blocks(t *testing.T) {
// Ensure fragment returns an empty checksum if no data exists for a block.
func TestFragment_Blocks_Empty(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set bits on a different block.
if _, err := f.setBit(100, 1); err != nil {
@ -989,7 +1075,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_LRUCache_Persistence(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU)
defer f.Close()
defer f.Clean(t)
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
@ -1074,7 +1160,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f0.Close()
defer f0.Clean(t)
// Set and then clear bits on the fragment.
if _, err := f0.setBit(1000, 1); err != nil {
@ -1135,7 +1221,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Close()
defer f.CleanKeep(b)
// Reset timer and execute benchmark.
b.ResetTimer()
@ -1148,7 +1234,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
func BenchmarkFragment_IntersectionCount(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(b)
f.MaxOpN = math.MaxInt32
// Generate some intersecting data.
@ -1179,7 +1265,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
func TestFragment_Tanimoto(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
src := NewRow(1, 2, 3)
@ -1202,7 +1288,7 @@ func TestFragment_Tanimoto(t *testing.T) {
func TestFragment_Zero_Tanimoto(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
src := NewRow(1, 2, 3)
@ -1227,7 +1313,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
func TestFragment_Snapshot_Run(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set bits on the fragment.
for i := uint64(1); i < 3; i++ {
@ -1254,7 +1340,7 @@ func TestFragment_Snapshot_Run(t *testing.T) {
// Ensure a fragment can set mutually exclusive values.
func TestFragment_SetMutex(t *testing.T) {
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
var cols []uint64
@ -1368,7 +1454,7 @@ func TestFragment_ImportSet(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1404,7 +1490,7 @@ func TestFragment_ImportSet(t *testing.T) {
func TestFragment_ConcurrentImport(t *testing.T) {
t.Run("bulkImportStandard", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
eg := errgroup.Group{}
eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) })
@ -1501,7 +1587,7 @@ func TestFragment_ImportMutex(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1620,7 +1706,7 @@ func TestFragment_ImportBool(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
// Set import.
err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{})
@ -1664,7 +1750,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
if err := f.Open(); err != nil {
b.Fatal(err)
}
defer f.Close()
defer f.CleanKeep(b)
b.ResetTimer()
// Reset timer and execute benchmark.
@ -1680,7 +1766,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
func BenchmarkFragment_FullSnapshot(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(b)
// Generate some intersecting data.
maxX := 1048576 / 2
sz := maxX
@ -1718,7 +1804,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
func BenchmarkFragment_Import(b *testing.B) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(b)
maxX := 1048576 * 5 * 2
sz := maxX
rows := make([]uint64, sz)
@ -1746,11 +1832,300 @@ func BenchmarkFragment_Import(b *testing.B) {
}
}
var (
rowCases = []uint64{2, 50, 1000, 100000}
colCases = []uint64{20, 1000, 50000, 500000}
concurrencyCases = []int{2, 4, 8, 16}
)
func BenchmarkImportRoaring(b *testing.B) {
for _, numRows := range rowCases {
data := getZipfRowsSliceRoaring(numRows, 1)
b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024)
for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much
b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) {
b.StopTimer()
for i := 0; i < b.N; i++ {
f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType)
b.StartTimer()
err := f.importRoaring(data, false)
if err != nil {
f.Clean(b)
b.Fatalf("import error: %v", err)
}
b.StopTimer()
f.Clean(b)
}
})
}
}
}
func BenchmarkImportRoaringConcurrent(b *testing.B) {
for _, numRows := range rowCases {
data := getZipfRowsSliceRoaring(numRows, 1)
b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024)
for _, concurrency := range concurrencyCases {
b.Run(fmt.Sprintf("%dRows%dConcurrency", numRows, concurrency), func(b *testing.B) {
b.StopTimer()
frags := make([]*fragment, concurrency)
for i := 0; i < b.N; i++ {
for j := 0; j < concurrency; j++ {
frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked)
}
eg := errgroup.Group{}
b.StartTimer()
for j := 0; j < concurrency; j++ {
j := j
eg.Go(func() error {
return frags[j].importRoaring(data, false)
})
}
err := eg.Wait()
if err != nil {
b.Errorf("importing fragment: %v", err)
}
b.StopTimer()
for j := 0; j < concurrency; j++ {
frags[j].Clean(b)
}
}
})
}
}
}
func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
for _, numRows := range rowCases {
for _, numCols := range colCases {
data := getZipfRowsSliceRoaring(numRows, 1)
updata := getUpdataRoaring(numRows, numCols, 1)
for _, concurrency := range concurrencyCases {
b.Run(fmt.Sprintf("%dRows%dCols%dConcurrency", numRows, numCols, concurrency), func(b *testing.B) {
b.StopTimer()
frags := make([]*fragment, concurrency)
for i := 0; i < b.N; i++ {
for j := 0; j < concurrency; j++ {
frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked)
frags[j].importRoaring(data, false)
}
eg := errgroup.Group{}
b.StartTimer()
for j := 0; j < concurrency; j++ {
j := j
eg.Go(func() error {
return frags[j].importRoaring(updata, false)
})
}
err := eg.Wait()
if err != nil {
b.Errorf("importing fragment: %v", err)
}
b.StopTimer()
for j := 0; j < concurrency; j++ {
frags[j].Clean(b)
}
}
})
}
}
}
}
func BenchmarkImportStandard(b *testing.B) {
for _, cacheType := range []string{CacheTypeRanked} {
for _, numRows := range rowCases {
rowIDs, columnIDs := getZipfRowsSliceStandard(numRows, 1)
b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) {
b.StopTimer()
for i := 0; i < b.N; i++ {
f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType)
b.StartTimer()
err := f.bulkImport(rowIDs, columnIDs, &ImportOptions{})
if err != nil {
b.Errorf("import error: %v", err)
}
b.StopTimer()
f.Clean(b)
}
})
}
}
}
func BenchmarkImportRoaringUpdate(b *testing.B) {
fileSize := make(map[string]int64)
names := []string{}
for _, cacheType := range []string{CacheTypeRanked} {
for _, numRows := range rowCases {
for _, numCols := range colCases {
data := getZipfRowsSliceRoaring(numRows, 1)
updata := getUpdataRoaring(numRows, numCols, 1)
name := fmt.Sprintf("%s%dRows%dCols", cacheType, numRows, numCols)
names = append(names, name)
b.Run(name, func(b *testing.B) {
b.StopTimer()
for i := 0; i < b.N; i++ {
f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType)
err := f.importRoaring(data, false)
if err != nil {
b.Errorf("import error: %v", err)
}
b.StartTimer()
err = f.importRoaring(updata, false)
if err != nil {
f.Clean(b)
b.Errorf("import error: %v", err)
}
b.StopTimer()
stat, _ := f.file.Stat()
fileSize[name] = stat.Size()
f.Clean(b)
}
})
}
}
}
for _, name := range names {
b.Logf("%s: %.2fMB\n", name, float64(fileSize[name])/1024/1024)
}
}
func TestGetZipfRowsSliceRoaring(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType)
data := getZipfRowsSliceRoaring(10, 1)
f.importRoaring(data, false)
if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
t.Fatalf("unexpected rows: %v", f.rows(0))
}
for i := uint64(1); i < 10; i++ {
if f.row(i).Count() >= f.row(i-1).Count() {
t.Fatalf("suspect distribution from getZipfRowsSliceRoaring")
}
}
}
// getZipfRowsSliceRoaring generates a random fragment with the given number of
// rows, and 1 bit set in each column. The row each bit is set in is chosen via
// the Zipf generator, and so will be skewed toward lower row numbers. If this
// is edited to change the data distribution, getZipfRowsSliceStandard should be
// edited as well.
func getZipfRowsSliceRoaring(numRows uint64, seed int64) []byte {
b := roaring.NewBitmap()
s := rand.NewSource(seed)
r := rand.New(s)
z := rand.NewZipf(r, 1.6, 50, numRows-1)
for i := uint64(0); i < ShardWidth; i++ {
row := z.Uint64()
b.DirectAdd(row*ShardWidth + i)
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
}
return buf.Bytes()
}
// getUpdataRoaring gets a byte slice containing a roaring bitmap which
// represents numCols set bits distributed randomly throughout a shard's column
// space and zipfianly throughout numRows rows.
func getUpdataRoaring(numRows, numCols uint64, seed int64) []byte {
b := roaring.NewBitmap()
s := rand.NewSource(seed)
r := rand.New(s)
z := rand.NewZipf(r, 1.6, 50, numRows-1)
for i := uint64(0); i < numCols; i++ {
col := uint64(r.Int63n(ShardWidth)) // assuming the number of repeats will be negligible
row := z.Uint64()
b.DirectAdd(row*ShardWidth + col)
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
}
return buf.Bytes()
}
// getZipfRowsSliceStandard is the same as getZipfRowsSliceRoaring, but returns
// row and column ids instead of a byte slice containing roaring bitmap data.
func getZipfRowsSliceStandard(numRows uint64, seed int64) (rowIDs, columnIDs []uint64) {
s := rand.NewSource(seed)
r := rand.New(s)
z := rand.NewZipf(r, 1.6, 50, numRows-1)
rowIDs, columnIDs = make([]uint64, ShardWidth), make([]uint64, ShardWidth)
for i := uint64(0); i < ShardWidth; i++ {
rowIDs[i] = z.Uint64()
columnIDs[i] = i
}
return rowIDs, columnIDs
}
func BenchmarkFileWrite(b *testing.B) {
for _, numRows := range rowCases {
data := getZipfRowsSliceRoaring(numRows, 1)
b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) {
b.StopTimer()
for i := 0; i < b.N; i++ {
f, err := ioutil.TempFile(TempDir, "")
if err != nil {
b.Fatalf("getting temp file: %v", err)
}
b.StartTimer()
_, err = f.Write(data)
if err != nil {
b.Fatal(err)
}
err = f.Sync()
if err != nil {
b.Fatal(err)
}
err = f.Close()
if err != nil {
b.Fatal(err)
}
b.StopTimer()
os.Remove(f.Name())
}
})
}
}
/////////////////////////////////////////////////////////////////////
func (f *fragment) Clean(t testing.TB) {
errc := f.Close()
errf := os.Remove(f.path)
errp := os.Remove(f.cachePath())
if errc != nil || errf != nil {
t.Fatal("cleaning up fragment: ", errc, errf, errp)
}
// not all fragments have cache files
if errp != nil && !os.IsNotExist(errp) {
t.Fatalf("cleaning up fragment cache: %v", errp)
}
}
// CleanKeep is just like Clean(), but it doesn't remove the
// fragment file (note that it DOES remove the cache file).
func (f *fragment) CleanKeep(t testing.TB) {
errc := f.Close()
errp := os.Remove(f.cachePath())
if errc != nil {
t.Fatal("closing fragment: ", errc, errp)
}
// not all fragments have cache files
if errp != nil && !os.IsNotExist(errp) {
t.Fatalf("cleaning up fragment cache: %v", errp)
}
}
// mustOpenFragment returns a new instance of Fragment with a temporary path.
func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
file, err := ioutil.TempFile(TempDir, "pilosa-fragment-")
if err != nil {
panic(err)
}
@ -1811,7 +2186,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) {
func TestFragment_RowsIteration(t *testing.T) {
t.Run("firstContainer", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
expectedAll := make([]uint64, 0)
expectedOdd := make([]uint64, 0)
@ -1838,7 +2213,7 @@ func TestFragment_RowsIteration(t *testing.T) {
t.Run("secondRow", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
expected := []uint64{1, 2}
if _, err := f.setBit(1, 66000); err != nil {
@ -1862,7 +2237,7 @@ func TestFragment_RowsIteration(t *testing.T) {
t.Run("combinations", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
expectedRows := make([]uint64, 0)
for r := uint64(1); r < uint64(10000); r += 100 {
@ -1909,7 +2284,7 @@ func TestFragment_RoaringImport(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Close()
defer f.Clean(t)
for num, input := range test {
buf := &bytes.Buffer{}
bm := roaring.NewBitmap(input...)
@ -1952,7 +2327,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Close()
defer f.Clean(t)
options := &ImportOptions{}
err := f.bulkImport(test.rowIDs, test.colIDs, options)
@ -2086,6 +2461,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 {
func TestFragmentRowIterator(t *testing.T) {
t.Run("basic", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean(t)
f.mustSetBits(0, 0)
f.mustSetBits(1, 0)
f.mustSetBits(2, 0)
@ -2114,11 +2490,11 @@ func TestFragmentRowIterator(t *testing.T) {
if !wrapped {
t.Fatalf("wrapped should be true after iterator is exhausted")
}
f.Close()
})
t.Run("skipped rows", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean(t)
f.mustSetBits(1, 0)
f.mustSetBits(3, 0)
f.mustSetBits(5, 0)
@ -2147,11 +2523,11 @@ func TestFragmentRowIterator(t *testing.T) {
if !wrapped {
t.Fatalf("wrapped should be true after iterator is exhausted")
}
f.Close()
})
t.Run("basic wrapped", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean(t)
f.mustSetBits(0, 0)
f.mustSetBits(1, 0)
f.mustSetBits(2, 0)
@ -2172,11 +2548,11 @@ func TestFragmentRowIterator(t *testing.T) {
t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
}
}
f.Close()
})
t.Run("skipped rows wrapped", func(t *testing.T) {
f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
defer f.Clean(t)
f.mustSetBits(1, 0)
f.mustSetBits(3, 0)
f.mustSetBits(5, 0)
@ -2197,7 +2573,5 @@ func TestFragmentRowIterator(t *testing.T) {
t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
}
}
f.Close()
})
}

21
gossip/gossip.go
View file

@ -333,15 +333,30 @@ func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver {
}
func (g *eventReceiver) NotifyJoin(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: n}
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}
}
func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: n}
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}
}
func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: n}
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}
}
func (g *eventReceiver) listen() {

10
handler.go
View file

@ -117,3 +117,13 @@ type BlockDataResponse struct {
RowIDs []uint64
ColumnIDs []uint64
}
type TranslateKeysRequest struct {
Index string
Field string
Keys []string
}
type TranslateKeysResponse struct {
IDs []uint64
}

5
holder.go
View file

@ -43,7 +43,7 @@ const (
fileLimit = 262144 // (512^2)
// existenceFieldName is the name of the internal field used to store existence values.
existenceFieldName = "exists"
existenceFieldName = "_exists"
)
// Holder represents a container for indexes.
@ -289,6 +289,9 @@ func (h *Holder) limitedSchema() []*IndexInfo {
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name(), Options: index.Options()}
for _, field := range index.Fields() {
if strings.HasPrefix(field.name, "_") {
continue
}
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
di.Fields = append(di.Fields, fi)
}

11
holder_test.go
View file

@ -391,27 +391,26 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
// Set data on the local holder.
// Set data on the local holder for node0.
t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC)
t2 := time.Date(2018, 8, 2, 12, 30, 0, 0, time.UTC)
hldr0.SetBitTime("i", "f", 0, 1, &t1)
hldr0.SetBitTime("i", "f", 0, 2, &t2)
// Set data on node1.
hldr1.SetBitTime("i", "f", 0, 22, &t2)
err = c[0].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 0: %v", err)
}
err = c[1].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 1: %v", err)
}
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a := hldr.RowTime("i", "f", 0, t1, quantum).Columns(); !reflect.DeepEqual(a, []uint64{1}) {
t.Errorf("unexpected columns(%d/0): %+v", i, a)
}
if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2, 22}) {
t.Errorf("unexpected columns(%d/0): %+v", i, a)
}
}

33
http/client.go
View file

@ -164,7 +164,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
}
return err
}
return nil
return errors.Wrap(resp.Body.Close(), "closing response body")
}
// FragmentNodes returns a list of nodes that own a shard.
@ -705,24 +705,22 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i
return nil
}
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
// RetrieveShardFromURI returns a ReadCloser which contains the data of the
// specified shard from the specified node. Caller *must* close the returned
// ReadCloser or risk leaking goroutines/tcp connections.
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI")
defer span.Finish()
node := &pilosa.Node{
URI: uri,
}
return c.backupShardNode(ctx, index, field, shard, node)
}
func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.backupShardNode")
defer span.Finish()
u := nodePathToURL(node, "/fragment/data")
u := nodePathToURL(node, "/internal/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"field": {field},
"view": {view},
"shard": {strconv.FormatUint(shard, 10)},
}.Encode()
@ -805,7 +803,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
return err
}
return nil
return errors.Wrap(resp.Body.Close(), "closing response body")
}
// FragmentBlocks returns a list of block checksums for a fragment on a host.
@ -999,15 +997,24 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
req.Header.Set("Accept", "application/json")
// Execute request.
_, err = c.executeRequest(req.WithContext(ctx))
return err
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
return errors.Wrap(resp.Body.Close(), "closing response body")
}
// executeRequest executes the given request and checks the Response
// executeRequest executes the given request and checks the Response. For
// responses with non-2XX status, the body is read and closed, and an error is
// returned. If the error is nil, the caller must ensure that the response body
// is closed.
func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) {
tracing.GlobalTracer.InjectHTTPHeaders(req)
resp, err := c.httpClient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
return nil, errors.Wrap(err, "executing request")
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {

10
http/client_test.go
View file

@ -710,9 +710,9 @@ func TestClient_ImportKeys(t *testing.T) {
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
}
// Verify Range.
// Verify range.
queryRequest := &pilosa.QueryRequest{
Query: fmt.Sprintf(`Range(%s>10)`, fldName),
Query: fmt.Sprintf(`Row(%s>10)`, fldName),
Remote: false,
}
@ -743,7 +743,7 @@ func TestClient_ImportKeys(t *testing.T) {
// Verify Range.
queryRequest = &pilosa.QueryRequest{
Query: fmt.Sprintf(`Range(%s>10)`, fldName),
Query: fmt.Sprintf(`Row(%s>10)`, fldName),
Remote: false,
}
@ -900,7 +900,7 @@ func TestClient_ImportExistence(t *testing.T) {
}
// Verify existence.
if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) {
if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) {
t.Fatalf("unexpected existence columns: %+v", a)
}
})
@ -935,7 +935,7 @@ func TestClient_ImportExistence(t *testing.T) {
}
// Verify existence.
if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
t.Fatalf("unexpected existence columns: %+v", a)
}
})

73
http/handler.go
View file

@ -24,8 +24,8 @@ import (
"io/ioutil"
"net"
"net/http"
_ "net/http/pprof"
"net/url" // Imported for its side-effect of registering pprof endpoints with the server.
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"net/url"
"reflect"
"runtime/debug"
"strconv"
@ -192,12 +192,14 @@ func (h *Handler) populateValidators() {
h.validators["PostClusterMessage"] = queryValidationSpecRequired()
h.validators["GetFragmentBlockData"] = queryValidationSpecRequired()
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired()
h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
h.validators["GetNodes"] = queryValidationSpecRequired()
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["GetTranslateData"] = queryValidationSpecRequired("offset")
h.validators["PostTranslateKeys"] = queryValidationSpecRequired()
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -261,6 +263,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage")
router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData")
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
@ -268,6 +271,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys")
router.Use(handler.queryArgValidator)
router.Use(handler.extractTracing)
@ -1212,6 +1216,27 @@ type getFragmentBlocksResponse struct {
Blocks []pilosa.FragmentBlock `json:"blocks"`
}
// handleGetFragmentData handles GET /internal/fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read shard parameter.
q := r.URL.Query()
shard, err := strconv.ParseUint(q.Get("shard"), 10, 64)
if err != nil {
http.Error(w, "shard required", http.StatusBadRequest)
return
}
// Retrieve fragment data from holder.
f, err := h.api.FragmentData(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// Stream fragment to response body.
if _, err := f.WriteTo(w); err != nil {
h.logger.Printf("error streaming fragment data: %s", err)
}
}
// handleGetVersion handles /version requests.
func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -1401,7 +1426,11 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques
type defaultClusterMessageResponse struct{}
// translateStoreBufferSize is the buffer size used for streaming data.
const translateStoreBufferSize = 65536
const translateStoreBufferSize = 1 << 16 // 64k
// translateStoreBufferSizeMax is the maximum size that the buffer is allowed
// to grow before raising an error.
const translateStoreBufferSizeMax = 1 << 22 // 4Mb
func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
@ -1424,16 +1453,30 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
}
// Copy from reader to client until store or client disconnect.
buf := make([]byte, translateStoreBufferSize)
useBufferSize := translateStoreBufferSize
buf := make([]byte, useBufferSize)
for {
// Read from store.
n, err := rdr.Read(buf)
if err == io.EOF {
return
} else if err == pilosa.ErrTranslateReadTargetUndersized {
// Increase the buffer size and try to read again.
useBufferSize *= 2
// Prevent the buffer from growing without bound.
if useBufferSize > translateStoreBufferSizeMax {
h.logger.Printf("http: translate store buffer exceeded max size: %s", err)
return
}
buf = make([]byte, useBufferSize)
continue
} else if err != nil {
h.logger.Printf("http: translate store read error: %s", err)
return
} else if n == 0 {
// Reset the default buffer size.
useBufferSize = translateStoreBufferSize
buf = make([]byte, useBufferSize)
continue
}
@ -1570,3 +1613,25 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
h.logger.Printf("writing import-roaring response: %v", err)
}
}
func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
return
}
buf, err := h.api.TranslateKeys(r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError)
}
// Write response.
_, err = w.Write(buf)
if err != nil {
h.logger.Printf("writing translate keys response: %v", err)
}
}

3
http/handler_internal_test.go
View file

@ -57,7 +57,6 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json)
}
}
}
}
@ -93,7 +92,6 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) {
t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual)
}
}
}
}
@ -178,6 +176,5 @@ func TestFieldOptionValidation(t *testing.T) {
t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual)
}
}
}
}

58
internal/clustertests/docker-compose-replication2.yml Normal file
View file

@ -0,0 +1,58 @@
version: '2'
services:
pilosa1:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
ports:
- "33455:10101"
environment:
- PILOSA_CLUSTER_COORDINATOR=true
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
- PILOSA_CLUSTER_REPLICAS=2
networks:
- pilosanet
command:
- "/pilosa server --bind pilosa1:10101"
pilosa2:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
ports:
- "33456:10101"
environment:
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
- PILOSA_CLUSTER_REPLICAS=2
networks:
- pilosanet
command:
- "/pilosa server --bind pilosa2:10101"
pilosa3:
build:
context: ../..
dockerfile: Dockerfile-clustertests
image: ptest
ports:
- "33457:10101"
environment:
- PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000
- PILOSA_CLUSTER_REPLICAS=2
networks:
- pilosanet
command:
- "/pilosa server --bind pilosa3:10101"
client1:
build:
context: .
environment:
- ENABLE_PILOSA_CLUSTER_TESTS=1
networks:
- pilosanet
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
- "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
networks:
pilosanet:

313
internal/private.pb.go
View file

@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} }
func (m *IndexMeta) String() string { return proto.CompactTextString(m) }
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{0}
return fileDescriptor_private_8095a89af06a70de, []int{0}
}
func (m *IndexMeta) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -93,7 +93,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} }
func (m *FieldOptions) String() string { return proto.CompactTextString(m) }
func (*FieldOptions) ProtoMessage() {}
func (*FieldOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{1}
return fileDescriptor_private_8095a89af06a70de, []int{1}
}
func (m *FieldOptions) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -189,7 +189,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} }
func (m *ImportResponse) String() string { return proto.CompactTextString(m) }
func (*ImportResponse) ProtoMessage() {}
func (*ImportResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{2}
return fileDescriptor_private_8095a89af06a70de, []int{2}
}
func (m *ImportResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -240,7 +240,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} }
func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) }
func (*BlockDataRequest) ProtoMessage() {}
func (*BlockDataRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{3}
return fileDescriptor_private_8095a89af06a70de, []int{3}
}
func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -316,7 +316,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} }
func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) }
func (*BlockDataResponse) ProtoMessage() {}
func (*BlockDataResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{4}
return fileDescriptor_private_8095a89af06a70de, []int{4}
}
func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -370,7 +370,7 @@ func (m *Cache) Reset() { *m = Cache{} }
func (m *Cache) String() string { return proto.CompactTextString(m) }
func (*Cache) ProtoMessage() {}
func (*Cache) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{5}
return fileDescriptor_private_8095a89af06a70de, []int{5}
}
func (m *Cache) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -417,7 +417,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} }
func (m *MaxShards) String() string { return proto.CompactTextString(m) }
func (*MaxShards) ProtoMessage() {}
func (*MaxShards) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{6}
return fileDescriptor_private_8095a89af06a70de, []int{6}
}
func (m *MaxShards) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -466,7 +466,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} }
func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) }
func (*CreateShardMessage) ProtoMessage() {}
func (*CreateShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{7}
return fileDescriptor_private_8095a89af06a70de, []int{7}
}
func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -527,7 +527,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} }
func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexMessage) ProtoMessage() {}
func (*DeleteIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{8}
return fileDescriptor_private_8095a89af06a70de, []int{8}
}
func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -575,7 +575,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} }
func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) }
func (*CreateIndexMessage) ProtoMessage() {}
func (*CreateIndexMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{9}
return fileDescriptor_private_8095a89af06a70de, []int{9}
}
func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -631,7 +631,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} }
func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) }
func (*CreateFieldMessage) ProtoMessage() {}
func (*CreateFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{10}
return fileDescriptor_private_8095a89af06a70de, []int{10}
}
func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -693,7 +693,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} }
func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteFieldMessage) ProtoMessage() {}
func (*DeleteFieldMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{11}
return fileDescriptor_private_8095a89af06a70de, []int{11}
}
func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -749,7 +749,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar
func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteAvailableShardMessage) ProtoMessage() {}
func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{12}
return fileDescriptor_private_8095a89af06a70de, []int{12}
}
func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -812,7 +812,7 @@ func (m *Field) Reset() { *m = Field{} }
func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (*Field) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{13}
return fileDescriptor_private_8095a89af06a70de, []int{13}
}
func (m *Field) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -873,7 +873,7 @@ func (m *Schema) Reset() { *m = Schema{} }
func (m *Schema) String() string { return proto.CompactTextString(m) }
func (*Schema) ProtoMessage() {}
func (*Schema) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{14}
return fileDescriptor_private_8095a89af06a70de, []int{14}
}
func (m *Schema) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -921,7 +921,7 @@ func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{15}
return fileDescriptor_private_8095a89af06a70de, []int{15}
}
func (m *Index) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -977,7 +977,7 @@ func (m *URI) Reset() { *m = URI{} }
func (m *URI) String() string { return proto.CompactTextString(m) }
func (*URI) ProtoMessage() {}
func (*URI) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{16}
return fileDescriptor_private_8095a89af06a70de, []int{16}
}
func (m *URI) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1041,7 +1041,7 @@ func (m *Node) Reset() { *m = Node{} }
func (m *Node) String() string { return proto.CompactTextString(m) }
func (*Node) ProtoMessage() {}
func (*Node) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{17}
return fileDescriptor_private_8095a89af06a70de, []int{17}
}
func (m *Node) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1110,7 +1110,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} }
func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) }
func (*NodeStateMessage) ProtoMessage() {}
func (*NodeStateMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{18}
return fileDescriptor_private_8095a89af06a70de, []int{18}
}
func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1165,7 +1165,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} }
func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) }
func (*NodeEventMessage) ProtoMessage() {}
func (*NodeEventMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{19}
return fileDescriptor_private_8095a89af06a70de, []int{19}
}
func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1221,7 +1221,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} }
func (m *NodeStatus) String() string { return proto.CompactTextString(m) }
func (*NodeStatus) ProtoMessage() {}
func (*NodeStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{20}
return fileDescriptor_private_8095a89af06a70de, []int{20}
}
func (m *NodeStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1283,7 +1283,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} }
func (m *IndexStatus) String() string { return proto.CompactTextString(m) }
func (*IndexStatus) ProtoMessage() {}
func (*IndexStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{21}
return fileDescriptor_private_8095a89af06a70de, []int{21}
}
func (m *IndexStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1338,7 +1338,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} }
func (m *FieldStatus) String() string { return proto.CompactTextString(m) }
func (*FieldStatus) ProtoMessage() {}
func (*FieldStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{22}
return fileDescriptor_private_8095a89af06a70de, []int{22}
}
func (m *FieldStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1394,7 +1394,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} }
func (m *ClusterStatus) String() string { return proto.CompactTextString(m) }
func (*ClusterStatus) ProtoMessage() {}
func (*ClusterStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{23}
return fileDescriptor_private_8095a89af06a70de, []int{23}
}
func (m *ClusterStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1458,7 +1458,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} }
func (m *BSIGroup) String() string { return proto.CompactTextString(m) }
func (*BSIGroup) ProtoMessage() {}
func (*BSIGroup) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{24}
return fileDescriptor_private_8095a89af06a70de, []int{24}
}
func (m *BSIGroup) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1528,7 +1528,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} }
func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) }
func (*CreateViewMessage) ProtoMessage() {}
func (*CreateViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{25}
return fileDescriptor_private_8095a89af06a70de, []int{25}
}
func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1591,7 +1591,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} }
func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteViewMessage) ProtoMessage() {}
func (*DeleteViewMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{26}
return fileDescriptor_private_8095a89af06a70de, []int{26}
}
func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1646,7 +1646,7 @@ type ResizeInstruction struct {
Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"`
Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"`
Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"`
Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"`
NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"`
ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
@ -1657,7 +1657,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} }
func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) }
func (*ResizeInstruction) ProtoMessage() {}
func (*ResizeInstruction) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{27}
return fileDescriptor_private_8095a89af06a70de, []int{27}
}
func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1714,9 +1714,9 @@ func (m *ResizeInstruction) GetSources() []*ResizeSource {
return nil
}
func (m *ResizeInstruction) GetSchema() *Schema {
func (m *ResizeInstruction) GetNodeStatus() *NodeStatus {
if m != nil {
return m.Schema
return m.NodeStatus
}
return nil
}
@ -1743,7 +1743,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} }
func (m *ResizeSource) String() string { return proto.CompactTextString(m) }
func (*ResizeSource) ProtoMessage() {}
func (*ResizeSource) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{28}
return fileDescriptor_private_8095a89af06a70de, []int{28}
}
func (m *ResizeSource) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1820,7 +1820,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp
func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) }
func (*ResizeInstructionComplete) ProtoMessage() {}
func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{29}
return fileDescriptor_private_8095a89af06a70de, []int{29}
}
func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1881,7 +1881,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} }
func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*SetCoordinatorMessage) ProtoMessage() {}
func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{30}
return fileDescriptor_private_8095a89af06a70de, []int{30}
}
func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1928,7 +1928,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa
func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) }
func (*UpdateCoordinatorMessage) ProtoMessage() {}
func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{31}
return fileDescriptor_private_8095a89af06a70de, []int{31}
}
func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -1976,7 +1976,7 @@ func (m *Topology) Reset() { *m = Topology{} }
func (m *Topology) String() string { return proto.CompactTextString(m) }
func (*Topology) ProtoMessage() {}
func (*Topology) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{32}
return fileDescriptor_private_8095a89af06a70de, []int{32}
}
func (m *Topology) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -2029,7 +2029,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} }
func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) }
func (*RecalculateCaches) ProtoMessage() {}
func (*RecalculateCaches) Descriptor() ([]byte, []int) {
return fileDescriptor_private_08d4c0c27f7a355f, []int{33}
return fileDescriptor_private_8095a89af06a70de, []int{33}
}
func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -3270,21 +3270,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if m.Schema != nil {
dAtA[i] = 0x2a
if m.ClusterStatus != nil {
dAtA[i] = 0x32
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size()))
n18, err := m.Schema.MarshalTo(dAtA[i:])
i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size()))
n18, err := m.ClusterStatus.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n18
}
if m.ClusterStatus != nil {
dAtA[i] = 0x32
if m.NodeStatus != nil {
dAtA[i] = 0x3a
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size()))
n19, err := m.ClusterStatus.MarshalTo(dAtA[i:])
i = encodeVarintPrivate(dAtA, i, uint64(m.NodeStatus.Size()))
n19, err := m.NodeStatus.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
@ -4169,14 +4169,14 @@ func (m *ResizeInstruction) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.Schema != nil {
l = m.Schema.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.ClusterStatus != nil {
l = m.ClusterStatus.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.NodeStatus != nil {
l = m.NodeStatus.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -8034,39 +8034,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Schema == nil {
m.Schema = &Schema{}
}
if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 6:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ClusterStatus", wireType)
@ -8100,6 +8067,39 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field NodeStatus", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.NodeStatus == nil {
m.NodeStatus = &NodeStatus{}
}
if err := m.NodeStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8877,79 +8877,80 @@ var (
ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_08d4c0c27f7a355f) }
func init() { proto.RegisterFile("private.proto", fileDescriptor_private_8095a89af06a70de) }
var fileDescriptor_private_08d4c0c27f7a355f = []byte{
// 1131 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5,
0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51,
0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c,
0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7,
0x20, 0xf1, 0x02, 0x88, 0x27, 0xe2, 0x92, 0x47, 0xa8, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xb3, 0xbb,
0x76, 0x1c, 0x12, 0x05, 0xee, 0xe6, 0xfc, 0xce, 0x99, 0xf3, 0x7d, 0xce, 0xec, 0x42, 0x77, 0x9c,
0xc5, 0x27, 0x4c, 0xf2, 0x8d, 0x71, 0x26, 0xa4, 0x08, 0x5a, 0x71, 0x2a, 0x79, 0x96, 0xb2, 0x84,
0x3c, 0x87, 0x76, 0x94, 0x0e, 0xf9, 0xd9, 0x0e, 0x97, 0x2c, 0x08, 0xc0, 0x7f, 0xc1, 0x27, 0x79,
0xe8, 0xad, 0x39, 0xeb, 0x2d, 0x8a, 0xe7, 0xe0, 0x03, 0x58, 0xdc, 0xcb, 0xd8, 0xe0, 0x78, 0xfb,
0x2c, 0xce, 0x25, 0x4f, 0x07, 0x3c, 0xf4, 0x91, 0x3b, 0x83, 0x92, 0x37, 0x0e, 0xdc, 0xfa, 0x2a,
0xe6, 0xc9, 0xf0, 0xbb, 0xb1, 0x8c, 0x45, 0x9a, 0x07, 0xef, 0x42, 0x7b, 0x8b, 0x0d, 0x8e, 0xf8,
0xde, 0x64, 0xcc, 0x51, 0x63, 0x9b, 0x56, 0x40, 0xc9, 0xed, 0xc7, 0xaf, 0xb5, 0xc6, 0x2e, 0xad,
0x80, 0x60, 0x0d, 0x3a, 0x7b, 0xf1, 0x88, 0x7f, 0x5f, 0xb0, 0x54, 0x16, 0xa3, 0xb0, 0x81, 0xb7,
0xeb, 0x90, 0x72, 0x15, 0x15, 0xb7, 0x90, 0x85, 0xe7, 0x60, 0x09, 0xbc, 0x9d, 0x38, 0x0d, 0xdb,
0x6b, 0xce, 0xba, 0x47, 0xd5, 0x11, 0x11, 0x76, 0x16, 0x82, 0x41, 0xd8, 0x59, 0x19, 0x62, 0x67,
0x3a, 0xc4, 0x5d, 0xd1, 0x97, 0x2c, 0x1d, 0xb2, 0x6c, 0x78, 0x10, 0xf3, 0xd3, 0xf0, 0x96, 0x0e,
0x71, 0x1a, 0x25, 0x04, 0x16, 0xa3, 0xd1, 0x58, 0x64, 0x92, 0xf2, 0x7c, 0x2c, 0xd2, 0x1c, 0x2d,
0x6e, 0x67, 0x59, 0xe8, 0xa0, 0x13, 0xea, 0x48, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0x0c, 0x8e, 0x7b,
0x4c, 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0xff, 0x43, 0x03, 0x73, 0x6c, 0xe4, 0x34, 0xa1,
0x50, 0xcc, 0x57, 0xe8, 0x6a, 0x14, 0x09, 0x85, 0xe2, 0x7d, 0xcc, 0x98, 0x4f, 0x35, 0xa1, 0xd0,
0xfe, 0x11, 0xcb, 0x86, 0x98, 0x29, 0x9f, 0x6a, 0x42, 0xc5, 0x82, 0xde, 0xea, 0xf4, 0xe0, 0x99,
0x44, 0xb0, 0x5c, 0xb3, 0x6f, 0xdc, 0x5c, 0x81, 0x26, 0x15, 0xa7, 0x51, 0x2f, 0x0f, 0x9d, 0x35,
0x6f, 0xdd, 0xa7, 0x86, 0xc2, 0x22, 0x88, 0xa4, 0x18, 0xa5, 0x8a, 0xe5, 0x22, 0xab, 0x02, 0xc8,
0x3d, 0x68, 0x60, 0x45, 0x54, 0x94, 0xd5, 0x5d, 0x75, 0x24, 0x3f, 0x3b, 0xd0, 0xde, 0x61, 0x67,
0xe8, 0x46, 0x1e, 0x3c, 0x85, 0x96, 0xcd, 0x13, 0x0a, 0x75, 0x1e, 0xbf, 0xbf, 0x61, 0x1b, 0x6c,
0xa3, 0x14, 0xdb, 0xb0, 0x32, 0xdb, 0xa9, 0xcc, 0x26, 0xb4, 0xbc, 0xf2, 0xf6, 0xe7, 0xd0, 0x9d,
0x62, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c,
0x73, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x03, 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47,
0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x79, 0xc6, 0x75, 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a,
0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, 0x37, 0xd3, 0xf1, 0x0f, 0x7a, 0x49,
0xdf, 0xfa, 0x70, 0xb5, 0x6c, 0xf0, 0x00, 0x7c, 0x35, 0x6a, 0xe8, 0x42, 0xe7, 0xf1, 0x9d, 0x2a,
0x4f, 0xe5, 0x14, 0x52, 0x14, 0x20, 0x89, 0x55, 0x8a, 0xfe, 0x5c, 0x19, 0xd8, 0x9c, 0x56, 0x7a,
0x68, 0x4c, 0x79, 0x68, 0x6a, 0xa5, 0x32, 0x55, 0x1f, 0x53, 0x63, 0xed, 0x99, 0x0d, 0xf7, 0xa6,
0xd6, 0xc8, 0x00, 0xde, 0xd1, 0x1a, 0xbe, 0x3c, 0x61, 0x71, 0xc2, 0x0e, 0x93, 0x6b, 0x56, 0x64,
0x8e, 0xe3, 0x21, 0x2c, 0xe0, 0xdd, 0xa8, 0x67, 0xa6, 0xc0, 0x92, 0xe4, 0x07, 0x23, 0xaf, 0x5a,
0x7f, 0x97, 0x8d, 0xb8, 0xd1, 0x86, 0xe7, 0x32, 0x5e, 0xf7, 0xea, 0x78, 0x95, 0x61, 0x35, 0x2e,
0x6a, 0xd5, 0x79, 0xca, 0x30, 0x12, 0xe4, 0x09, 0x34, 0xfb, 0x83, 0x23, 0x3e, 0x62, 0xc1, 0x87,
0xb0, 0x80, 0x1e, 0xf2, 0xdc, 0x74, 0xf4, 0xed, 0x99, 0x4a, 0x51, 0xcb, 0x27, 0x3d, 0x13, 0xd9,
0x5c, 0x9f, 0x1e, 0x40, 0x13, 0xad, 0xe7, 0xa1, 0x3f, 0xab, 0x06, 0x71, 0x6a, 0xd8, 0x64, 0x1b,
0xbc, 0x7d, 0x1a, 0xa9, 0x49, 0x45, 0x0f, 0xac, 0x16, 0x43, 0x29, 0xdd, 0x5f, 0x8b, 0x5c, 0x9a,
0x3c, 0xe1, 0x59, 0x61, 0x2f, 0x45, 0x26, 0x31, 0x47, 0x5d, 0x8a, 0x67, 0x92, 0x83, 0xbf, 0x2b,
0x86, 0x3c, 0x58, 0x04, 0x37, 0xea, 0x19, 0x1d, 0x6e, 0xd4, 0x0b, 0xde, 0x43, 0xf5, 0x26, 0x35,
0xdd, 0xca, 0x89, 0x7d, 0x1a, 0x51, 0x34, 0x7c, 0x1f, 0xba, 0x51, 0xbe, 0x25, 0x44, 0x36, 0x8c,
0x53, 0x26, 0x45, 0x66, 0xde, 0x80, 0x69, 0x10, 0x27, 0x48, 0x32, 0xa9, 0x37, 0x76, 0x9b, 0x6a,
0x82, 0x3c, 0x83, 0x25, 0x65, 0x14, 0x09, 0x5b, 0xef, 0x15, 0x68, 0x2a, 0xac, 0x74, 0xc2, 0x50,
0x95, 0x06, 0xb7, 0xae, 0xe1, 0x5b, 0xad, 0x61, 0xfb, 0x84, 0xa7, 0xb2, 0xd6, 0x31, 0x48, 0xa3,
0x82, 0x2e, 0xd5, 0x44, 0x40, 0x74, 0x80, 0x26, 0x92, 0xc5, 0x2a, 0x12, 0x85, 0x52, 0xe4, 0x91,
0x5f, 0x1d, 0x00, 0xeb, 0x50, 0x91, 0x97, 0x57, 0x9c, 0xcb, 0xaf, 0x04, 0xeb, 0xb6, 0xf2, 0x66,
0x5a, 0x96, 0x2a, 0x29, 0x8d, 0x53, 0xdb, 0x19, 0x1f, 0x57, 0x9d, 0xa1, 0x4b, 0x7a, 0x77, 0xa6,
0x33, 0xb4, 0xd5, 0xaa, 0x3f, 0x5e, 0x42, 0xa7, 0x86, 0xcf, 0xed, 0x92, 0x8f, 0xca, 0x2e, 0x71,
0x67, 0x55, 0x22, 0x6e, 0x54, 0xda, 0x5e, 0x79, 0x01, 0x9d, 0x1a, 0x3c, 0x57, 0xe3, 0x3a, 0xdc,
0x9e, 0x9e, 0x43, 0xbb, 0xdf, 0x67, 0x61, 0x12, 0x43, 0x77, 0x2b, 0x29, 0x72, 0xc9, 0x33, 0xa3,
0x4e, 0x3d, 0x0a, 0x1a, 0x28, 0x8b, 0x57, 0x01, 0xf3, 0xeb, 0x17, 0xdc, 0x87, 0x86, 0x4a, 0xa3,
0x1e, 0xa7, 0x8b, 0x39, 0xd6, 0x4c, 0x72, 0x00, 0xad, 0xcd, 0x7e, 0xf4, 0x3c, 0x13, 0xc5, 0x78,
0xae, 0xd3, 0xf6, 0x4d, 0x77, 0x2f, 0xbe, 0xe9, 0xde, 0x85, 0x37, 0xdd, 0x2f, 0xdf, 0x74, 0xd2,
0x87, 0x65, 0xbd, 0x2a, 0xd5, 0x14, 0xdf, 0x64, 0xe1, 0xd8, 0x87, 0xd4, 0xab, 0x3d, 0xa4, 0x7d,
0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x54, 0xfa, 0xbb, 0x0b, 0xcb, 0x94, 0xe7, 0xf1, 0x6b, 0x1e, 0xa5,
0xb9, 0xcc, 0x8a, 0x81, 0xda, 0x49, 0xea, 0xfe, 0x37, 0xe2, 0xd0, 0x64, 0xdb, 0xa3, 0x9a, 0xb8,
0x4e, 0xa7, 0x07, 0x8f, 0xa0, 0x33, 0x3b, 0xb3, 0x17, 0x45, 0xeb, 0x22, 0xc1, 0x23, 0x58, 0xe8,
0x8b, 0x22, 0x1b, 0x94, 0xed, 0x5b, 0xdb, 0x93, 0xda, 0x33, 0xcd, 0xa6, 0x56, 0xac, 0x36, 0x1a,
0x8d, 0x2b, 0x46, 0xe3, 0xe9, 0x4c, 0x2b, 0x85, 0x4d, 0xbc, 0xf0, 0x56, 0x75, 0x61, 0x8a, 0x4d,
0xa7, 0xa5, 0xc9, 0x2f, 0x0e, 0xdc, 0xaa, 0xbb, 0x70, 0xad, 0xc1, 0x2d, 0x2b, 0xe2, 0xce, 0xad,
0x88, 0x37, 0xaf, 0x22, 0x7e, 0x55, 0x91, 0xea, 0x9b, 0xa0, 0x51, 0xfb, 0x26, 0x20, 0xc7, 0x70,
0xef, 0x42, 0x99, 0xb6, 0xc4, 0x68, 0xac, 0xfa, 0xe1, 0x5f, 0x94, 0x4b, 0xad, 0xb4, 0x2c, 0x33,
0x85, 0x6a, 0x53, 0x4d, 0x90, 0x4f, 0xe1, 0x6e, 0x9f, 0xcb, 0x5a, 0x91, 0x6c, 0xb7, 0xad, 0x81,
0xb7, 0xcb, 0x4f, 0x2f, 0x09, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xee, 0x8f, 0x87, 0x4c, 0xf2, 0x1b,
0xdd, 0xde, 0x84, 0xd6, 0x9e, 0x18, 0x8b, 0x44, 0xbc, 0x9a, 0x5c, 0x31, 0xf5, 0x21, 0x2c, 0xe8,
0xfd, 0xad, 0xd7, 0x48, 0x9b, 0x5a, 0x92, 0xdc, 0x51, 0x0d, 0x3d, 0x60, 0xc9, 0xa0, 0x48, 0x94,
0x1b, 0xea, 0x7b, 0x31, 0xdf, 0x5c, 0xfa, 0xe3, 0x7c, 0xd5, 0xf9, 0xf3, 0x7c, 0xd5, 0x79, 0x73,
0xbe, 0xea, 0xfc, 0xf6, 0xd7, 0xea, 0xff, 0x0e, 0x9b, 0xf8, 0xdf, 0xf1, 0xe4, 0xef, 0x00, 0x00,
0x00, 0xff, 0xff, 0x61, 0x80, 0xe4, 0xef, 0x88, 0x0c, 0x00, 0x00,
var fileDescriptor_private_8095a89af06a70de = []byte{
// 1139 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0xc5,
0x1b, 0xff, 0xef, 0x21, 0x8e, 0xfd, 0x39, 0x4e, 0x93, 0x6d, 0x9b, 0xff, 0x16, 0x50, 0x08, 0xa3,
0x8a, 0x86, 0x4a, 0x84, 0xaa, 0xe5, 0x82, 0x53, 0xa5, 0x92, 0x38, 0x94, 0xa5, 0x24, 0x94, 0x71,
0x92, 0x3b, 0x2e, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0x63, 0x76, 0x67, 0x93, 0xb8, 0x17, 0xdc,
0x82, 0xc4, 0x0b, 0xf0, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x42, 0x15, 0x5e, 0x04, 0xcd, 0x37, 0x33,
0xbb, 0x6b, 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xbe, 0xdf, 0x77, 0x3e, 0xae, 0x0d, 0x9d, 0x51, 0x16,
0x9f, 0x30, 0xc9, 0x37, 0x46, 0x99, 0x90, 0x22, 0x68, 0xc6, 0xa9, 0xe4, 0x59, 0xca, 0x12, 0xf2,
0x1c, 0x5a, 0x51, 0x3a, 0xe0, 0x67, 0x3b, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x05, 0x1f, 0xe7, 0xa1,
0xb7, 0xe6, 0xac, 0x37, 0x29, 0xbe, 0x83, 0xf7, 0x61, 0x71, 0x2f, 0x63, 0xfd, 0xe3, 0xed, 0xb3,
0x38, 0x97, 0x3c, 0xed, 0xf3, 0xd0, 0x47, 0xee, 0x14, 0x4a, 0xde, 0x38, 0xb0, 0xf0, 0x55, 0xcc,
0x93, 0xc1, 0x77, 0x23, 0x19, 0x8b, 0x34, 0x0f, 0xde, 0x81, 0xd6, 0x16, 0xeb, 0x1f, 0xf1, 0xbd,
0xf1, 0x88, 0xa3, 0xc5, 0x16, 0xad, 0x80, 0x92, 0xdb, 0x8b, 0x5f, 0x6b, 0x8b, 0x1d, 0x5a, 0x01,
0xc1, 0x1a, 0xb4, 0xf7, 0xe2, 0x21, 0xff, 0xbe, 0x60, 0xa9, 0x2c, 0x86, 0xe1, 0x1c, 0x6a, 0xd7,
0x21, 0x15, 0x2a, 0x1a, 0x6e, 0x22, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0x3b, 0x71, 0x1a, 0xb6, 0xd6,
0x9c, 0x75, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x2c, 0x04, 0x83, 0xb0, 0xb3, 0x32, 0xc5, 0xf6, 0x64,
0x8a, 0xbb, 0xa2, 0x27, 0x59, 0x3a, 0x60, 0xd9, 0xe0, 0x20, 0xe6, 0xa7, 0xe1, 0x82, 0x4e, 0x71,
0x12, 0x25, 0x04, 0x16, 0xa3, 0xe1, 0x48, 0x64, 0x92, 0xf2, 0x7c, 0x24, 0xd2, 0x1c, 0x3d, 0x6e,
0x67, 0x59, 0xe8, 0x60, 0x10, 0xea, 0x49, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0xf4, 0x8f, 0xbb, 0x4c,
0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0x3b, 0x30, 0x87, 0x35, 0x36, 0x72, 0x9a, 0x50, 0x28,
0xd6, 0x2b, 0x74, 0x35, 0x8a, 0x84, 0x42, 0x51, 0x1f, 0x2b, 0xe6, 0x53, 0x4d, 0x28, 0xb4, 0x77,
0xc4, 0xb2, 0x01, 0x56, 0xca, 0xa7, 0x9a, 0x50, 0xb9, 0x60, 0xb4, 0xba, 0x3c, 0xf8, 0x26, 0x11,
0x2c, 0xd7, 0xfc, 0x9b, 0x30, 0x57, 0xa0, 0x41, 0xc5, 0x69, 0xd4, 0xcd, 0x43, 0x67, 0xcd, 0x5b,
0xf7, 0xa9, 0xa1, 0xb0, 0x09, 0x22, 0x29, 0x86, 0xa9, 0x62, 0xb9, 0xc8, 0xaa, 0x00, 0x72, 0x0f,
0xe6, 0xb0, 0x23, 0x2a, 0xcb, 0x4a, 0x57, 0x3d, 0xc9, 0xcf, 0x0e, 0xb4, 0x76, 0xd8, 0x19, 0x86,
0x91, 0x07, 0x4f, 0xa1, 0x69, 0xeb, 0x84, 0x42, 0xed, 0xc7, 0xef, 0x6d, 0xd8, 0x01, 0xdb, 0x28,
0xc5, 0x36, 0xac, 0xcc, 0x76, 0x2a, 0xb3, 0x31, 0x2d, 0x55, 0xde, 0xfa, 0x1c, 0x3a, 0x13, 0x2c,
0xe5, 0xef, 0x98, 0x8f, 0x6d, 0x55, 0x8f, 0xf9, 0x58, 0xe5, 0x7f, 0xc2, 0x92, 0x82, 0x63, 0xad,
0x7c, 0xaa, 0x89, 0xcf, 0xdc, 0x4f, 0x1c, 0x72, 0x00, 0xc1, 0x56, 0xc6, 0x99, 0xe4, 0xe8, 0x64,
0x87, 0xe7, 0x39, 0x7b, 0xc5, 0x2f, 0xaf, 0xb8, 0xae, 0xa2, 0x5b, 0xaf, 0x62, 0xd9, 0x07, 0xaf,
0xd6, 0x07, 0xf2, 0x10, 0x82, 0x2e, 0x4f, 0xb8, 0xe4, 0x66, 0x3b, 0xfe, 0xc1, 0x2e, 0xe9, 0xd9,
0x18, 0xae, 0x96, 0x0d, 0x1e, 0x80, 0xaf, 0x56, 0x0d, 0x43, 0x68, 0x3f, 0xbe, 0x5d, 0xd5, 0xa9,
0xdc, 0x42, 0x8a, 0x02, 0x24, 0xb1, 0x46, 0x31, 0x9e, 0x2b, 0x13, 0x9b, 0x31, 0x4a, 0x0f, 0x8d,
0x2b, 0x0f, 0x5d, 0xad, 0x54, 0xae, 0xea, 0x6b, 0x6a, 0xbc, 0x3d, 0xb3, 0xe9, 0xde, 0xd4, 0x1b,
0xe9, 0xc3, 0xdb, 0xda, 0xc2, 0x97, 0x27, 0x2c, 0x4e, 0xd8, 0x61, 0x72, 0xcd, 0x8e, 0xcc, 0x08,
0x3c, 0x84, 0x79, 0xd4, 0x8d, 0xba, 0x66, 0x0b, 0x2c, 0x49, 0x7e, 0x30, 0xf2, 0x6a, 0xf4, 0x77,
0xd9, 0x90, 0x1b, 0x6b, 0xf8, 0x2e, 0xf3, 0x75, 0xaf, 0xce, 0x57, 0x39, 0x56, 0xeb, 0xa2, 0x4e,
0x9d, 0xa7, 0x1c, 0x23, 0x41, 0x9e, 0x40, 0xa3, 0xd7, 0x3f, 0xe2, 0x43, 0x16, 0x7c, 0x00, 0xf3,
0x18, 0x21, 0xcf, 0xcd, 0x44, 0xdf, 0x9a, 0xea, 0x14, 0xb5, 0x7c, 0xd2, 0x35, 0x99, 0xcd, 0x8c,
0xe9, 0x01, 0x34, 0xd0, 0x7b, 0x1e, 0xfa, 0xd3, 0x66, 0x10, 0xa7, 0x86, 0x4d, 0xb6, 0xc1, 0xdb,
0xa7, 0x91, 0xda, 0x54, 0x8c, 0xc0, 0x5a, 0x31, 0x94, 0xb2, 0xfd, 0xb5, 0xc8, 0xa5, 0xa9, 0x13,
0xbe, 0x15, 0xf6, 0x52, 0x64, 0x12, 0x6b, 0xd4, 0xa1, 0xf8, 0x26, 0x39, 0xf8, 0xbb, 0x62, 0xc0,
0x83, 0x45, 0x70, 0xa3, 0xae, 0xb1, 0xe1, 0x46, 0xdd, 0xe0, 0x5d, 0x34, 0x6f, 0x4a, 0xd3, 0xa9,
0x82, 0xd8, 0xa7, 0x11, 0x45, 0xc7, 0xf7, 0xa1, 0x13, 0xe5, 0x5b, 0x42, 0x64, 0x83, 0x38, 0x65,
0x52, 0x64, 0xe6, 0x1b, 0x30, 0x09, 0xe2, 0x06, 0x49, 0x26, 0xf5, 0xc5, 0x6e, 0x51, 0x4d, 0x90,
0x67, 0xb0, 0xa4, 0x9c, 0x22, 0x61, 0xfb, 0xbd, 0x02, 0x0d, 0x85, 0x95, 0x41, 0x18, 0xaa, 0xb2,
0xe0, 0xd6, 0x2d, 0x7c, 0xab, 0x2d, 0x6c, 0x9f, 0xf0, 0x54, 0xd6, 0x26, 0x06, 0x69, 0x34, 0xd0,
0xa1, 0x9a, 0x08, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x58, 0x65, 0xa2, 0x50, 0x8a, 0x3c, 0xf2, 0xab,
0x03, 0x60, 0x03, 0x2a, 0xf2, 0x52, 0xc5, 0xb9, 0x5c, 0x25, 0x58, 0xb7, 0x9d, 0x37, 0xdb, 0xb2,
0x54, 0x49, 0x69, 0x9c, 0xda, 0xc9, 0xf8, 0xa8, 0x9a, 0x0c, 0xdd, 0xd2, 0xbb, 0x53, 0x93, 0xa1,
0xbd, 0x56, 0xf3, 0xf1, 0x12, 0xda, 0x35, 0x7c, 0xe6, 0x94, 0x7c, 0x58, 0x4e, 0x89, 0x3b, 0x6d,
0x12, 0x71, 0x63, 0xd2, 0xce, 0xca, 0x0b, 0x68, 0xd7, 0xe0, 0x99, 0x16, 0xd7, 0xe1, 0xd6, 0xe4,
0x1e, 0xda, 0xfb, 0x3e, 0x0d, 0x93, 0x18, 0x3a, 0x5b, 0x49, 0x91, 0x4b, 0x9e, 0x19, 0x73, 0xea,
0xa3, 0xa0, 0x81, 0xb2, 0x79, 0x15, 0x30, 0xbb, 0x7f, 0xc1, 0x7d, 0x98, 0x53, 0x65, 0xd4, 0xeb,
0x74, 0xb1, 0xc6, 0x9a, 0x49, 0x0e, 0xa0, 0xb9, 0xd9, 0x8b, 0x9e, 0x67, 0xa2, 0x18, 0xcd, 0x0c,
0xda, 0x7e, 0xd3, 0xdd, 0x8b, 0xdf, 0x74, 0xef, 0xc2, 0x37, 0xdd, 0x2f, 0xbf, 0xe9, 0xa4, 0x07,
0xcb, 0xfa, 0x54, 0xaa, 0x2d, 0xbe, 0xc9, 0xc1, 0xb1, 0x1f, 0x52, 0xaf, 0xf6, 0x21, 0xed, 0xc1,
0xb2, 0xbe, 0x67, 0xff, 0xa5, 0xd1, 0xdf, 0x5d, 0x58, 0xa6, 0x3c, 0x8f, 0x5f, 0xf3, 0x28, 0xcd,
0x65, 0x56, 0xf4, 0xd5, 0x4d, 0x52, 0xfa, 0xdf, 0x88, 0x43, 0x53, 0x6d, 0x8f, 0x6a, 0xe2, 0x3a,
0x93, 0x1e, 0x3c, 0x82, 0xf6, 0xf4, 0xce, 0x5e, 0x14, 0xad, 0x8b, 0x04, 0x8f, 0x60, 0xbe, 0x27,
0x8a, 0xac, 0x5f, 0x8e, 0x6f, 0xed, 0x4e, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xe0, 0xe9, 0xd4,
0x80, 0x84, 0x0d, 0xf4, 0xf2, 0xff, 0x4a, 0x6f, 0x82, 0x4d, 0xa7, 0xc6, 0xe9, 0xe3, 0xfa, 0x2e,
0x86, 0xf3, 0xa8, 0x7b, 0x67, 0x32, 0x42, 0xa3, 0x58, 0x93, 0x23, 0xbf, 0x38, 0xb0, 0x50, 0x0f,
0xe7, 0x5a, 0x4b, 0x5c, 0x76, 0xc7, 0x9d, 0xd9, 0x1d, 0x6f, 0x56, 0x77, 0xfc, 0xaa, 0x3b, 0xd5,
0xef, 0x83, 0xb9, 0xda, 0xef, 0x03, 0x72, 0x0c, 0xf7, 0x2e, 0xb4, 0x6c, 0x4b, 0x0c, 0x47, 0x6a,
0x36, 0xfe, 0x45, 0xeb, 0xd4, 0x79, 0xcb, 0x32, 0xd3, 0xb4, 0x16, 0xd5, 0x04, 0xf9, 0x14, 0xee,
0xf6, 0xb8, 0xac, 0x35, 0xcc, 0x4e, 0xde, 0x1a, 0x78, 0xbb, 0xfc, 0xf4, 0x92, 0xf4, 0x15, 0x8b,
0x7c, 0x01, 0xe1, 0xfe, 0x68, 0xc0, 0x24, 0xbf, 0x91, 0xf6, 0x26, 0x34, 0xf7, 0xc4, 0x48, 0x24,
0xe2, 0xd5, 0xf8, 0x8a, 0x0b, 0x10, 0xc2, 0xbc, 0xbe, 0xe5, 0xfa, 0xa4, 0xb4, 0xa8, 0x25, 0xc9,
0x6d, 0x35, 0xdc, 0x7d, 0x96, 0xf4, 0x8b, 0x44, 0x85, 0xa1, 0x7e, 0x3b, 0xe6, 0x9b, 0x4b, 0x7f,
0x9c, 0xaf, 0x3a, 0x7f, 0x9e, 0xaf, 0x3a, 0x6f, 0xce, 0x57, 0x9d, 0xdf, 0xfe, 0x5a, 0xfd, 0xdf,
0x61, 0x03, 0xff, 0x83, 0x3c, 0xf9, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xd8, 0x6d, 0x1f, 0x94,
0x0c, 0x00, 0x00,
}

2
internal/private.proto
View file

@ -159,7 +159,7 @@ message ResizeInstruction {
Node Node = 2;
Node Coordinator = 3;
repeated ResizeSource Sources = 4;
Schema Schema = 5;
NodeStatus NodeStatus = 7;
ClusterStatus ClusterStatus = 6;
}

929
internal/public.pb.go
View file

File diff suppressed because it is too large Load diff

17
internal/public.proto
View file

@ -22,6 +22,7 @@ message Pair {
message FieldRow{
string Field = 1;
uint64 RowID = 2;
string RowKey = 3;
}
message GroupCount{
@ -34,12 +35,6 @@ message ValCount {
int64 Count = 2;
}
message Bit {
uint64 RowID = 1;
uint64 ColumnID = 2;
int64 Timestamp = 3;
}
message ColumnAttrSet {
uint64 ID = 1;
string Key = 3;
@ -106,6 +101,16 @@ message ImportValueRequest {
repeated int64 Values = 6;
}
message TranslateKeysRequest {
string Index = 1;
string Field = 2;
repeated string Keys = 3;
}
message TranslateKeysResponse {
repeated uint64 IDs = 3;
}
message ImportRoaringRequestView {
string Name = 1;
bytes Data = 2;

11
pilosa.go
View file

@ -49,14 +49,16 @@ var (
ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]")
ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
ErrReservedName = errors.New("reserved index or field name")
// ErrFragmentNotFound is returned when a fragment does not exist.
ErrFragmentNotFound = errors.New("fragment not found")
ErrQueryRequired = errors.New("query required")
ErrQueryCancelled = errors.New("query cancelled")
ErrQueryTimeout = errors.New("query timeout")
ErrTooManyWrites = errors.New("too many write commands")
ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard")
// TODO(2.0) poorly named - used when a *node* doesn't own a shard. Probably
// we won't need this error at all by 2.0 though.
ErrClusterDoesNotOwnShard = errors.New("node does not own shard")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
@ -129,9 +131,6 @@ const TimeFormat = "2006-01-02T15:04"
// validateName ensures that the name is a valid format.
func validateName(name string) error {
if name == existenceFieldName {
return ErrReservedName
}
if !nameRegexp.Match([]byte(name)) {
return ErrName
}

5
pilosa_internal_test.go
View file

@ -20,7 +20,7 @@ import (
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"a", "ab", "ab1", "b-c", "d_e", "exists",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
@ -33,8 +33,7 @@ func TestValidateName(t *testing.T) {
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
"exists",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists",
}
for _, name := range names {
if validateName(name) == nil {

177
pql/ast.go
View file

@ -27,28 +27,34 @@ import (
type Query struct {
Calls []*Call
lastField string
lastCond Token
inList bool
callStack []*Call
callStack []*callStackElem
conditional []string
}
func (q *Query) startCall(name string) {
newCall := &Call{Name: name}
q.callStack = append(q.callStack, newCall)
q.callStack = append(q.callStack, &callStackElem{call: newCall})
if len(q.callStack) == 1 {
q.Calls = append(q.Calls, newCall)
} else {
calls := q.callStack[len(q.callStack)-2].Children
q.callStack[len(q.callStack)-2].Children = append(calls, newCall)
} else if prevElem := q.callStack[len(q.callStack)-2]; prevElem.lastField == "" {
prevElem.call.Children = append(prevElem.call.Children, newCall)
}
}
func (q *Query) endCall() {
// endCall removes the last element from the call stack and returns the call.
func (q *Query) endCall() *Call {
elem := q.callStack[len(q.callStack)-1]
q.callStack[len(q.callStack)-1] = nil
q.callStack = q.callStack[:len(q.callStack)-1]
return elem.call
}
func (q *Query) lastCallStackElem() *callStackElem {
if len(q.callStack) == 0 {
return nil
}
return q.callStack[len(q.callStack)-1]
}
func (q *Query) addPosNum(key, value string) {
@ -63,9 +69,9 @@ func (q *Query) addPosStr(key, value string) {
func (q *Query) startConditional() {
q.conditional = make([]string, 0)
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
elem := q.lastCallStackElem()
if elem.call.Args == nil {
elem.call.Args = make(map[string]interface{})
}
}
@ -85,51 +91,52 @@ func (q *Query) endConditional() {
if q.conditional[1] == "<" {
low++
}
if q.conditional[3] == "<=" {
high++
if q.conditional[3] == "<" {
high--
}
call := q.callStack[len(q.callStack)-1]
call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
elem := q.lastCallStackElem()
elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
q.conditional = nil
}
func (q *Query) addField(field string) {
if q.lastField != "" {
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField))
elem := q.lastCallStackElem()
if elem == nil || elem.lastField != "" {
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField))
}
q.lastField = field
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
elem.lastField = field
if elem.call.Args == nil {
elem.call.Args = make(map[string]interface{})
}
}
func (q *Query) addVal(val interface{}) {
if q.lastField == "" {
elem := q.lastCallStackElem()
if elem == nil || elem.lastField == "" {
panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, val)
if elem.inList {
list := elem.call.Args[elem.lastField].([]interface{})
elem.call.Args[elem.lastField] = append(list, val)
return
}
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
if elem.lastCond != ILLEGAL {
elem.call.Args[elem.lastField] = &Condition{
Op: elem.lastCond,
Value: val,
}
} else {
call.Args[q.lastField] = val
elem.call.Args[elem.lastField] = val
}
q.lastField = ""
q.lastCond = ILLEGAL
elem.lastField = ""
elem.lastCond = ILLEGAL
}
func (q *Query) addNumVal(val string) {
if q.lastField == "" {
elem := q.lastCallStackElem()
if elem == nil || elem.lastField == "" {
panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
}
var ival interface{}
@ -142,70 +149,70 @@ func (q *Query) addNumVal(val string) {
if err != nil {
panic(err)
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
if q.lastCond != ILLEGAL {
list := call.Args[q.lastField].(*Condition).Value.([]interface{})
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
if elem.inList {
if elem.lastCond != ILLEGAL {
list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{})
elem.call.Args[elem.lastField] = &Condition{
Op: elem.lastCond,
Value: append(list, ival),
}
} else {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, ival)
list := elem.call.Args[elem.lastField].([]interface{})
elem.call.Args[elem.lastField] = append(list, ival)
}
return
} else if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
} else if elem.lastCond != ILLEGAL {
elem.call.Args[elem.lastField] = &Condition{
Op: elem.lastCond,
Value: ival,
}
} else {
call.Args[q.lastField] = ival
elem.call.Args[elem.lastField] = ival
}
q.lastField = ""
q.lastCond = ILLEGAL
elem.lastField = ""
elem.lastCond = ILLEGAL
}
func (q *Query) startList() {
call := q.callStack[len(q.callStack)-1]
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
elem := q.lastCallStackElem()
if elem.lastCond != ILLEGAL {
elem.call.Args[elem.lastField] = &Condition{
Op: elem.lastCond,
Value: make([]interface{}, 0),
}
} else {
call.Args[q.lastField] = make([]interface{}, 0)
elem.call.Args[elem.lastField] = make([]interface{}, 0)
}
q.inList = true
elem.inList = true
}
func (q *Query) endList() {
q.inList = false
q.lastField = ""
q.lastCond = ILLEGAL
elem := q.lastCallStackElem()
elem.inList = false
elem.lastField = ""
elem.lastCond = ILLEGAL
}
func (q *Query) addGT() {
q.lastCond = GT
q.lastCallStackElem().lastCond = GT
}
func (q *Query) addLT() {
q.lastCond = LT
q.lastCallStackElem().lastCond = LT
}
func (q *Query) addGTE() {
q.lastCond = GTE
q.lastCallStackElem().lastCond = GTE
}
func (q *Query) addLTE() {
q.lastCond = LTE
q.lastCallStackElem().lastCond = LTE
}
func (q *Query) addEQ() {
q.lastCond = EQ
q.lastCallStackElem().lastCond = EQ
}
func (q *Query) addNEQ() {
q.lastCond = NEQ
q.lastCallStackElem().lastCond = NEQ
}
func (q *Query) addBTWN() {
q.lastCond = BETWEEN
q.lastCallStackElem().lastCond = BETWEEN
}
// WriteCallN returns the number of mutating calls.
@ -229,6 +236,13 @@ func (q *Query) String() string {
return strings.Join(a, "\n")
}
type callStackElem struct {
call *Call
lastField string
lastCond Token
inList bool
}
// Call represents a function call in the AST.
type Call struct {
Name string
@ -241,13 +255,25 @@ type Call struct {
// Returns the field as a string if present, or an error if not.
func (c *Call) FieldArg() (string, error) {
for arg := range c.Args {
if !strings.HasPrefix(arg, "_") {
if !IsReservedArg(arg) {
return arg, nil
}
}
return "", fmt.Errorf("No field argument specified")
}
func IsReservedArg(name string) bool {
if strings.HasPrefix(name, "_") {
return true
}
switch name {
case "from", "to":
return true
default:
return false
}
}
// BoolArg is for reading the value at key from call.Args as a bool. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a bool. An error is
@ -277,6 +303,9 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
}
switch tval := val.(type) {
case int64:
if tval < 0 {
return 0, true, fmt.Errorf("value for '%s' must be positive, but got %v", key, tval)
}
return uint64(tval), true, nil
case uint64:
return tval, true, nil
@ -329,6 +358,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
}
}
// CallArg is for reading the value at key from call.Args as a Call. If the
// key is not in Call.Args, the value of the returned value will be nil, and
// the error will be nil. An error is returned if the value is not a Call.
func (c *Call) CallArg(key string) (*Call, bool, error) {
val, ok := c.Args[key]
if !ok {
return nil, false, nil
}
switch tval := val.(type) {
case *Call:
return tval, true, nil
default:
return nil, true, fmt.Errorf("could not convert %v of type %T to Call in Call.CallArg", tval, tval)
}
}
// keys returns a list of argument keys in sorted order.
func (c *Call) keys() []string {
a := make([]string, 0, len(c.Args))

9
pql/pql.peg
View file

@ -13,12 +13,12 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
/ 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
/ 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()}
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
allargs <- Call (comma Call)* (comma args)? / args / sp
args <- arg (comma args)? sp
arg <- ( field sp '=' sp value
/ field sp COND sp value
/ conditional
)
COND <- ( '><' { p.addBTWN() }
/ '<=' { p.addLTE() }
@ -28,13 +28,12 @@ COND <- ( '><' { p.addBTWN() }
/ '<' { p.addLT() }
/ '>' { p.addGT() }
)
conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
timerange <- field sp '=' sp value comma <timestampfmt> {p.addPosStr("_start", buffer[begin:end])} comma <timestampfmt> {p.addPosStr("_end", buffer[begin:end])}
value <- ( item
/ lbrack { p.startList() } list rbrack { p.endList() }
)
@ -42,8 +41,10 @@ list <- item (comma list)?
item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
/ 'true' &(comma / sp close) { p.addVal(true) }
/ 'false' &(comma / sp close) { p.addVal(false) }
/ timestampfmt { p.addVal(buffer[begin:end]) }
/ < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
/ < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
/ < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }
/ '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
@ -76,5 +77,5 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*
timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]
timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt
timestampfmt <- '"' <timestampbasicfmt> '"' / '\'' <timestampbasicfmt> '\'' / <timestampbasicfmt>
timestamp <- <timestampfmt> {p.addPosStr("_timestamp", buffer[begin:end])}

2688
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

157
pql/pqlpeg_test.go
View file

@ -8,7 +8,7 @@ import (
func TestPEG(t *testing.T) {
p := PQL{Buffer: `
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
p.Init()
err := p.Parse()
if err != nil {
@ -202,51 +202,59 @@ func TestPEGWorking(t *testing.T) {
ncalls: 1},
{
name: "RangeLT",
input: "Range(a < 4)",
input: "Row(a < 4)",
ncalls: 1},
{
name: "RangeGT",
input: "Range(a > 4)",
input: "Row(a > 4)",
ncalls: 1},
{
name: "RangeLTE",
input: "Range(a <= 4)",
input: "Row(a <= 4)",
ncalls: 1},
{
name: "RangeGTE",
input: "Range(a >= 4)",
input: "Row(a >= 4)",
ncalls: 1},
{
name: "RangeEQ",
input: "Range(a == 4)",
input: "Row(a == 4)",
ncalls: 1},
{
name: "RangeNEQ",
input: "Range(a != null)",
input: "Row(a != null)",
ncalls: 1},
{
name: "RangeLTLT",
input: "Range(4 < a < 9)",
input: "Row(4 < a < 9)",
ncalls: 1},
{
name: "RangeLTLTE",
input: "Range(4 < a <= 9)",
input: "Row(4 < a <= 9)",
ncalls: 1},
{
name: "RangeLTELT",
input: "Range(4 <= a < 9)",
input: "Row(4 <= a < 9)",
ncalls: 1},
{
name: "RangeLTELTE",
input: "Range(4 <= a <= 9)",
input: "Row(4 <= a <= 9)",
ncalls: 1},
{
name: "RangeTime",
input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)",
input: "Row(a=4, from=2010-07-04T00:00, to=2010-08-04T00:00)",
ncalls: 1},
{
name: "RangeTimeQuotes",
input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`,
input: `Row(a=4, from='2010-07-04T00:00', to="2010-08-04T00:00")`,
ncalls: 1},
{
name: "RangeTimeFromQuotes",
input: `Row(a=4, from='2010-07-04T00:00')`,
ncalls: 1},
{
name: "RangeTimeToQuotes",
input: `Row(a=4, to="2010-08-04T00:00")`,
ncalls: 1},
{
name: "Dashed Frame",
@ -302,10 +310,10 @@ func TestPEGErrors(t *testing.T) {
input: "Clear(9)"},
{
name: "RangeTimeGT",
input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"},
input: "Row(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"},
{
name: "RangeTimeOneStamp",
input: "Range(a=4, 2010-07-04T00:00)"},
input: "Row(a=4, 2010-07-04T00:00)"},
}
for i, test := range tests {
@ -423,9 +431,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeEQ",
call: "Range(a==7)",
call: "Row(a==7)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: EQ,
@ -435,9 +443,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeLT",
call: "Range(a<7)",
call: "Row(a<7)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: LT,
@ -447,9 +455,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeLTE",
call: "Range(a<=7)",
call: "Row(a<=7)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: LTE,
@ -459,9 +467,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeGTE",
call: "Range(a>=7)",
call: "Row(a>=7)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: GTE,
@ -471,9 +479,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeGT",
call: "Range(a>7)",
call: "Row(a>7)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: GT,
@ -483,9 +491,9 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeNEQ",
call: "Range(a!=null)",
call: "Row(a!=null)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: NEQ,
@ -495,9 +503,33 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "RangeLTELT",
call: "Range(4 <= a < 9)",
call: "Row(4 <= a < 9)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(8)},
},
},
}},
{
name: "RangeLTLT",
call: "Row(4 < a < 9)",
exp: &Call{
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(8)},
},
},
}},
{
name: "RangeLTELTE",
call: "Row(4 <= a <= 9)",
exp: &Call{
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
@ -506,10 +538,10 @@ func TestPQLDeepEquality(t *testing.T) {
},
}},
{
name: "RangeLTLT",
call: "Range(4 < a < 9)",
name: "RangeLTLTE",
call: "Row(4 < a <= 9)",
exp: &Call{
Name: "Range",
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
@ -517,30 +549,6 @@ func TestPQLDeepEquality(t *testing.T) {
},
},
}},
{
name: "RangeLTELTE",
call: "Range(4 <= a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(10)},
},
},
}},
{
name: "RangeLTLTE",
call: "Range(4 < a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(10)},
},
},
}},
{
name: "Sum",
call: "Sum(field=f)",
@ -612,6 +620,43 @@ func TestPQLDeepEquality(t *testing.T) {
},
},
}},
{
name: "GroupBy",
call: "GroupBy(Rows(), filter=Row(a=1))",
exp: &Call{
Name: "GroupBy",
Args: map[string]interface{}{
"filter": &Call{
Name: "Row",
Args: map[string]interface{}{
"a": int64(1),
},
},
},
Children: []*Call{
{Name: "Rows"},
},
}},
{
name: "GroupByFilterRangeLTLT",
call: "GroupBy(Rows(), filter=Row(4 < a < 9))",
exp: &Call{
Name: "GroupBy",
Args: map[string]interface{}{
"filter": &Call{
Name: "Row",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(8)},
},
},
},
},
Children: []*Call{
{Name: "Rows"},
},
}},
}
for i, test := range tests {

6
roaring/containers.go
View file

@ -156,6 +156,12 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found
return &sliceIterator{e: sc, i: i}, found
}
func (sc *sliceContainers) Repair() {
for _, c := range sc.containers {
c.Repair()
}
}
type sliceIterator struct {
e *sliceContainers
i int

438
roaring/roaring.go
View file

@ -98,8 +98,12 @@ type Containers interface {
Count() uint64
//Reset will clear the containers collection to allow for recycling during snapshot
// Reset clears the containers collection to allow for recycling during snapshot
Reset()
// Repair will repair the cardinality of any containers whose cardinality were corrupted
// due to optimized operations.
Repair()
}
type ContainerIterator interface {
@ -394,10 +398,26 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
return output
}
// Union returns the bitwise union of b and other.
func (b *Bitmap) Union(other *Bitmap) *Bitmap {
// Union returns the bitwise union of b and others as a new bitmap.
func (b *Bitmap) Union(others ...*Bitmap) *Bitmap {
output := NewBitmap()
if len(others) == 1 {
b.unionIntoTargetSingle(output, others[0])
return output
}
output.UnionInPlace(b)
output.UnionInPlace(others...)
return output
}
// UnionInPlace returns the bitwise union of b and others, modifying
// b in place.
func (b *Bitmap) UnionInPlace(others ...*Bitmap) {
b.unionInPlace(others...)
}
func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
iiter, _ := b.Containers.Iterator(0)
jiter, _ := other.Containers.Iterator(0)
i, j := iiter.Next(), jiter.Next()
@ -405,21 +425,265 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap {
kj, cj := jiter.Value()
for i || j {
if i && (!j || ki < kj) {
output.Containers.Put(ki, ci.Clone())
target.Containers.Put(ki, ci.Clone())
i = iiter.Next()
ki, ci = iiter.Value()
} else if j && (!i || ki > kj) {
output.Containers.Put(kj, cj.Clone())
target.Containers.Put(kj, cj.Clone())
j = jiter.Next()
kj, cj = jiter.Value()
} else { // ki == kj
output.Containers.Put(ki, union(ci, cj))
target.Containers.Put(ki, union(ci, cj))
i, j = iiter.Next(), jiter.Next()
ki, ci = iiter.Value()
kj, cj = jiter.Value()
}
}
return output
}
// unionIntoTarget stores the union of b and others into target. b and others will
// be left unchanged (unless one of them is also target), but target will be modified
// in place.
//
// This function performs an n-way union of n bitmaps. It performs this in an
// optimized manner looping through all the bitmaps and performing unions one
// container at a time. As a result, instead of generating many intermediary
// containers for each union operation for a given container key, only one
// new container needs to be allocated (or re-used) regardless of how many bitmaps
// participate in the union. This significantly reduces allocations. In addition,
// because we perform the unions one container at a time across all the bitmaps, we
// can calculate summary statistics that allow us to make more efficient decisions
// up front. For example, imagine trying to perform a union across the following three
// bitsets:
//
// 1. Bitmap A: Single array container at key 0 with 400 values in it.
// 2. Bitmap B: Single array container at key 0 with 500 values in it.
// 3. Bitmap C: Single array container at key 0 with 3500 values in it.
//
// Naive approach:
//
// 1. Perform union of bitmap A and B, container by container
// a. 400 + 500 < ArrayMaxSize so likely we will choose to allocate a new array
// container and then perform a unionArrayArray operation to merge the two
// arrays into the new array container.
// 2. Perform a union of the bitmap generated in the step above with bitmap C.
// 900 + 3500 > ArrayMaxSize so we will need to upgrade to a bitset container which
// we will have to allocate, and then we will have to perform two unions into the
// new bitmap container: one for the array container generated in the previous step,
// and one for the bitset container in bitmap C.
//
// Approach taken by this function:
//
// 1. Detect that bitmaps A, B, and C all have containers for key 0.
// 2. Estimate the resulting cardinality of the union of all their containers to be
// 400 + 500 + 3500 > ArrayMaxSize and decide upfront to use a bitset for the target
// container. Note that this is just an approximation of the final cardinality and can
// be off by a wide margin if there is a lot of overlap between containers, but that is
// fine, we'll still get the same result at the end, we'll just be more biased towards
// using bitmap containers will still being able to use array containers when all the
// cardinalities are small.
// 3. Union the containers from bitmaps A, B, and C into the new bitset container directly
// using fast bitwise operations.
//
// In the naive approach, we had to allocate two containers, whereas in the optimized approach
// we only had to allocate one container, and we also had to perform less union operations. This
// example is simplistic, but the impact in terms of CPU cycles and memory allocations achieved
// by using the optimized alogorithm when unioning many large bitmaps can be huge.
//
// An additional optimization that this function makes is that it recognizes that even when
// CPU support is present, performing the popcount() operation isn't free. Imagine a scenario
// where 10 bitset containers are being unioned together one after the next. If every
// bitset<->bitset union operation needs to keep the containers' cardinality up to date, then
// the algorithm will waste a lot of time performing intermediary popcount() operations that
// will immediately be invalidated by the next union operation. As a result, we allow the cardinality
// of containers to degrade when we perform the in-place union operations, and then when the algorithm
// completes we "repair" all the containers by performing the popcount() operation one time. This means
// that we only ever have to do O(1) popcount operations per container instead of O(n) where n is the
// number of containers with the same key that are being unioned together.
//
// The algorithm works by iterating through all of the containers in all of the bitmaps concurrently.
// At every "tick" of the outermost loop, we increment our pointer into the bitmaps list of containers
// by 1 (if we haven't reached the end of the containers for that bitmap.)
//
// We then loop through all of the "current" values(containers) for all of the bitmaps
// and for each container with a specific key that we encounter, we scan forward to see if any of the
// other bitmaps have a container for the same key. If so, we calculate some summary statistics and
// then use that information to make a decision about how to union all of the containers with the same
// key together, perform the union, mark the unioned containers as "handled" and then move on to the next
// batch of containers that share the same key.
//
// We repeat this process until every single bitmaps current container has been "handled". Then we start the
// outer loop over again and the process repeats until we've iterated through every container in every bitmap
// and unioned everything into a single target bitmap.
//
// The diagram below shows the iteration state of four different bitmaps as the algorithm progresses them.
// The diagrams should BE interpreted from left -> right, top -> bottom. The X's represent a container in
// the bitmap at a specific key, ^ symbol represents the bitmaps current container iteration position,
// and the - symbol represents a container that is at the current iteration position, but has been marked as "handled".
//
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________|
// ^ | _ |
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 2 |_______X________X______X___| | |_______X_______________X___| | |_______X_______________X___|
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________|
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________|
// ^ | _ |
// ------------------------------------------------------------------------------------------------------------------------
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________|
// _ | ^ | _
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 2 |_______X_______________X___| | |_______X_______________X___| | |_______X_______________X___|
// _ | ^ | ^
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________|
// _ | |
// ---------------------------- | ---------------------------- | ----------------------------
// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________|
// _
func (b *Bitmap) unionInPlace(others ...*Bitmap) {
var (
requiredSliceSize = len(others)
// To avoid having to allocate a slice everytime, if the number of bitmaps
// being unioned is small enough we can just use this stack-allocated array.
staticHandledIters = [20]handledIter{}
bitmapIters handledIters
target = b
)
if requiredSliceSize <= 20 {
bitmapIters = staticHandledIters[:0]
} else {
bitmapIters = make(handledIters, 0, requiredSliceSize)
}
for _, other := range others {
otherIter, _ := other.Containers.Iterator(0)
if otherIter.Next() {
bitmapIters = append(bitmapIters, handledIter{
iter: otherIter,
hasNext: true,
handled: false,
})
}
}
// Loop until we've exhausted every iter.
hasNext := true
for hasNext {
// Loop until every iters current value has been handled.
for i, iIter := range bitmapIters {
if !iIter.hasNext || iIter.handled {
// Either we've exhausted this iter (it has no more containers), or
// we've already handled the current container by unioning it with
// one of the containers we encountered earlier.
continue
}
var (
iKey, iContainer = iIter.iter.Value()
// Summary statistics about all the containers in the other bitmaps
// that share the same key so we can make smarter union strategy
// decisions later. Note that we slice to [i:] not [i+1:] because we
// want to include the current containers information in the stats.
summaryStats = bitmapIters[i:].calculateSummaryStats(iKey)
)
if summaryStats.isOnlyContainerWithKey {
// TODO(rartoul): We can avoid these clones if we can determine
// if the container is coming from an immutable bitmap and we
// know that we can mark the target bitmap as immutable as well.
target.Containers.Put(iKey, iContainer.Clone())
bitmapIters[i].handled = true
continue
}
// There was more than one container across the bitmaps with key iKey
// so we need to calculate a union.
if summaryStats.hasMaxRange {
// One (or more) of the containers represented the maximum possible
// range that a container can store, so instead of calculating a
// union we can generate an RLE container that represents the entire
// range.
container := &Container{
runs: []interval16{{start: 0, last: maxContainerVal}},
containerType: containerRun,
n: maxContainerVal + 1,
}
target.Containers.Put(iKey, container)
bitmapIters.markItersWithCurrentKeyAsHandled(i, iKey)
continue
}
// Use a bitmap container for the target bitmap, and union everything
// into it.
//
// TODO(rartoul): Add another conditional case for n < ArrayMaxSize
// (or some fraction of that value) to avoid allocating expensive
// bitmaps when unioning many low-density array containers, but this
// will require writing a union in place algorithm for an array container
// that accepts multiple different containers to union into it for
// efficiency.
container := target.Containers.Get(iKey)
// If target already has a bitmap container for iKey then we can reuse that,
// otherwise we have to allocate a new one or convert the existing container
// into a bitmap container.
if container == nil {
buf := make([]uint64, bitmapN)
ob := buf[:bitmapN]
container = &Container{
bitmap: ob,
n: 0,
containerType: containerBitmap,
}
} else if container.isArray() {
container.arrayToBitmap()
} else if container.isRun() {
container.runToBitmap()
}
// Once we've acquired a bitmap container (either by reusing the existing one
// or allocating a new one) then the last step is to iterate through all the
// other containers to see which ones have the same key, and union all of them
// into the target bitmap container. Only need to loop starting from i because
// anything previous to that has already been handled.
for j := i; j < len(bitmapIters); j++ {
jIter := bitmapIters[j]
jKey, jContainer := jIter.iter.Value()
if iKey == jKey {
if jContainer.isArray() {
unionBitmapArrayInPlace(container, jContainer)
} else if jContainer.isRun() {
unionBitmapRunInPlace(container, jContainer)
} else {
unionBitmapBitmapInPlace(container, jContainer)
}
bitmapIters[j].handled = true
}
}
// Now that we've calculated a container that is a union of all the containers
// with the same key across all the bitmaps, we store it in the list of containers
// for the target.
target.Containers.Put(iKey, container)
}
hasNext = bitmapIters.next()
}
// Performing the popcount() operation with every union is wasteful because
// its likely the value will be invalidated by the next union operation. As
// a result, when we're performing all of our in-place unions we allow the value of
// n (container cardinality) to fall out of sync, and then at the very end we perform
// a "Repair" to recalculate all the container values. That way we never popcount()
// an entire bitmap container more than once per bulk union operation.
target.Containers.Repair()
}
// Difference returns the difference of b and other.
@ -1823,6 +2087,28 @@ func (c *Container) check() error {
return a
}
// Repair repairs the cardinality of c if it has been corrupted by
// optimized operations.
func (c *Container) Repair() {
if c.isBitmap() {
c.bitmapRepair()
}
}
func (c *Container) bitmapRepair() {
n := int32(0)
// Manually unroll loop to make it a little faster.
// TODO(rartoul): Can probably make this a few x faster using
// SIMD instructions.
for i := 0; i < bitmapN; i += 4 {
n += int32(popcount(c.bitmap[i]))
n += int32(popcount(c.bitmap[i+1]))
n += int32(popcount(c.bitmap[i+2]))
n += int32(popcount(c.bitmap[i+3]))
}
c.n = n
}
// containerInfo represents a point-in-time snapshot of container stats.
type containerInfo struct {
Key uint64 // container key
@ -2383,6 +2669,15 @@ func unionBitmapRun(a, b *Container) *Container {
return output
}
// unions the run b into the bitmap a, mutating a in place. The n value of
// a will need to be repaired after the fact.
func unionBitmapRunInPlace(a, b *Container) {
statsHit("union/BitmapRun")
for j := 0; j < len(b.runs); j++ {
a.bitmapSetRangeIgnoreN(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
}
}
const maxBitmap = 0xFFFFFFFFFFFFFFFF
// sets all bits in [i, j) (c must be a bitmap container)
@ -2408,6 +2703,25 @@ func (c *Container) bitmapSetRange(i, j uint64) {
}
}
// sets all bits in [i, j) (c must be a bitmap container) without updating
// the value of n, meaning it will need to be repaired after the fact.
func (c *Container) bitmapSetRangeIgnoreN(i, j uint64) {
x := i >> 6
y := (j - 1) >> 6
var X uint64 = maxBitmap << (i % 64)
var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64))
if x == y {
c.bitmap[x] |= (X & Y)
} else {
c.bitmap[x] |= X
for i := x + 1; i < y; i++ {
c.bitmap[i] = maxBitmap
}
c.bitmap[y] |= Y
}
}
// xor's all bits in [i, j) with all true (c must be a bitmap container).
func (c *Container) bitmapXorRange(i, j uint64) {
x := i >> 6
@ -2502,6 +2816,14 @@ func unionArrayBitmap(a, b *Container) *Container {
return output
}
// unions array b into bitmap a, mutating a in place. The n value
// of a will need to be repaired after the fact.
func unionBitmapArrayInPlace(a, b *Container) {
for _, v := range b.array {
a.bitmap[v>>6] |= (uint64(1) << (v % 64))
}
}
func unionBitmapBitmap(a, b *Container) *Container {
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
@ -2528,6 +2850,28 @@ func unionBitmapBitmap(a, b *Container) *Container {
return output
}
// unions bitmap b into bitmap a, mutating a in place. The n value of
// a will need to be repaired after the fact.
func unionBitmapBitmapInPlace(a, b *Container) {
// local variables added to prevent BCE checks in loop
// see https://go101.org/article/bounds-check-elimination.html
var (
ab = a.bitmap[:bitmapN]
bb = b.bitmap[:bitmapN]
)
// Manually unroll loop to make it a little faster.
// TODO(rartoul): Can probably make this a few x faster using
// SIMD instructions.
for i := 0; i < bitmapN; i += 4 {
ab[i] |= bb[i]
ab[i+1] |= bb[i+1]
ab[i+2] |= bb[i+2]
ab[i+3] |= bb[i+3]
}
}
func difference(a, b *Container) *Container {
if a.isArray() {
if b.isArray() {
@ -3636,3 +3980,83 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) {
}
}
}
// handledIter and handledIters are wrappers around Bitmap Container iterators
// and assist with the unionIntoTarget algorithm by abstracting away some tedious
// operations.
type handledIter struct {
iter ContainerIterator
hasNext bool
handled bool
}
type handledIters []handledIter
func (w handledIters) next() bool {
hasNext := false
for i, wrapped := range w {
next := wrapped.iter.Next()
w[i].hasNext = next
w[i].handled = false
if next {
hasNext = true
}
}
return hasNext
}
func (w handledIters) markItersWithCurrentKeyAsHandled(startIdx int, key uint64) {
for i := startIdx; i < len(w); i++ {
wrapped := w[i]
currKey, _ := wrapped.iter.Value()
if currKey == key {
w[i].handled = true
}
}
}
func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummaryStats {
summary := containerUnionSummaryStats{}
for _, iter := range w {
// Calculate key-level statistics here
currKey, currContainer := iter.iter.Value()
if key == currKey {
summary.isOnlyContainerWithKey = false
summary.n += currContainer.n
if currContainer.n == maxContainerVal+1 {
summary.hasMaxRange = true
summary.n = maxContainerVal + 1
return summary
}
}
}
return summary
}
// Summary statistics about all the containers in the other bitmaps
// that share the same key so we can make smarter union strategy
// decisions.
type containerUnionSummaryStats struct {
// Estimated cardinality of the union of all containers with the same
// key across all bitmaps. This calculation is very rough as we just sum
// the cardinality of the container across the different bitmaps which could
// result in very inflated values, but it allows us to avoid allocating
// expensive bitmaps when unioning many low density containers.
n int32
// Whether any other is the only container across all the bitmaps
// with the specified key. If true, we can skip all the unioning logic
// and just clone the container into target.
isOnlyContainerWithKey bool
// Whether any of the containers with the specified keys are storing every possible
// value that they can. If so, we can short-circuit all the unioning logic and use
// a RLE container with a single value in it. This is an optimization to
// avoid using an expensive bitmap container for bitmaps that have some
// extremely dense containers.
hasMaxRange bool
}

28
roaring/roaring_internal_test.go
View file

@ -3267,6 +3267,34 @@ func TestUnmarshalOfficialRoaring(t *testing.T) {
}
func BenchmarkUnionBitmapBitmapInPlace(b *testing.B) {
b1 := newTestBitmapContainer()
b2 := newTestBitmapContainer()
for n := 0; n < b.N; n++ {
unionBitmapBitmapInPlace(b1, b2)
}
}
func BenchmarkBitmapRepair(b *testing.B) {
b1 := newTestBitmapContainer()
for n := 0; n < b.N; n++ {
b1.bitmapRepair()
}
}
func newTestBitmapContainer() *Container {
var (
buf = make([]uint64, bitmapN)
ob = buf[:bitmapN]
container = &Container{
bitmap: ob,
n: 0,
containerType: containerBitmap,
}
)
return container
}
/*
// This function exercises an arcane edge case in dead code.
// It doesn't need to be run right now.

170
roaring/roaring_test.go
View file

@ -23,6 +23,7 @@ import (
"sort"
"testing"
"testing/quick"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/roaring"
@ -396,7 +397,132 @@ func TestBitmap_Union1(t *testing.T) {
if n := result.Count(); n != 75007 {
t.Fatalf("unexpected n: %d", n)
}
}
func TestBitmap_UnionInPlace1(t *testing.T) {
var (
bm0 = roaring.NewFileBitmap(0, 2683177)
bm1 = roaring.NewFileBitmap()
result = roaring.NewBitmap()
)
for i := uint64(628); i < 2683301; i++ {
bm1.Add(i)
}
bm1.Add(4000000)
result.UnionInPlace(bm0, bm1)
if n := result.Count(); n != 2682675 {
t.Fatalf("unexpected n: %d", n)
}
bm := testBM()
result = roaring.NewBitmap()
result.UnionInPlace(bm, bm0)
if n := result.Count(); n != 75009 {
t.Fatalf("unexpected n: %d", n)
}
result = roaring.NewBitmap()
result.UnionInPlace(bm, bm)
if n := result.Count(); n != 75007 {
t.Fatalf("unexpected n: %d", n)
}
// Make sure the bitmaps weren't mutated.
if n := bm0.Count(); n != 2 {
t.Fatalf("unexpected n: %d", n)
}
if n := bm1.Count(); n != 2682674 {
t.Fatalf("unexpected n: %d", n)
}
}
// TestBitmap_UnionInPlaceProp is a manual property test that randomly generates
// a number of different bitmaps with random vals and unions them together. It
// then compares the result against a reference implementation (golang map) to
// ensure that all the unions were handled correctly.
func TestBitmap_UnionInPlaceProp(t *testing.T) {
var (
seed = time.Now().UnixNano()
source = rand.NewSource(seed)
rng = rand.New(source)
numTests = 100
maxNumIntsPerBatch = 100
maxNumBatches = 100
maxRangePercent = 2
// Need to limit the range of possible numbers that we generate
// otherwise two randomly generated numbers landing in the same
// container would be extremely unlikely, leaving container merging
// behavior untested.
maxUint64Val = 1000000
)
for i := 0; i < numTests; i++ {
var (
// We will use sets as the "reference" implementation.
sets = []map[uint64]struct{}{}
bitmaps = []*roaring.Bitmap{}
)
// Ensure there are at least two batches.
numBatches := rng.Intn(maxNumBatches) + 2
for j := 0; j < numBatches; j++ {
// For each "batch" create the equivalent set and bitmap.
var (
set = map[uint64]struct{}{}
bitmap = roaring.NewBitmap()
)
if rng.Intn(100) <= maxRangePercent {
// Generate max range RLE containers with a configurable
// probability to ensure that code-path is exercised.
start := rng.Intn((maxUint64Val))
// Add a continuous sequence of numbers that is 2x as long as the maximum
// size of a container to ensure we generate a maxRange container.
for x := start; x < (start + 2*(0xffff+1)); x++ {
set[uint64(x)] = struct{}{}
bitmap.Add(uint64(x))
}
}
// Generate and add a bunch of random values.
numIntsPerBatch := rng.Intn(maxNumIntsPerBatch)
for x := 0; x < numIntsPerBatch; x++ {
num := uint64(rng.Intn(maxUint64Val))
set[num] = struct{}{}
bitmap.Add(num)
}
sets = append(sets, set)
bitmaps = append(bitmaps, bitmap)
}
// "Union" all the sets into the first one.
set0 := sets[0]
for _, set := range sets[1:] {
for val := range set {
set0[val] = struct{}{}
}
}
// Union all the bitmaps into the first one.
bitmap0 := bitmaps[0]
bitmap0.UnionInPlace(bitmaps[1:]...)
// Ensure the unioned set and bitmap have the same cardinality.
if len(set0) != int(bitmap0.Count()) {
t.Fatalf("cardinality of set is: %d, but bitmap is: %d, failed with seed: %d",
len(set0), bitmap0.Count(), seed)
}
// Ensure the unioned set and bitmap have the exact same values.
for val := range set0 {
if !bitmap0.Contains(val) {
t.Fatalf("set contained %d, but bitmap did not, failed with seed: %d",
val, seed)
}
}
}
}
func TestBitmap_Intersection_Empty(t *testing.T) {
@ -544,11 +670,35 @@ func TestBitmap_Union(t *testing.T) {
bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
result := bm0.Union(bm1)
if n := result.Count(); n != 5 {
t.Fatalf("unexpected n: %d", n)
}
}
func TestBitmap_UnionInPlace(t *testing.T) {
var (
bm0 = roaring.NewFileBitmap(0, 1000001, 1000002, 1000003)
bm1 = roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
result = roaring.NewBitmap()
)
result.UnionInPlace(bm0, bm1)
// Make sure the union worked.
if n := result.Count(); n != 5 {
t.Fatalf("unexpected n: %d", n)
}
// Make sure the other bitmaps weren't mutated.
if n := bm0.Count(); n != 4 {
t.Fatalf("unexpected n: %d", n)
}
if n := bm1.Count(); n != 4 {
t.Fatalf("unexpected n: %d", n)
}
}
func TestBitmap_Xor(t *testing.T) {
bm0 := testBM()
bm1 := roaring.NewFileBitmap(0, 1, 2, 3)
@ -1385,3 +1535,23 @@ func BenchmarkSliceDescendingStriped(b *testing.B) {
}
}
}
func BenchmarkUnion(b *testing.B) {
data := getBenchData(b)
for n := 0; n < b.N; n++ {
data.a1.
Union(data.a2).
Union(data.b).
Union(data.r1).
Union(data.r2)
}
}
func BenchmarkUnionBulk(b *testing.B) {
data := getBenchData(b)
bm := roaring.NewBitmap()
for n := 0; n < b.N; n++ {
bm.
UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
}
}

13
row.go
View file

@ -42,6 +42,19 @@ func NewRow(columns ...uint64) *Row {
return r
}
func (r *Row) IsEmpty() bool {
if len(r.segments) == 0 {
return true
}
for i := range r.segments {
if r.segments[i].n > 0 {
return false
}
}
return true
}
// Merge merges data from other into r.
func (r *Row) Merge(other *Row) {
var segments []rowSegment

14
row_test.go
View file

@ -111,3 +111,17 @@ func TestRow_Difference_Segment(t *testing.T) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp)
}
}
func TestRow_IsEmpty(t *testing.T) {
r1 := pilosa.NewRow(1, ShardWidth)
r2 := pilosa.NewRow(0, 2*ShardWidth)
res := r2.Intersect(r1)
if r1.IsEmpty() {
t.Fatal("r1 Should Not Be Empty\n")
}
if !res.IsEmpty() {
t.Fatal("Result Should Be Empty\n")
}
}

2
server.go
View file

@ -428,7 +428,7 @@ func (s *Server) SyncData() error {
}
func (s *Server) monitorAntiEntropy() {
if s.antiEntropyInterval == 0 {
if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 {
return // anti entropy disabled
}
s.cluster.initializeAntiEntropy()

44
server/cluster_test.go
View file

@ -206,6 +206,16 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatal(err)
}
// exp is the expected result for the Row queries that follow.
exp := `{"results":[{"attrs":{},"columns":[1,1300000]}]}` + "\n"
// Verify the data exists on the single node.
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
// Configure node1
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
@ -221,6 +231,18 @@ func TestClusterResize_AddNode(t *testing.T) {
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
// Verify the data exists on both nodes.
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
if res, err := m1.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
})
t.Run("SkippedShard", func(t *testing.T) {
// Configure node0
@ -247,6 +269,16 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatal(err)
}
// exp is the expected result for the Row queries that follow.
exp := `{"results":[{"attrs":{},"columns":[1,2400000]}]}` + "\n"
// Verify the data exists on the single node.
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
// Configure node1
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
@ -262,6 +294,18 @@ func TestClusterResize_AddNode(t *testing.T) {
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
// Verify the data exists on both nodes.
if res, err := m0.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
if res, err := m1.Query("i", "", `Row(f=1)`); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result: %s", res)
}
})
}

85
server/handler_test.go
View file

@ -713,6 +713,91 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("unexpected body: %q", w.Body.String())
}
})
t.Run("translate keys", func(t *testing.T) {
// create index
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i1-tr", strings.NewReader(`{"options":{"keys":true}}`))
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
// create field
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("POST", "/index/i1-tr/field/f1", strings.NewReader(`{"options":{"keys":true}}`))
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
// set some bits
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("POST", "/index/i1-tr/query", strings.NewReader(`Set("col1", f1="row1")`))
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
// Generate request body for translate column keys request
reqBody, err := cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: "i1-tr",
Keys: []string{"col1", "col2", "col3"},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
target := []uint64{1, 2, 3}
resp := pilosa.TranslateKeysResponse{}
err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(target, resp.IDs) {
t.Fatalf("%v != %v", target, resp.IDs)
}
// Generate request body for translate row keys request
reqBody, err = cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{
Index: "i1-tr",
Field: "f1",
Keys: []string{"row1", "row2"},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
target = []uint64{1, 2}
resp = pilosa.TranslateKeysResponse{}
err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(target, resp.IDs) {
t.Fatalf("%v != %v", target, resp.IDs)
}
})
}
func TestClusterTranslator(t *testing.T) {

24
server/server.go
View file

@ -174,30 +174,6 @@ func (m *Command) Wait() error {
}
}
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
m.logOutput = f
err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
if err != nil {
return errors.Wrap(err, "dup2ing stderr onto logfile")
}
}
if m.Config.Verbose {
m.logger = logger.NewVerboseLogger(m.logOutput)
} else {
m.logger = logger.NewStandardLogger(m.logOutput)
}
return nil
}
// SetupServer uses the cluster configuration to set up this server.
func (m *Command) SetupServer() error {
err := m.setupLogger()

166
server/server_test.go
View file

@ -15,8 +15,10 @@
package server_test
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"math/rand"
@ -31,10 +33,18 @@ import (
"github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/roaring"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
"golang.org/x/sync/errgroup"
)
var runStress bool
func init() { // nolint: gochecknoinits
flag.BoolVar(&runStress, "stress", false, "Enable stress tests (time consuming)")
}
// Ensure program can process queries and maintain consistency.
func TestMain_Set_Quick(t *testing.T) {
if testing.Short() {
@ -237,6 +247,59 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
}
func TestMain_GroupBy(t *testing.T) {
m := test.MustRunCommand()
defer m.Close()
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
}
if err := client.CreateFieldWithOptions(context.Background(), "i", "generalk", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
}
if err := client.CreateFieldWithOptions(context.Background(), "i", "subk", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
}
query := `
Set(0, generalk="ten")
Set(1, generalk="ten")
Set(1001, generalk="ten")
Set(2, generalk="eleven")
Set(1002, generalk="eleven")
Set(2, generalk="twelve")
Set(1002, generalk="twelve")
Set(0, subk="one-hundred")
Set(1, subk="one-hundred")
Set(3, subk="one-hundred")
Set(1001, subk="one-hundred")
Set(2, subk="one-hundred-ten")
Set(0, subk="one-hundred-ten")
`
// Set columns on row.
if _, err := m.Query("i", "", query); err != nil {
t.Fatal(err)
}
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred"}}, Count: 3},
{Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "eleven"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "twelve"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1},
}
// Query row.
if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil {
t.Fatal(err)
} else {
test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount))
}
}
// Ensure the host can be parsed.
func TestConfig_Parse_Host(t *testing.T) {
if c, err := ParseConfig(`bind = "local"`); err != nil {
@ -678,6 +741,15 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
defer cluster.Close()
cmd1 := cluster[1]
for _, com := range cluster {
nodes := com.API.Hosts(context.Background())
for _, n := range nodes {
if n.State != "READY" {
t.Fatalf("unexpected node state after upping cluster: %v", nodes)
}
}
}
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
@ -745,3 +817,97 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
}
// TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
func TestClusterExhaustingConnections(t *testing.T) {
if !runStress {
t.Skip("stress")
}
cluster := test.MustRunCluster(t, 5)
defer cluster.Close()
cmd1 := cluster[1]
for _, com := range cluster {
nodes := com.API.Hosts(context.Background())
for _, n := range nodes {
if n.State != "READY" {
t.Fatalf("unexpected node state after upping cluster: %v", nodes)
}
}
}
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
eg := errgroup.Group{}
for i := 0; i < 20; i++ {
i := i
eg.Go(func() error {
for j := i; j < 10000; j += 20 {
_, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{
Index: "testidx",
Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth),
})
if err != nil {
return err
}
}
return nil
})
}
err := eg.Wait()
if err != nil {
t.Fatalf("setting lots of shards: %v", err)
}
}
func TestClusterExhaustingConnectionsImport(t *testing.T) {
if !runStress {
t.Skip("stress")
}
cluster := test.MustRunCluster(t, 5)
defer cluster.Close()
cmd1 := cluster[1]
for _, com := range cluster {
nodes := com.API.Hosts(context.Background())
for _, n := range nodes {
if n.State != "READY" {
t.Fatalf("unexpected node state after upping cluster: %v", nodes)
}
}
}
cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
bm := roaring.NewBitmap()
bm.DirectAdd(0)
buf := &bytes.Buffer{}
bm.WriteTo(buf)
data := buf.Bytes()
eg := errgroup.Group{}
for i := uint64(0); i < 20; i++ {
i := i
eg.Go(func() error {
for j := i; j < 10000; j += 20 {
if (j-i)%1000 == 0 {
fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000)
}
err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{
Views: map[string][]byte{
"": data,
},
})
if err != nil {
return err
}
}
return nil
})
}
err := eg.Wait()
if err != nil {
t.Fatalf("setting lots of shards: %v", err)
}
}

35
server/setup_logger.go Normal file
View file

@ -0,0 +1,35 @@
// +build !arm64
package server
import (
"os"
"syscall"
"github.com/pilosa/pilosa/logger"
"github.com/pkg/errors"
)
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
m.logOutput = f
err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
if err != nil {
return errors.Wrap(err, "dup2ing stderr onto logfile")
}
}
if m.Config.Verbose {
m.logger = logger.NewVerboseLogger(m.logOutput)
} else {
m.logger = logger.NewStandardLogger(m.logOutput)
}
return nil
}

33
server/setup_logger_arm64.go Normal file
View file

@ -0,0 +1,33 @@
package server
import (
"os"
"syscall"
"github.com/pilosa/pilosa/logger"
"github.com/pkg/errors"
)
// setupLogger sets up the logger based on the configuration.
func (m *Command) setupLogger() error {
if m.Config.LogPath == "" {
m.logOutput = m.Stderr
} else {
f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return errors.Wrap(err, "opening file")
}
m.logOutput = f
err = syscall.Dup3(int(f.Fd()), int(os.Stderr.Fd()), 0)
if err != nil {
return errors.Wrap(err, "dup2ing stderr onto logfile")
}
}
if m.Config.Verbose {
m.logger = logger.NewVerboseLogger(m.logOutput)
} else {
m.logger = logger.NewStandardLogger(m.logOutput)
}
return nil
}

58
test/pilosa.go
View file

@ -22,12 +22,14 @@ import (
gohttp "net/http"
"os"
"path"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/encoding/proto"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
@ -69,7 +71,7 @@ func newCommand(opts ...server.CommandOption) *Command {
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
m.Config.Translation.MapSize = 100000
m.Config.Translation.MapSize = 140000
if testing.Verbose() {
m.Command.Stdout = os.Stdout
@ -185,6 +187,49 @@ func (m *Command) Query(index, rawQuery, query string) (string, error) {
return resp.Body, nil
}
func (m *Command) QueryProtobuf(indexName string, query string) (*pilosa.QueryResponse, error) {
var ser proto.Serializer
queryReq := &pilosa.QueryRequest{
Index: indexName,
Query: query,
}
body, err := ser.Marshal(queryReq)
if err != nil {
return nil, err
}
req, err := gohttp.NewRequest(
"POST",
fmt.Sprintf("%s/index/%s/query", m.URL(), indexName),
bytes.NewReader(body),
)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Accept", "application/x-protobuf")
resp, err := gohttp.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
response := &pilosa.QueryResponse{}
err = ser.Unmarshal(buf, response)
if err != nil {
return nil, err
}
return response, nil
}
// RecalculateCaches is deprecated. Use MustRecalculateCaches.
func (m *Command) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
@ -380,6 +425,17 @@ func MustDo(method, urlStr string, body string) *httpResponse {
return &httpResponse{Response: resp, Body: string(buf)}
}
func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) {
if len(results) != len(expected) {
t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected)
}
for i, result := range results {
if !reflect.DeepEqual(expected[i], result) {
t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i])
}
}
}
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*gohttp.Response

18
translate.go
View file

@ -29,10 +29,11 @@ const (
)
var (
ErrTranslateStoreClosed = errors.New("pilosa: translate store closed")
ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed")
ErrReplicationNotSupported = errors.New("pilosa: replication not supported")
ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only")
ErrTranslateStoreClosed = errors.New("pilosa: translate store closed")
ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed")
ErrReplicationNotSupported = errors.New("pilosa: replication not supported")
ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only")
ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized")
)
// TranslateStore is the storage for translation string-to-uint64 values.
@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) {
return 0, nil
}
// Shorten buffer to maximum read size.
if max := sz - r.offset; int64(len(p)) > max {
if max := sz - r.offset; max > int64(len(p)) {
// If p is not large enough to hold a single entry,
// return an error so the client can increase the
// size of p and try again.
return 0, ErrTranslateReadTargetUndersized
} else if int64(len(p)) > max {
// Shorten buffer to maximum read size.
p = p[:max]
}

19
utils_internal_test.go
View file

@ -24,6 +24,7 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
)
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -371,10 +372,26 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.holder.applySchema(instr.Schema); err != nil {
if err := destCluster.holder.applySchema(instr.NodeStatus.Schema); err != nil {
return err
}
// Sync available shards.
for _, is := range instr.NodeStatus.Indexes {
for _, fs := range is.Fields {
f := destCluster.holder.Field(is.Name, fs.Name)
// if we don't know about a field locally, log an error because
// fields should be created and synced prior to shard creation
if f == nil {
continue
}
if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil {
return errors.Wrap(err, "adding remote available shards")
}
}
}
for _, src := range instr.Sources {
srcCluster := t.clusterByID(src.Node.ID)

36
view.go
View file

@ -206,37 +206,45 @@ func (v *view) recalculateCaches() {
// CreateFragmentIfNotExists returns a fragment in the view by shard.
func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
v.mu.Lock()
defer v.mu.Unlock()
return v.createFragmentIfNotExists(shard)
frag, msg, err := v.createFragmentIfNotExists(shard)
// if msg is not nil, then a new shard was created
if err == nil && msg != nil {
// Broadcast a message that a new max shard was just created.
if err = v.broadcaster.SendSync(msg); err != nil {
frag.close()
return nil, errors.Wrap(err, "sending createshard message")
}
v.mu.Lock()
v.fragments[shard] = frag
v.mu.Unlock()
}
return frag, err
}
func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) {
func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardMessage, error) {
v.mu.Lock()
defer v.mu.Unlock()
// Find fragment in cache first.
if frag := v.fragments[shard]; frag != nil {
return frag, nil
return frag, nil, nil
}
// Initialize and open fragment.
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return nil, errors.Wrap(err, "opening fragment")
return nil, nil, errors.Wrap(err, "opening fragment")
}
frag.RowAttrStore = v.rowAttrStore
// Broadcast a message that a new max shard was just created.
if err := v.broadcaster.SendSync(&CreateShardMessage{
msg := &CreateShardMessage{
Index: v.index,
Field: v.field,
Shard: shard,
}); err != nil {
frag.close()
return nil, errors.Wrap(err, "sending createshard message")
}
// Save to lookup.
v.fragments[shard] = frag
return frag, nil
return frag, msg, nil
}
func (v *view) newFragment(path string, shard uint64) *fragment {