scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1503 from molecula/disco

Browse source 
DisCo
This commit is contained in:
Travis Turner 2021-03-04 17:24:42 -06:00 • committed by GitHub
commit fa9e17878f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
154 changed files with 9942 additions and 17113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/config.yml
View file

@ -218,7 +218,7 @@ workflows:
- setup
matrix:
parameters:
test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_bolt"]
test_make_target: ["test-race", "test-txstore-rbf_bolt"]
- test:
name: test-shardwidth-22
context: molecula

8
Makefile
View file

@ -248,13 +248,13 @@ docker-test:
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
@ -336,8 +336,8 @@ install-gometalinter:
GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
test-txstore-rbf:
PILOSA_TXSRC=rbf $(MAKE) testv-race
PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race
test-txstore-rbf_bolt:
PILOSA_TXSRC=rbf_bolt $(MAKE) testv-race
PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race

335
api.go
View file

@ -32,9 +32,11 @@ import (
"sync"
"time"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -43,6 +45,9 @@ import (
// API provides the top level programmatic interface to Pilosa. It is usually
// wrapped by a handler which provides an external interface (e.g. HTTP).
type API struct {
mu sync.Mutex
closed bool // protected by mu
holder *Holder
cluster *cluster
server *Server
@ -108,11 +113,12 @@ func NewAPI(opts ...apiOption) (*API, error) {
// validAPIMethods specifies the api methods that are valid for each
// cluster state.
var validAPIMethods = map[string]map[apiMethod]struct{}{
ClusterStateStarting: methodsCommon,
ClusterStateNormal: appendMap(methodsCommon, methodsNormal),
ClusterStateDegraded: appendMap(methodsCommon, methodsNormal),
ClusterStateResizing: appendMap(methodsCommon, methodsResizing),
var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{
disco.ClusterStateStarting: methodsCommon,
disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal),
disco.ClusterStateDegraded: appendMap(methodsCommon, methodsDegraded),
disco.ClusterStateResizing: appendMap(methodsCommon, methodsResizing),
disco.ClusterStateDown: methodsCommon,
}
func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} {
@ -127,7 +133,10 @@ func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} {
}
func (api *API) validate(f apiMethod) error {
state := api.cluster.State()
state, err := api.cluster.State()
if err != nil {
return errors.Wrap(err, "getting cluster state")
}
if _, ok := validAPIMethods[state][f]; ok {
return nil
}
@ -136,6 +145,14 @@ func (api *API) validate(f apiMethod) error {
// Close closes the api and waits for it to shutdown.
func (api *API) Close() error {
// only close once
api.mu.Lock()
defer api.mu.Unlock()
if api.closed {
return nil
}
api.closed = true
close(api.importWork)
api.importWorkersWG.Wait()
api.tracker.Stop()
@ -202,34 +219,19 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
return nil, errors.Wrap(err, "validating api method")
}
if !api.holder.isCoordinator() {
if err := api.server.defaultClient.CreateIndex(ctx, indexName, options); err != nil {
return nil, errors.Wrap(err, "forwarding CreateIndex to coordinator")
}
return api.holder.Index(indexName), nil
// Populate the create index message.
cim := &CreateIndexMessage{
Index: indexName,
CreatedAt: timestamp(),
Meta: options,
}
// Create index.
index, err := api.holder.CreateIndex(indexName, options)
index, err := api.holder.CreateIndexAndBroadcast(cim)
if err != nil {
return nil, errors.Wrap(err, "creating index")
}
createdAt := timestamp()
index.mu.Lock()
index.createdAt = createdAt
index.mu.Unlock()
// Send the create index message to all nodes.
err = api.server.SendSync(
&CreateIndexMessage{
Index: indexName,
CreatedAt: createdAt,
Meta: &options,
})
if err != nil {
return nil, errors.Wrap(err, "sending CreateIndex message")
}
api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
return index, nil
}
@ -289,20 +291,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
return nil, errors.Wrap(err, "validating api method")
}
// Apply functional options.
fo := FieldOptions{}
for _, opt := range opts {
err := opt(&fo)
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "applying option"))
}
}
if !api.holder.isCoordinator() {
if err := api.server.defaultClient.CreateFieldWithOptions(ctx, indexName, fieldName, fo); err != nil {
return nil, errors.Wrap(err, "forwarding CreateField to coordinator")
}
return api.holder.Field(indexName, fieldName), nil
// Apply and validate functional options.
fo, err := newFieldOptions(opts...)
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "applying option"))
}
// Find index.
@ -311,27 +303,20 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
return nil, newNotFoundError(ErrIndexNotFound, indexName)
}
// Populate the create field message.
cfm := &CreateFieldMessage{
Index: indexName,
Field: fieldName,
CreatedAt: timestamp(),
Meta: fo,
}
// Create field.
field, err := index.CreateField(fieldName, opts...)
field, err := index.CreateFieldAndBroadcast(cfm)
if err != nil {
return nil, errors.Wrap(err, "creating field")
}
createdAt := timestamp()
field.mu.Lock()
field.createdAt = createdAt
field.mu.Unlock()
// Send the create field message to all nodes.
err = api.server.SendSync(&CreateFieldMessage{
Index: indexName,
Field: fieldName,
CreatedAt: createdAt,
Meta: &fo,
})
if err != nil {
api.server.logger.Printf("problem sending CreateField message: %s", err)
return nil, errors.Wrap(err, "sending CreateField message")
}
api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return field, nil
}
@ -521,7 +506,10 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
qcx := api.Txf().NewQcx()
defer qcx.Abort()
nodes := api.cluster.shardNodes(indexName, shard)
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
nodes := snap.ShardNodes(indexName, shard)
errCh := make(chan error, len(nodes))
for _, node := range nodes {
node := node
@ -643,9 +631,12 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
return errors.Wrap(err, "validating api method")
}
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
// Validate that this handler owns the shard.
if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
if !snap.OwnsShard(api.NodeID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.NodeID(), shard, indexName)
return ErrClusterDoesNotOwnShard
}
@ -692,7 +683,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
if index.Keys() {
if store := index.TranslateStore(api.cluster.idPartition(indexName, columnID)); store == nil {
if store := index.TranslateStore(snap.IDToShardPartition(indexName, columnID)); store == nil {
return errors.Wrap(err, "partition does not exist")
} else if colStr, err = store.TranslateID(columnID); err != nil {
return errors.Wrap(err, "translating column")
@ -718,7 +709,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// ShardNodes returns the node and all replicas which should contain a shard's data.
func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) {
func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*topology.Node, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes")
defer span.Finish()
@ -726,7 +717,10 @@ func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64)
return nil, errors.Wrap(err, "validating api method")
}
return api.cluster.shardNodes(indexName, shard), nil
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
return snap.ShardNodes(indexName, shard), nil
}
// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to
@ -831,23 +825,29 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i
}
// Hosts returns a list of the hosts in the cluster including their ID,
// URL, and which is the coordinator.
func (api *API) Hosts(ctx context.Context) []*Node {
// URL, and which is the primary.
func (api *API) Hosts(ctx context.Context) []*topology.Node {
span, _ := tracing.StartSpanFromContext(ctx, "API.Hosts")
defer span.Finish()
return api.cluster.Nodes()
}
func (api *API) HostStates(ctx context.Context) map[string]string {
span, _ := tracing.StartSpanFromContext(ctx, "API.HostStates")
defer span.Finish()
return api.cluster.AllNodeStates()
// Node gets the ID, URI and primary status for this particular node.
func (api *API) Node() *topology.Node {
return api.server.node()
}
// Node gets the ID, URI and coordinator status for this particular node.
func (api *API) Node() *Node {
node := api.server.node()
return &node
// NodeID gets the ID alone, so it doesn't have to do a complete lookup
// of the node, searching by its ID, to return the ID it searched for.
func (api *API) NodeID() string {
return api.server.nodeID
}
// PrimaryNode returns the primary node for the cluster.
func (api *API) PrimaryNode() *topology.Node {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
return snap.PrimaryFieldTranslationNode()
}
// NodeUsage represents all usage measurements for one node.
@ -1019,10 +1019,19 @@ func (err MessageProcessingError) Unwrap() error {
// Schema returns information about each index in Pilosa including which fields
// they contain.
func (api *API) Schema(ctx context.Context) []*IndexInfo {
func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error) {
if err := api.validate(apiSchema); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
defer span.Finish()
return api.holder.Schema(false)
if withViews {
return api.holder.Schema()
}
return api.holder.limitedSchema()
}
// SchemaDetails returns information about each index in Pilosa including which
@ -1030,7 +1039,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo {
func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
defer span.Finish()
schema := api.holder.Schema(false)
schema, err := api.holder.Schema()
if err != nil {
return nil, errors.Wrap(err, "getting schema")
}
for _, index := range schema {
for _, field := range index.Fields {
q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name)
@ -1064,29 +1076,12 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
return errors.Wrap(err, "validating api method")
}
// set CreatedAt for indexes and fields (if empty), and then apply schema.
for _, index := range s.Indexes {
if index.CreatedAt == 0 {
index.CreatedAt = timestamp()
}
for _, field := range index.Fields {
if field.CreatedAt == 0 {
field.CreatedAt = timestamp()
}
}
err := api.holder.applySchema(s)
if err != nil {
return errors.Wrap(err, "applying schema")
}
if !remote {
nodes := api.cluster.Nodes()
for i, node := range nodes {
err := api.server.defaultClient.PostSchema(ctx, &node.URI, s, true)
if err != nil {
return errors.Wrapf(err, "forwarding post schema to node %d of %d", i+1, len(nodes))
}
}
}
return errors.Wrap(api.holder.applySchema(s), "applying schema")
return nil
}
// Views returns the views in the given field.
@ -1379,7 +1374,7 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts .
return nil
}
// Import bulk imports data into a particular index,field,shard.
// ImportWithTx bulk imports data into a particular index,field,shard.
func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.Import")
defer span.Finish()
@ -1407,7 +1402,7 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
"field", req.Field)
// Unless explicitly ignoring key validation (meaning keys have been
// translated to ids in a previous step at the coordinator node), then
// translated to ids in a previous step at the primary node), then
// check to see if keys need translation.
if !options.IgnoreKeyCheck {
// Translate row keys.
@ -1514,7 +1509,7 @@ func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueReque
return api.ImportValueWithTx(ctx, qcx, req, opts...)
}
// ImportValue bulk imports values into a particular field.
// ImportValueWithTx bulk imports values into a particular field.
func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) (err0 error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.ImportValue")
defer span.Finish()
@ -1546,7 +1541,7 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
"index", req.Index,
"field", req.Field)
// Unless explicitly ignoring key validation (meaning keys have been
// translate to ids in a previous step at the coordinator node), then
// translate to ids in a previous step at the primary node), then
// check to see if keys need translation.
if !options.IgnoreKeyCheck {
// Translate column keys.
@ -1773,9 +1768,11 @@ func (api *API) LongQueryTime() time.Duration {
}
func (api *API) validateShardOwnership(indexName string, shard uint64) error {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
// Validate that this handler owns the shard.
if !api.cluster.ownsShard(api.Node().ID, indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.Node().ID, shard, indexName)
if !snap.OwnsShard(api.NodeID(), indexName, shard) {
api.server.logger.Printf("node %s does not own shard %d of index %s", api.NodeID(), shard, indexName)
return ErrClusterDoesNotOwnShard
}
return nil
@ -1800,60 +1797,26 @@ func (api *API) indexField(indexName string, fieldName string, shard uint64) (*I
return index, field, nil
}
// SetCoordinator makes a new Node the cluster coordinator.
func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.SetCoordinator")
defer span.Finish()
if err := api.validate(apiSetCoordinator); err != nil {
return nil, nil, errors.Wrap(err, "validating api method")
}
oldNode = api.cluster.nodeByID(api.cluster.Coordinator)
newNode = api.cluster.nodeByID(id)
if newNode == nil {
return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node")
}
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == api.Node().ID {
return oldNode, newNode, api.cluster.setCoordinator(newNode)
}
// Send the set-coordinator message to new node.
err = api.server.SendTo(
newNode,
&SetCoordinatorMessage{
New: newNode,
})
if err != nil {
return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err)
}
return oldNode, newNode, nil
}
// RemoveNode puts the cluster into the "RESIZING" state and begins the job of
// removing the given node.
func (api *API) RemoveNode(id string) (*Node, error) {
func (api *API) RemoveNode(id string) (*topology.Node, error) {
if err := api.validate(apiRemoveNode); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
removeNode := api.cluster.nodeByID(id)
if removeNode == nil {
if !api.cluster.topologyContainsNode(id) {
return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
}
removeNode = &Node{
ID: id,
}
if api.cluster.disCo.ID() == id {
return nil, errors.Wrapf(ErrPreconditionFailed, "cannot issue node removal request to the node being removed, id=%s", id)
}
// Start the resize process (similar to NodeJoin)
err := api.cluster.nodeLeave(id)
if err != nil {
return removeNode, errors.Wrap(err, "calling node leave")
removeNode := api.cluster.nodeByID(id)
if removeNode == nil {
return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
}
if err := api.cluster.removeNode(id); err != nil {
return nil, errors.Wrapf(err, "removing node %s", id)
}
return removeNode, nil
}
@ -1863,14 +1826,17 @@ func (api *API) ResizeAbort() error {
return errors.Wrap(err, "validating api method")
}
err := api.cluster.completeCurrentJob(resizeJobStateAborted)
return errors.Wrap(err, "complete current job")
return api.cluster.resizeAbortAndBroadcast()
}
// State returns the cluster state which is usually "NORMAL", but could be
// "STARTING", "RESIZING", or potentially others. See cluster.go for more
// "STARTING", "RESIZING", or potentially others. See disco.go for more
// details.
func (api *API) State() string {
func (api *API) State() (disco.ClusterState, error) {
if err := api.validate(apiState); err != nil {
return "", errors.Wrap(err, "validating api method")
}
return api.cluster.State()
}
@ -1906,10 +1872,10 @@ func (api *API) Info() serverInfo {
CPUMHz: mhz,
CPUType: si.CPUModel(),
Memory: mem,
TxSrc: api.holder.txf.TxType(),
StorageBackend: api.holder.txf.TxType(),
ReplicaN: api.cluster.ReplicaN,
ShardHash: api.cluster.Hasher.Name(),
KeyHash: api.cluster.Topology.Hasher.Name(),
KeyHash: api.cluster.Hasher.Name(),
}
}
@ -2093,7 +2059,10 @@ func (api *API) CreateFieldKeys(ctx context.Context, index, field string, keys .
// PrimaryReplicaNodeURL returns the URL of the cluster's primary replica.
func (api *API) PrimaryReplicaNodeURL() url.URL {
node := api.cluster.PrimaryReplicaNode()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
node := snap.PrimaryReplicaNode(api.NodeID())
if node == nil {
return url.URL{}
}
@ -2201,11 +2170,14 @@ func (api *API) ReserveIDs(key IDAllocKey, session [32]byte, offset uint64, coun
return nil, errors.Wrap(err, "validating api method")
}
if api.holder.isCoordinator() {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) {
return api.holder.ida.reserve(key, session, offset, count)
}
return nil, errors.New("cannot reserve IDs on a non-coordinator node")
return nil, errors.New("cannot reserve IDs on a non-primary node")
}
func (api *API) CommitIDs(key IDAllocKey, session [32]byte, count uint64) error {
@ -2213,11 +2185,14 @@ func (api *API) CommitIDs(key IDAllocKey, session [32]byte, count uint64) error
return errors.Wrap(err, "validating api method")
}
if api.holder.isCoordinator() {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) {
return api.holder.ida.commit(key, session, count)
}
return errors.New("cannot commit IDs on a non-coordinator node")
return errors.New("cannot commit IDs on a non-primary node")
}
func (api *API) ResetIDAlloc(index string) error {
@ -2225,11 +2200,14 @@ func (api *API) ResetIDAlloc(index string) error {
return errors.Wrap(err, "validating api method")
}
if api.holder.isCoordinator() {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
if !snap.IsPrimaryFieldTranslationNode(api.NodeID()) {
return api.holder.ida.reset(index)
}
return errors.New("cannot reset IDs on a non-coordinator node")
return errors.New("cannot reset IDs on a non-primary node")
}
// TranslateIndexDB is an internal function to load the index keys database
@ -2260,7 +2238,7 @@ type serverInfo struct {
CPUPhysicalCores int `json:"cpuPhysicalCores"`
CPULogicalCores int `json:"cpuLogicalCores"`
CPUMHz int `json:"cpuMHz"`
TxSrc string `json:"txSrc"`
StorageBackend string `json:"storageBackend"`
}
type apiMethod int
@ -2293,10 +2271,9 @@ const (
apiRecalculateCaches
apiRemoveNode
apiResizeAbort
//apiSchema // not implemented
apiSetCoordinator
apiSchema
apiShardNodes
//apiState // not implemented
apiState
//apiStatsWithTags // not implemented
//apiVersion // not implemented
apiViews
@ -2314,13 +2291,35 @@ const (
var methodsCommon = map[apiMethod]struct{}{
apiClusterMessage: {},
apiSetCoordinator: {},
apiState: {},
}
var methodsResizing = map[apiMethod]struct{}{
apiFragmentData: {},
apiTranslateData: {},
apiResizeAbort: {},
apiSchema: {},
}
var methodsDegraded = map[apiMethod]struct{}{
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiFieldAttrDiff: {},
apiIndex: {},
apiIndexAttrDiff: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},
apiShardNodes: {},
apiSchema: {},
apiViews: {},
apiStartTransaction: {},
apiFinishTransaction: {},
apiTransactions: {},
apiGetTransaction: {},
apiActiveQueries: {},
}
var methodsNormal = map[apiMethod]struct{}{
@ -2343,6 +2342,7 @@ var methodsNormal = map[apiMethod]struct{}{
apiRecalculateCaches: {},
apiRemoveNode: {},
apiShardNodes: {},
apiSchema: {},
apiViews: {},
apiApplySchema: {},
apiStartTransaction: {},
@ -2351,7 +2351,4 @@ var methodsNormal = map[apiMethod]struct{}{
apiGetTransaction: {},
apiActiveQueries: {},
apiPastQueries: {},
apiIDReserve: {},
apiIDCommit: {},
apiIDReset: {},
}

42
api_test.go
View file

@ -161,7 +161,6 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
t.Fatal(err)
}
}
})
}
@ -225,7 +224,7 @@ func TestAPI_Import(t *testing.T) {
colKeys = colKeys[:N]
// Import data with keys to the coordinator (node0) and verify that it gets
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: indexName,
@ -270,7 +269,11 @@ func TestAPI_Import(t *testing.T) {
// Relies on the previous test creating an index with TrackExistence and
// adding some data.
t.Run("SchemaHasNoExists", func(t *testing.T) {
schema := m1.API.Schema(context.Background())
schema, err := m1.API.Schema(context.Background(), false)
if err != nil {
t.Fatal(err)
}
for _, f := range schema[0].Fields {
if f.Name == "_exists" {
t.Fatalf("found _exists field in schema")
@ -279,7 +282,6 @@ func TestAPI_Import(t *testing.T) {
t.Fatalf("found internal field '%s' in schema output", f.Name)
}
}
})
}
@ -300,6 +302,7 @@ func TestAPI_ImportValue(t *testing.T) {
)
defer c.Close()
coord := c.GetPrimary()
m0 := c.GetNode(0)
m1 := c.GetNode(1)
@ -308,11 +311,11 @@ func TestAPI_ImportValue(t *testing.T) {
index := "valck"
field := "f"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
_, err = coord.API.CreateField(ctx, index, field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -326,7 +329,7 @@ func TestAPI_ImportValue(t *testing.T) {
// Column keys are sharded so their order is not guaranteed.
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the coordinator (node0) and verify that it gets
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
@ -335,8 +338,8 @@ func TestAPI_ImportValue(t *testing.T) {
Values: values,
}
qcx := m0.API.Txf().NewQcx()
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
qcx := coord.API.Txf().NewQcx()
if err := coord.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
@ -377,7 +380,7 @@ func TestAPI_ImportValue(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
// Generate some keyed records.
// Generate some records.
values := []float64{}
colIDs := []uint64{}
for i := 0; i < 10; i++ {
@ -385,8 +388,8 @@ func TestAPI_ImportValue(t *testing.T) {
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
// Import data with keys to node1 and verify that it gets translated and
// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -432,16 +435,16 @@ func TestAPI_ImportValue(t *testing.T) {
fgnIndex := "fgnvalstr"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
_, err = coord.API.CreateIndex(ctx, fgnIndex, pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating foreign index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field,
_, err = coord.API.CreateField(ctx, index, field,
pilosa.OptFieldTypeInt(0, math.MaxInt64),
pilosa.OptFieldForeignIndex(fgnIndex),
)
@ -458,8 +461,9 @@ func TestAPI_ImportValue(t *testing.T) {
colIDs = append(colIDs, uint64(i))
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
// Import data with keys to the node0 and verify that it gets translated
// and forwarded to the owner of shard 0 (node1; because of
// offsetModHasher)
req := &pilosa.ImportValueRequest{
Index: index,
Field: field,
@ -474,8 +478,8 @@ func TestAPI_ImportValue(t *testing.T) {
pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
// Query node1.
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
t.Fatal(err)
} else if ids := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected columns: observerd %+v; expected '%+v'", ids, []uint64{1})

24
apimethod_string.go
View file

@ -30,19 +30,25 @@ func _() {
_ = x[apiRecalculateCaches-19]
_ = x[apiRemoveNode-20]
_ = x[apiResizeAbort-21]
_ = x[apiSetCoordinator-22]
_ = x[apiSchema-22]
_ = x[apiShardNodes-23]
_ = x[apiViews-24]
_ = x[apiApplySchema-25]
_ = x[apiStartTransaction-26]
_ = x[apiFinishTransaction-27]
_ = x[apiTransactions-28]
_ = x[apiGetTransaction-29]
_ = x[apiState-24]
_ = x[apiViews-25]
_ = x[apiApplySchema-26]
_ = x[apiStartTransaction-27]
_ = x[apiFinishTransaction-28]
_ = x[apiTransactions-29]
_ = x[apiGetTransaction-30]
_ = x[apiActiveQueries-31]
_ = x[apiPastQueries-32]
_ = x[apiIDReserve-33]
_ = x[apiIDCommit-34]
_ = x[apiIDReset-35]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransaction"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 332, 345, 353, 367, 386, 406, 421, 438}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 197, 213, 222, 236, 244, 260, 268, 288, 301, 315, 324, 337, 345, 353, 367, 386, 406, 421, 438, 454, 468, 480, 491, 501}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

9
bolt.go
View file

@ -29,9 +29,8 @@ import (
"time"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/rbf"
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/storage"
// On Bolt only, we still use the long txkey, because
// this allows Max() to work readily.
@ -130,7 +129,7 @@ func boltPath(path string) string {
// if one does not exist for its bpath. Otherwise it returns
// the existing instance. This insures only one boltDB
// per bpath in this pilosa node.
func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rbfcfg.Config) (DBWrapper, error) {
func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
path := boltPath(path0)
r.mu.Lock()
@ -171,7 +170,7 @@ func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb
// re-sync during recovery.
// NoFreelistSync bool
if rbfcfg != nil && !rbfcfg.FsyncEnabled {
if cfg != nil && !cfg.FsyncEnabled {
db.NoSync = true
db.NoFreelistSync = true
} else {
@ -479,7 +478,7 @@ func (tx *BoltTx) Type() string {
}
func (tx *BoltTx) UseRowCache() bool {
return rbf.EnableRowCache()
return storage.EnableRowCache()
}
// Pointer gives us a memory address for the underlying transaction for debugging.

830
boltdb/translate.go
View file

@ -22,13 +22,11 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"sync"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pkg/errors"
"github.com/zeebo/blake3"
bolt "go.etcd.io/bbolt"
"runtime/pprof"
@ -98,10 +96,6 @@ type TranslateStore struct {
Path string
}
func (s *TranslateStore) GetStorePath() string {
return s.Path
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
return &TranslateStore{
@ -578,827 +572,3 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string {
}
return string(boltKey)
}
func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) {
sum = &pilosa.TranslatorSummary{}
hasher := blake3.New()
err = s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketKeys)
if bkt == nil {
panic("bucketKeys not found")
}
cur := bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
input := append(k, v...)
_, _ = hasher.Write(input)
sum.KeyCount++
}
bkt = tx.Bucket(bucketIDs)
if bkt == nil {
panic("bucketIDs not found")
}
cur = bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
input := append(k, v...)
_, _ = hasher.Write(input)
sum.IDCount++
}
return nil
})
if err != nil {
return nil, err
}
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
sum.Checksum = string(buf[:])
return sum, nil
}
func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) {
sum = &pilosa.TranslatorSummary{}
hasher := blake3.New()
if partitionID != s.partitionID {
panic(fmt.Sprintf("inconsistent partitionID arg %v with TranslateStore.paritionID %v", partitionID, s.partitionID))
}
firstPrimary := topo.PrimaryNodeIndex(partitionID)
err = s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketKeys) // key -> id
if bkt == nil {
panic("bucketKeys not found")
}
cur := bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
input := append(k, v...)
//vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), k='%v', v=%x", partitionID, s.Path, string(k), v)
_, _ = hasher.Write(input)
sum.KeyCount++
}
bkt = tx.Bucket(bucketIDs) // id -> key
if bkt == nil {
panic("bucketIDs not found")
}
cur = bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
// should the primary be the same for each key in this partition?
id := btou64(k)
shard := id / pilosa.ShardWidth
ks := string(v)
primary := topo.GetPrimaryForColKeyTranslation(s.index, ks)
if firstPrimary < 0 {
firstPrimary = primary
} else {
if primary != firstPrimary {
panic(fmt.Sprintf("s.index='%v' primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v; topo='%v'", s.index, primary, firstPrimary, ks, id, shard, partitionID, topo.String()))
}
}
// Verify the invariant that the primaries agree. Just a sanity check.
primaryForShard := topo.GetPrimaryForShardReplication(s.index, shard)
if primaryForShard != firstPrimary {
panic(fmt.Sprintf("primaryForShard (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primaryForShard, firstPrimary, ks, id, shard, partitionID))
}
input := append(k, v...)
//vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), idBucket id=%x key='%v'", partitionID, s.Path, id, ks)
_, _ = hasher.Write(input)
sum.IDCount++
}
return nil
})
if err != nil {
return nil, err
}
sum.PrimaryNodeIndex = firstPrimary
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
sum.Checksum = string(buf[:])
return sum, nil
}
func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error {
return s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketKeys)
if bkt == nil {
panic("bucketKeys not found")
}
cur := bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
walk(string(k), btou64(v))
}
return nil
})
}
func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error {
return s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketIDs)
if bkt == nil {
panic("bucketIDs not found")
}
cur := bkt.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
walk(string(v), btou64(k))
}
return nil
})
}
// call s.notifyWrite() when done
func (s *TranslateStore) SetFwdRevMaps(tx *bolt.Tx, fwd map[string]uint64, rev map[uint64]string) (err error) {
localTx := false
if tx == nil {
localTx = true
tx, err = s.db.Begin(true)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
}
// reinitialize buckets
err = tx.DeleteBucket(bucketKeys)
if err != nil {
return err
}
err = tx.DeleteBucket(bucketIDs)
if err != nil {
return err
}
if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
return err
}
key2id := tx.Bucket(bucketKeys)
for k, v := range fwd {
err := key2id.Put([]byte(k), u64tob(v))
if err != nil {
return err
}
}
id2key := tx.Bucket(bucketIDs)
for k, v := range rev {
err := id2key.Put(u64tob(k), []byte(v))
if err != nil {
return err
}
}
if localTx {
return tx.Commit()
}
return nil
}
func (s *TranslateStore) GetFwdRevMaps(tx *bolt.Tx) (fwd map[string]uint64, rev map[uint64]string, err error) {
fwd = make(map[string]uint64)
rev = make(map[uint64]string)
key2id := tx.Bucket(bucketKeys)
err = key2id.ForEach(func(k, v []byte) error {
fwd[string(k)] = btou64(v)
return nil
})
if err != nil {
return
}
id2key := tx.Bucket(bucketIDs)
err = id2key.ForEach(func(k, v []byte) error {
rev[btou64(k)] = string(v)
return nil
})
return
}
//var vv = pilosa.VV
// helpers for repair
// muint64 holds multiple unit64
type muint64 struct {
slc []uint64
}
func (m *muint64) String() (s string) {
for _, e := range m.slc {
s += fmt.Sprintf("%x, ", e)
}
return
}
// mstring holds multiple strings
type mstring struct {
slc []string
}
func (m *mstring) String() (s string) {
for _, e := range m.slc {
s += e + ","
}
return
}
func addToProblemKeys(problemKeys map[string]*muint64, k string, v uint64, noValue bool) {
mu, already := problemKeys[k]
if !already {
mu = &muint64{}
problemKeys[k] = mu
}
if !noValue {
mu.slc = append(mu.slc, v)
}
}
func addToProblemIDs(problemIDs map[uint64]*mstring, k uint64, v string, noValue bool) {
mu, already := problemIDs[k]
if !already {
mu = &mstring{}
problemIDs[k] = mu
}
if !noValue {
mu.slc = append(mu.slc, v)
}
}
// only actually apply the fixes if applyKeyRepairs is true.
// if anything changed, return changed == true.
func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) {
// strategy: get the full set of keys; the domain keys from
// the forward key->id mapping, and the range keys from the reverse id->key mapping.
// Then march through them and make sure they are mapped correctly.
// At the moment we do try to reuse dangling IDs instead of making
// new ones. This might not always be possible, but we hope for
// now that it suffices b/c it minimizes the amount of fragment
// re-write we may have to do.
/*
// ============ profiling ===============
fd, err := ioutil.TempFile(".", "cpu.prof")
if err != nil {
panic(err)
}
_ = pprof.StartCPUProfile(fd)
defer func() {
pprof.StopCPUProfile()
fd.Close()
}()
// ============ end profiling ===============
*/
tx, err := s.db.Begin(true)
if err != nil {
return false, err
}
defer func() {
_ = tx.Rollback()
}()
fwd, rev, err := s.GetFwdRevMaps(tx)
if err != nil {
return false, err
}
// place to store the correct stuff.
//
// fwd2, rev2: new, repaired versions.
// INVAR: they only contain (correct) invertible mappings.
fwd2 := make(map[string]uint64)
rev2 := make(map[uint64]string)
// and a place to store the problems.
problemKeys := make(map[string]*muint64)
problemIDs := make(map[uint64]*mstring)
fwdscan:
for k, v := range fwd {
_, already := fwd2[k]
if already {
// k has already been repaired. don't worry about further.
continue fwdscan
} else {
// if its already invertible, then just keep it, no need to repair it
// INVAR: k is not in fwd2 (at least not yet).
rkey, ok := rev[v]
if !ok {
// k -> v -> X
addToProblemIDs(problemIDs, v, k, false)
addToProblemKeys(problemKeys, k, v, false)
continue fwdscan
}
if rkey == k {
// yay. a good, invertible, mapping. no repair needed.
if k == "" {
panic("bad empty key")
}
fwd2[k] = v
rev2[v] = k
continue fwdscan
}
// some kind of problem.
// what kind?
// Define problemKey as: 2nd key mapping to id already in fwd2.
// Define problemID as: 2nd ID mapping to key already in fwd2.
// k -> v -> rkey, and rkey != k.
v2, ok := fwd[rkey]
if ok {
// k -> v -> rkey -> v2, where v2 ?= v
if v2 == v {
// k -> v -> rkey -> v
// just have a problemKey in k.
if rkey == "" {
panic("bad empty rkey")
}
fwd2[rkey] = v
rev2[v] = rkey
addToProblemKeys(problemKeys, k, v, true)
continue fwdscan
}
// this is i = 1 test case. :)
// k -> v -> rkey -> v2, where v != v2, and k != rkey.
addToProblemKeys(problemKeys, k, v, false)
addToProblemIDs(problemIDs, v, rkey, false)
continue fwdscan
} else {
// k -> v -> rkey -> X(nil), and rkey != k.
addToProblemKeys(problemKeys, k, v, false)
addToProblemKeys(problemKeys, rkey, 0, true)
addToProblemIDs(problemIDs, v, rkey, false)
}
}
}
revscan:
for id, key := range rev {
k1, already := rev2[id]
_ = k1
if already {
// fine, already there.
continue
}
// if its already invertible, keep it.
rid, ok := fwd[key]
if !ok {
// id -> key -> X
addToProblemKeys(problemKeys, key, 0, true)
addToProblemIDs(problemIDs, id, key, false)
continue revscan
}
if rid == id {
// id -> key -> id. good. but should have been added to fwd2/rev2 above.
panic("should have been added to fwd2/rev2 above!")
} else {
// id -> key -> rid, where id != rid
// so rid -> ?
keyr, ok := rev[rid]
if !ok {
// id -> key -> rid -> X, where id != rid
addToProblemKeys(problemKeys, key, rid, false)
addToProblemKeys(problemKeys, key, id, false)
addToProblemIDs(problemIDs, rid, key, false)
continue revscan
}
if keyr == key {
// id -> key -> rid -> key. So rid is correct and id is dangling.
//
// Heuristic: ASSUME here, that the 2 consistent links key->rid->key are correct,
// and that the single id -> key is in the wrong. This DOESN'T HAVE
// TO BE THE CASE.
if key == "" {
panic("bad empty key")
}
rev2[rid] = key
fwd2[key] = rid
addToProblemIDs(problemIDs, id, "", true)
} else {
// this is test case i = 0. Must handle it.
// id -> key -> rid -> keyr, id != rid, keyr != key.
id2, ok := fwd[keyr]
if ok && id2 == rid {
// id -> key -> rid -> keyr -> rid, id != rid, keyr != key.
// so rid -> keyr -> rid is good.
if keyr == "" {
panic("bad empty keyr")
}
fwd2[keyr] = rid
rev2[rid] = keyr
// and id -> key -> rid is bad, b/c id != rid.
addToProblemKeys(problemKeys, key, rid, false)
addToProblemIDs(problemIDs, id, key, false)
continue revscan
}
// one of these 3 cases holds. all have the same treatment.
// 1) id2 == id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, keyr != key.
// 2) id2 != id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, id2 != id, keyr != key.
// 3) !ok: id -> key -> rid -> keyr -> X, id != rid, keyr != key.
addToProblemIDs(problemIDs, id, key, false)
addToProblemKeys(problemKeys, key, rid, false)
addToProblemIDs(problemIDs, rid, keyr, false)
}
}
}
//vv("problemKeys = '%v'", problemKeys)
//vv("problemIDs = '%v'", problemIDs)
newIDs := make(map[uint64]bool)
// assign new IDs to any problemKeys; but first
// try to reuse already allocated IDs that are just dangling.
loopProblemKeys:
for key, ids := range problemKeys {
// first try a minor repair, maybe it was just mssing from rev
// and we can avoid allocate another id.
// sanity check
v2, already := fwd2[key]
if already {
panic(fmt.Sprintf("should not get here since fwd2 is only correct invertibles: key='%v', v2='%x'", key, v2))
}
// INVAR: we have no correct mapping for key in fwd2.
// treat the danglers as "suggestions" for the correction.
for k, id := range ids.slc {
_ = k
_, already = rev2[id]
if !already {
// is this correct?
// id is not in rev2, and key is not in fwd2.
// therefore, we can add them both and maintain consistency.
//vv("add %v to fwd2", key)
if key == "" {
panic("bad empty key")
}
fwd2[key] = id
rev2[id] = key
continue loopProblemKeys
}
}
// INVAR: key -> ? don't know. We didn't find a usable suggestion for the id.
// yes, we get here. We have key. We are looking for a suitable id for it.
// can we get a usable id from the problemIDs?
found := false
suggestions:
for idp, mkeyp := range problemIDs {
for _, candk := range mkeyp.slc {
//vv("checking problemIDs, ipd=%x, candk='%v'; candk==key is %v", idp, candk, candk == key)
if candk == key {
// we have a suggestion from problemIDs that idp might work, doing key -> idp.
// Validate that this is possible.
k2, already := rev2[idp]
_ = k2
if already {
//vv("idp is already in rev2: idp=%v, k2=%v", idp, k2)
continue suggestions
}
// idp works. put it in the correct set.
if key == "" {
panic("bad empty key")
}
rev2[idp] = key
fwd2[key] = idp
found = true
break suggestions
}
}
}
if !found {
id2 := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
//vv("could not minor repair, allocating new id2 = %v instead", id2)
newIDs[id2] = true
if key == "" {
panic("bad empty key")
}
fwd2[key] = id2
rev2[id2] = key
}
} // end problemKeys
//for id, keys := range problemIDs {
//}
if verbose {
reportIfGainedOrLostIDs(s, fwd, fwd2, rev, rev2, newIDs)
reportIfGainedOrLostKeys(s, fwd, fwd2, rev, rev2)
}
adds, changes, changeIDs, err := makeStringKeyChanges(verbose, applyKeyRepairs, tx, s, topo, fwd, fwd2, rev, rev2, newIDs)
if err != nil {
return false, err
}
_, _, _ = adds, changes, changeIDs
//vv("changedIDs = '%#v'", changeIDs)
if len(adds) > 0 || len(changes) > 0 || len(changeIDs) > 0 || len(newIDs) > 0 {
changed = true
}
//vv("newIDs = '%#v'", newIDs)
//vv("fwd2 = '%#v'", fwd2)
//vv("rev2 = '%#v'", rev2)
err = tx.Commit()
if err == nil {
s.notifyWrite()
}
return changed, err
}
func reportIfGainedOrLostIDs(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string, newIDs map[uint64]bool) {
// get all IDs ever mentioned
before := make(map[uint64]bool)
after := make(map[uint64]bool)
for _, id := range fwd {
before[id] = true
}
for _, id := range fwd2 {
if !newIDs[id] {
after[id] = true
}
}
for id := range rev {
before[id] = true
}
for id := range rev2 {
if !newIDs[id] {
after[id] = true
}
}
nb := len(before)
na := len(after)
if nb != na {
fmt.Printf("# needs-repair: Num ID before %v != Num ID after %v, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", nb, na, s.Path, len(fwd), len(rev), len(fwd2), len(rev2))
}
if len(newIDs) > 0 {
fmt.Printf("# needs-repair: adding newIDs '%#v', for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", newIDs, s.Path, len(fwd), len(rev), len(fwd2), len(rev2))
}
}
func reportIfGainedOrLostKeys(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string) {
// get all IDs ever mentioned
nb := len(fwd)
na := len(fwd2)
if nb != na {
diffAB := mapDiffStrings(fwd, fwd2)
diffBA := mapDiffStrings(fwd2, fwd)
fmt.Printf("# needs-repair: Num Keys before != Num Keys after, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v. fwd - fwd2 = '%#v'; fwd2-fwd = '%#v'\n", s.Path, len(fwd), len(rev), len(fwd2), len(rev2), diffAB, diffBA)
}
}
// return A - B
func mapDiffStrings(mapA, mapB map[string]uint64) (r []string) {
for a := range mapA {
_, ok := mapB[a]
if !ok {
r = append(r, a)
}
}
sort.Strings(r)
return
}
type BeforeAfterKeyChange struct {
BeforeID uint64
AfterID uint64
}
type BeforeAfterIDChange struct {
IsDelete bool
IsAdd bool
BeforeString string
AfterString string
}
// do the minimal state update.
// fwd2 is the "after" map, all string keys repaired.
func makeStringKeyChanges(
verbose bool,
applyKeyRepairs bool,
tx *bolt.Tx,
s *TranslateStore,
topo *pilosa.Topology,
fwd, fwd2 map[string]uint64,
rev, rev2 map[uint64]string,
newIDs map[uint64]bool,
) (
adds map[string]uint64,
changeKeys map[string]*BeforeAfterKeyChange,
changeIDs map[uint64]*BeforeAfterIDChange,
err error,
) {
//vv("makeStringKeyChanges called")
//vv("fwd2 = '%#v'", fwd2)
//vv("rev2 = '%#v'", rev2)
//vv("fwd = '%#v'", fwd)
//vv("rev = '%#v'", rev)
var action string
if applyKeyRepairs {
action = "applying "
}
// addition of string key
adds = make(map[string]uint64)
// change of the mapping of key -> id.
changeKeys = make(map[string]*BeforeAfterKeyChange)
// changes to bucketIDs
changeIDs = make(map[uint64]*BeforeAfterIDChange)
localTx := false
if applyKeyRepairs && tx == nil {
localTx = true
tx, err = s.db.Begin(true)
if err != nil {
return
}
defer func() {
//vv("tx.Rollback happening")
_ = tx.Rollback()
}()
}
key2id := tx.Bucket(bucketKeys)
id2key := tx.Bucket(bucketIDs)
// make a copy of rev2 that we can delete from, to see if
// any additions left in rev2 need to be added after all of
// rev is analyzed.
rev2cp := make(map[uint64]string)
for id, k := range rev2 {
rev2cp[id] = k
}
// first we clean up any stale IDs from id2key. Then the fwd2 pass
// that follows will write to both key2id and id2key.
for id, key := range rev {
//vv("makeStringKeyChanges on rev2: id=%x -> key='%v'", id, key)
key2, ok := rev2[id]
if !ok {
changeIDs[id] = &BeforeAfterIDChange{IsDelete: true}
if verbose {
fmt.Printf("# %vkey-translation-delete-id: (id %x -> %v). Remaining for that key: ('%v' -> %x)\n", action, id, key, key, fwd2[key])
}
if applyKeyRepairs {
u := u64tob(id)
err = id2key.Delete(u)
if err != nil {
return
}
}
continue
}
delete(rev2cp, id)
if key2 != key {
u := u64tob(id)
k := []byte(key2)
changeIDs[id] = &BeforeAfterIDChange{
BeforeString: key,
AfterString: key2,
}
if verbose {
fmt.Printf("# %vkey-translation-update-id: (id %x -> %v). fwd2 for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2])
}
if applyKeyRepairs {
err = id2key.Put(u, k)
if err != nil {
return
}
}
}
}
// anything leftover in rev2cp is stuff that is new, only
// in rev2 and not in rev. It needs to be added.
for id, key2 := range rev2cp {
u := u64tob(id)
k := []byte(key2)
changeIDs[id] = &BeforeAfterIDChange{
IsAdd: true,
//BeforeString: left empty
AfterString: key2,
}
if verbose {
fmt.Printf("# %vkey-translation-add-id: (id %x -> %v). Fwd for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2])
}
if applyKeyRepairs {
err = id2key.Put(u, k)
if err != nil {
return
}
}
}
// We assume here that fwd2 is a super-set of fwd. No string keys
// should be deleted in the repair. Confirm that.
for key, id := range fwd {
_, ok := fwd2[key]
if !ok {
panic(fmt.Sprintf("fwd2 is missing a string key from fwd. key='%v' -> id='%x'", key, id))
}
}
for key2, id2 := range fwd2 {
//vv("makeStringKeyChanges on fwd2, key2='%v', id2=%x", key2, id2)
isPrimary := false
if topo != nil {
primary := topo.GetPrimaryForColKeyTranslation(s.index, key2)
isPrimary = s.partitionID == primary
}
_ = isPrimary
id, ok := fwd[key2]
if !ok {
adds[key2] = id2
u2 := u64tob(id2)
k2 := []byte(key2)
if verbose {
fmt.Printf("# %vkey-translation-new-key: ('%v' -> %x) added: isPrimary: %v\n", action, key2, id2, isPrimary)
}
if applyKeyRepairs {
err = key2id.Put(k2, u2)
if err != nil {
return
}
err = id2key.Put(u2, k2)
if err != nil {
return
}
}
continue
}
if id != id2 {
changeKeys[key2] = &BeforeAfterKeyChange{
BeforeID: id,
AfterID: id2,
}
if verbose {
fmt.Printf("# %vkey-translation-change-id: ('%v' -> %x) changes to ('%v' -> %x); isPrimary: %v\n", action, key2, id, key2, id2, isPrimary)
}
if applyKeyRepairs {
u2 := u64tob(id2)
k2 := []byte(key2)
err = key2id.Put(k2, u2)
if err != nil {
return
}
err = id2key.Put(u2, k2)
if err != nil {
return
}
}
}
}
if localTx {
err = tx.Commit()
}
return
}
func (s *TranslateStore) DumpBolt(label string) {
fmt.Printf("dumping bolt %v : path='%v'\n", label, s.Path)
_ = s.KeyWalker(func(key string, col uint64) {
fmt.Printf("keyWalker: key '%v' -> col '%x'\n", key, col)
})
_ = s.IDWalker(func(key string, col uint64) {
fmt.Printf("idWalker: id '%x' -> key '%v'\n", col, key)
})
fmt.Printf("DONE with dumping bolt %v; path='%v'\n", label, s.Path)
}

173
boltdb/translate_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/topology"
)
//var vv = pilosa.VV
@ -540,7 +541,7 @@ func MustNewTranslateStore() *boltdb.TranslateStore {
panic(err)
}
s := boltdb.NewTranslateStore("I", "F", 0, pilosa.DefaultPartitionN)
s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN)
s.Path = f.Name()
return s
}
@ -627,173 +628,3 @@ func MustCloseTranslateStore(s *boltdb.TranslateStore) {
panic(err)
}
}
func TestCryptoHashPerKey(t *testing.T) {
s := MustOpenNewTranslateStore()
defer MustCloseTranslateStore(s)
// hash one translation
expect := map[int]string{
1: string([]byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12}),
2: string([]byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0}),
3: string([]byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9}),
}
for n := 1; n < 4; n++ {
var batch0 []string
for i := 0; i < n; i++ {
batch0 = append(batch0, fmt.Sprintf("key%d", i))
}
// Populate the store with the keys in batch0.
batch0IDs, err := s.TranslateKeys(batch0, true)
_ = batch0IDs
if err != nil {
t.Fatal(err)
}
// done with setup
sum, err := s.ComputeTranslatorSummaryCols(0, pilosa.NewTopology(&pilosa.Jmphasher{}, pilosa.DefaultPartitionN, 1, nil))
if err != nil {
panic(err)
}
nkey := sum.KeyCount
nid := sum.IDCount
observedChecksum := sum.Checksum
if nkey != n {
panic("wrong key count")
}
if nkey != nid {
panic("key count should match id count")
}
// shardwidth 22 has different hashes, of course.
if pilosa.ShardWidth == 20 {
expectedChecksum := expect[n]
if observedChecksum != expectedChecksum {
panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum))
}
}
}
}
func TestTranslateStore_RepairNonInvertibleStringKeyTranslation(t *testing.T) {
const N = 6
// before repair
var fwd [N]map[string]uint64
var rev [N]map[uint64]string
// after repair
var fwd2 [N]map[string]uint64
var rev2 [N]map[uint64]string
// case 0: forward is messed up (unlikely but check for it anyway, be sure we can repair)
// "key0" -> id 0 // correct.
// "key1" -> id 0 // wrong. after Repair, should see key1 -> 1 (0xec0002)
//
// id 0 -> "key0" // correct
// id 1 -> "key1" // correct
//
fwd[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00001}
rev[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
fwd2[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
rev2[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
// case 1: reverse is messed up (we have seen this in the past)
// "key0" -> id 0 // correct
// "key1" -> id 1 // correct
//
// id 0 -> "key0" // correct.
// id 1 -> "key0" // wrong. after Repair, should see id 1 -> "key1"
//
fwd[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
rev[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key0"}
fwd2[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
rev2[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
// case 2: only present in reverse.
fwd[2] = map[string]uint64{}
rev[2] = map[uint64]string{0xec00001: "key0"}
fwd2[2] = map[string]uint64{"key0": 0xec00001}
rev2[2] = map[uint64]string{0xec00001: "key0"}
// case 3: same thing. with camoflage.
fwd[3] = map[string]uint64{"key1": 0xec00002}
rev[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
fwd2[3] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
rev2[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
// case 4: only present in forward.
fwd[4] = map[string]uint64{"key0": 0xec00001}
rev[4] = map[uint64]string{}
fwd2[4] = map[string]uint64{"key0": 0xec00001}
rev2[4] = map[uint64]string{0xec00001: "key0"}
// case 5: same thing. with camoflage.
fwd[5] = map[string]uint64{"key0": 0xec00001}
rev[5] = map[uint64]string{0xec00002: "key1"}
fwd2[5] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002}
rev2[5] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"}
// case 6: we had an id, but b/c of the fix, that id is no longer used.
// now that id might still be used in the fragment for a column,
// and so we will need to remove that id/column from the fragment.
// encapsulated: "did it affect the state of the fields?"
for i := 0; i < 5; i++ {
//println("i = ", i)
s := MustOpenNewTranslateStore()
defer MustCloseTranslateStore(s)
if err := s.SetFwdRevMaps(nil, fwd[i], rev[i]); err != nil {
t.Fatal(err)
}
if err := verifyState("setup", i, s, fwd[i], rev[i]); err != nil {
t.Fatal(err)
}
var topo *pilosa.Topology
verbose := false
applyKeyRepairs := true
changed, err := s.RepairKeys(topo, verbose, applyKeyRepairs)
if err != nil {
t.Fatal(err)
}
if !changed {
t.Fatalf("expected changes!")
}
if err := verifyState("afterRepair", i, s, fwd2[i], rev2[i]); err != nil {
t.Fatal(err)
}
}
}
func verifyState(label string, i int, s *boltdb.TranslateStore, fwd map[string]uint64, rev map[uint64]string) error {
// verify the setup
const writable = true
for key, expectID := range fwd {
id, err := s.TranslateKey(key, !writable)
if err != nil {
return err
}
if id != expectID {
return fmt.Errorf("fwd %v problem. i=%v, for key '%v', expected %x, observed %x", label, i, key, expectID, id)
}
}
for id, expectKey := range rev {
key, err := s.TranslateID(id)
if err != nil {
return err
}
if key != expectKey {
return fmt.Errorf("rev %v problem. i=%v, for id '%x', expected %v, observed %v", label, i, id, expectKey, key)
}
}
return nil
}

41
broadcast.go
View file

@ -17,6 +17,7 @@ package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pkg/errors"
)
@ -26,11 +27,22 @@ type Serializer interface {
Unmarshal([]byte, Message) error
}
// NopSerializer represents a Serializer that doesn't do anything.
var NopSerializer Serializer = &nopSerializer{}
type nopSerializer struct{}
// Marshal is a no-op implementation of Serializer Marshal method.
func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
// Unmarshal is a no-op implementation of Serializer Unmarshal method.
func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
// broadcaster is an interface for broadcasting messages.
type broadcaster interface {
SendSync(Message) error
SendAsync(Message) error
SendTo(*Node, Message) error
SendTo(*topology.Node, Message) error
}
// Message is the interface implemented by all core pilosa types which can be serialized to messages.
@ -49,7 +61,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
func (nopBroadcaster) SendAsync(Message) error { return nil }
// SendTo is a no-op implementation of Broadcaster SendTo method.
func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil }
// Broadcast message types.
const (
@ -63,13 +75,14 @@ const (
messageTypeClusterStatus
messageTypeResizeInstruction
messageTypeResizeInstructionComplete
messageTypeSetCoordinator
messageTypeUpdateCoordinator
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeLoadSchemaMessage
messageTypeNodeEvent
messageTypeNodeStatus
messageTypeTransaction
messageTypeResizeNodeMessage
messageTypeResizeAbortMessage
)
// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@ -105,20 +118,22 @@ func getMessage(typ byte) Message {
return &ResizeInstruction{}
case messageTypeResizeInstructionComplete:
return &ResizeInstructionComplete{}
case messageTypeSetCoordinator:
return &SetCoordinatorMessage{}
case messageTypeUpdateCoordinator:
return &UpdateCoordinatorMessage{}
case messageTypeNodeState:
return &NodeStateMessage{}
case messageTypeRecalculateCaches:
return &RecalculateCaches{}
case messageTypeLoadSchemaMessage:
return &LoadSchemaMessage{}
case messageTypeNodeEvent:
return &NodeEvent{}
case messageTypeNodeStatus:
return &NodeStatus{}
case messageTypeTransaction:
return &TransactionMessage{}
case messageTypeResizeNodeMessage:
return &ResizeNodeMessage{}
case messageTypeResizeAbortMessage:
return &ResizeAbortMessage{}
default:
panic(fmt.Sprintf("unknown message type %d", typ))
}
@ -146,20 +161,22 @@ func getMessageType(m Message) byte {
return messageTypeResizeInstruction
case *ResizeInstructionComplete:
return messageTypeResizeInstructionComplete
case *SetCoordinatorMessage:
return messageTypeSetCoordinator
case *UpdateCoordinatorMessage:
return messageTypeUpdateCoordinator
case *NodeStateMessage:
return messageTypeNodeState
case *RecalculateCaches:
return messageTypeRecalculateCaches
case *LoadSchemaMessage:
return messageTypeLoadSchemaMessage
case *NodeEvent:
return messageTypeNodeEvent
case *NodeStatus:
return messageTypeNodeStatus
case *TransactionMessage:
return messageTypeTransaction
case *ResizeNodeMessage:
return messageTypeResizeNodeMessage
case *ResizeAbortMessage:
return messageTypeResizeAbortMessage
default:
panic(fmt.Sprintf("don't have type for message %#v", m))
}

93
client.go
View file

@ -18,6 +18,9 @@ import (
"context"
"io"
"time"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/topology"
)
// Bit represents the intersection of a row and a column. It can be specified by
@ -51,10 +54,10 @@ type InternalClient interface {
MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error
PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error)
Nodes(ctx context.Context) ([]*Node, error)
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error)
Nodes(ctx context.Context) ([]*topology.Node, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error
ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error
@ -67,69 +70,75 @@ type InternalClient interface {
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error)
RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error
FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error
StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
FinishTransaction(ctx context.Context, id string) (*Transaction, error)
Transactions(ctx context.Context) (map[string]*Transaction, error)
GetTransaction(ctx context.Context, id string) (*Transaction, error)
GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error)
GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error)
GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
}
//===============
// InternalQueryClient is the internal interface for querying a node.
type InternalQueryClient interface {
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error)
QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error)
TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error)
TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error)
TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error)
FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error)
FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error)
FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error)
CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error)
CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
}
type nopInternalQueryClient struct{}
func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) {
func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, ids []uint64) ([]string, error) {
func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) {
func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) {
func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *URI, index string, keys ...string) (map[string]uint64, error) {
func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *URI, index string, field string, keys ...string) (map[string]uint64, error) {
func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
@ -153,17 +162,17 @@ func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64,
return nil, nil
}
func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n nopInternalClient) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error {
func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error {
return nil
}
func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) {
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Nodes(ctx context.Context) ([]*Node, error) {
func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
@ -179,11 +188,11 @@ func (n nopInternalClient) ImportValue2(ctx context.Context, req *ImportValueReq
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}
func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error {
func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error {
return nil
}
@ -209,25 +218,25 @@ func (n nopInternalClient) CreateField(ctx context.Context, index, field string)
func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n nopInternalClient) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error {
func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
return nil
}
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) {
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) {
func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
@ -244,10 +253,10 @@ func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Tran
return nil, nil
}
func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *URI) (map[string]NodeUsage, error) {
func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
return nil, nil
}
func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *URI) ([]PastQueryStatus, error) {
func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
return nil, nil
}

2766
cluster.go
View file

File diff suppressed because it is too large Load diff

678
cluster_internal_test.go
View file

@ -15,42 +15,80 @@
package pilosa
import (
"bytes"
"fmt"
"math/rand"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
"strconv"
"strings"
"testing"
"testing/quick"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/gorilla/mux"
"github.com/pilosa/pilosa/v2/logger"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pkg/errors"
"github.com/pilosa/pilosa/v2/topology"
)
// GlobalPortMap avoids many races and port conflicts when setting
// up ports for test clusters. Used for tests only.
var globalPortMap *GlobalPortMapper
func init() {
globalPortMap = NewGlobalPortMapper(300)
}
// GlobalPortMapper maintains a pool of available ports by
// holding them open until GetPort() is called.
type GlobalPortMapper struct {
availPorts map[int]net.Listener
}
// reserve n ports
func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) {
pm = &GlobalPortMapper{
availPorts: make(map[int]net.Listener),
}
for i := 0; i < n; i++ {
lsn, _ := net.Listen("tcp", ":0")
r := lsn.Addr()
port := r.(*net.TCPAddr).Port
pm.availPorts[port] = lsn
}
return
}
func (pm *GlobalPortMapper) GetPort() (port int, err error) {
for port, lsn := range pm.availPorts {
lsn.Close()
return port, nil
}
return -1, fmt.Errorf("no more ports available")
}
func (pm *GlobalPortMapper) MustGetPort() int {
port, err := pm.GetPort()
if err != nil {
panic(err)
}
return port
}
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
uri0, err := pnet.NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
uri1, err := pnet.NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node0 := &topology.Node{ID: "node0", URI: *uri0}
node1 := &topology.Node{ID: "node1", URI: *uri1}
c := newCluster()
c.addNodeBasicSorted(node0)
@ -90,6 +128,22 @@ func TestFragCombos(t *testing.T) {
}
}
// newHolderWithTempPath returns a new instance of Holder.
func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
if err != nil {
panic(err)
}
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = backend
h := NewHolder(path, cfg)
panicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
})
return h
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(tb testing.TB, name string) *Index {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
@ -110,27 +164,27 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
// Ensure that fragSources creates the correct fragment mapping.
func TestFragSources(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
uri0, err := pnet.NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
uri1, err := pnet.NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
uri2, err := pnet.NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
uri3, err := NewURIFromAddress("host3")
uri3, err := pnet.NewURIFromAddress("host3")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node3 := &Node{ID: "node3", URI: *uri3}
node0 := &topology.Node{ID: "node0", URI: *uri0}
node1 := &topology.Node{ID: "node1", URI: *uri1}
node2 := &topology.Node{ID: "node2", URI: *uri2}
node3 := &topology.Node{ID: "node3", URI: *uri3}
c1 := newCluster()
c1.ReplicaN = 1
@ -224,8 +278,8 @@ func TestFragSources(t *testing.T) {
"node0": {},
"node1": {},
"node2": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -236,11 +290,11 @@ func TestFragSources(t *testing.T) {
idx: idx,
expected: map[string][]*ResizeSource{
"node0": {
{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
{&topology.Node{ID: "node1", URI: pnet.URI{Scheme: "http", Host: "host1", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(1)},
},
"node1": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
},
},
err: "",
@ -251,11 +305,11 @@ func TestFragSources(t *testing.T) {
idx: idx,
expected: map[string][]*ResizeSource{
"node0": {
{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(0)},
{&topology.Node{ID: "node2", URI: pnet.URI{Scheme: "http", Host: "host2", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(2)},
},
"node1": {
{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
{&topology.Node{ID: "node0", URI: pnet.URI{Scheme: "http", Host: "host0", Port: 10101}, IsPrimary: false}, "i", "f", "standard", uint64(3)},
},
"node2": {},
},
@ -304,37 +358,37 @@ func TestFragSources(t *testing.T) {
// Ensure that fragSources creates the correct fragment mapping.
func TestResizeJob(t *testing.T) {
uri0, err := NewURIFromAddress("host0")
uri0, err := pnet.NewURIFromAddress("host0")
if err != nil {
t.Fatal(err)
}
uri1, err := NewURIFromAddress("host1")
uri1, err := pnet.NewURIFromAddress("host1")
if err != nil {
t.Fatal(err)
}
uri2, err := NewURIFromAddress("host2")
uri2, err := pnet.NewURIFromAddress("host2")
if err != nil {
t.Fatal(err)
}
node0 := &Node{ID: "node0", URI: *uri0}
node1 := &Node{ID: "node1", URI: *uri1}
node2 := &Node{ID: "node2", URI: *uri2}
node0 := &topology.Node{ID: "node0", URI: *uri0}
node1 := &topology.Node{ID: "node1", URI: *uri1}
node2 := &topology.Node{ID: "node2", URI: *uri2}
tests := []struct {
existingNodes []*Node
node *Node
existingNodes []*topology.Node
node *topology.Node
action string
expectedIDs map[string]bool
}{
{
existingNodes: []*Node{node0, node1},
existingNodes: []*topology.Node{node0, node1},
node: node2,
action: resizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
existingNodes: []*topology.Node{node0, node1, node2},
node: node2,
action: resizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
@ -355,22 +409,27 @@ func TestResizeJob(t *testing.T) {
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := cluster{
nodes: []*Node{
noder: topology.NewLocalNoder([]*topology.Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
}),
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
cNodes := c.noder.Nodes()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
// Verify nodes are distributed.
if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.nodes[0], c.nodes[1]}) {
if a := snap.PartitionNodes(0); !reflect.DeepEqual(a, []*topology.Node{cNodes[0], cNodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.nodes[2], c.nodes[0]}) {
if a := snap.PartitionNodes(2); !reflect.DeepEqual(a, []*topology.Node{cNodes[2], cNodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
@ -381,7 +440,7 @@ func TestCluster_Partition(t *testing.T) {
c := newCluster()
c.partitionN = partitionN
partitionID := c.shardToShardPartition(index, shard)
partitionID := topology.ShardToShardPartition(index, shard, partitionN)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN)
}
@ -411,7 +470,7 @@ func TestHasher(t *testing.T) {
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
hasher := &Jmphasher{}
hasher := &topology.Jmphasher{}
if got := hasher.Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
@ -423,7 +482,12 @@ func TestHasher(t *testing.T) {
func TestCluster_ContainsShards(t *testing.T) {
c := NewTestCluster(t, 5)
c.ReplicaN = 3
shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
cNodes := c.noder.Nodes()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
shards := snap.ContainsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), cNodes[2])
if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected shars for node's index: %v", shards)
@ -431,20 +495,22 @@ func TestCluster_ContainsShards(t *testing.T) {
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
const urisCount = 4
var uris []pnet.URI
arbitraryPorts := []int{17384, 17385, 17386, 17387}
for i := 0; i < urisCount; i++ {
uris = append(uris, NewTestURIFromHostPort(fmt.Sprintf("node%d", i), uint16(arbitraryPorts[i])))
}
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
node0 := &topology.Node{ID: "node0", URI: uris[0]}
node1 := &topology.Node{ID: "node1", URI: uris[1]}
node2 := &topology.Node{ID: "node2", URI: uris[2]}
node3 := &topology.Node{ID: "node3", URI: uris[3]}
nodes := []*Node{node0, node1, node2}
nodes := []*topology.Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
actual := topology.Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
@ -452,24 +518,24 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
actual := topology.Nodes(topology.Nodes(nodes).Filter(nodes[1])).URIs()
expected := []pnet.URI{uris[0], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
actual := topology.Nodes(topology.Nodes(nodes).FilterURI(uris[1])).URIs()
expected := []pnet.URI{uris[0], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
actualTrue := topology.Nodes(nodes).Contains(node1)
actualFalse := topology.Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
@ -479,9 +545,9 @@ func TestCluster_Nodes(t *testing.T) {
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
clone := topology.Nodes(nodes).Clone()
actual := topology.Nodes(clone).URIs()
expected := []pnet.URI{uris[0], uris[1], uris[2]}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
@ -489,9 +555,9 @@ func TestCluster_Nodes(t *testing.T) {
}
func TestCluster_PreviousNode(t *testing.T) {
node0 := &Node{ID: "node0"}
node1 := &Node{ID: "node1"}
node2 := &Node{ID: "node2"}
node0 := &topology.Node{ID: "node0"}
node1 := &topology.Node{ID: "node1"}
node2 := &topology.Node{ID: "node2"}
t.Run("OneNode", func(t *testing.T) {
c := newCluster()
@ -542,350 +608,6 @@ func TestCluster_PreviousNode(t *testing.T) {
})
}
// NEXT: move this test to internal and unexport IsCoordinator
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *newCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *newCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.isCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.isCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.addNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.nodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(t, 1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
nodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.nodeIDs, node.Topology.nodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(t, 0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{node.Node.ID},
}
if err := tc.WriteTopology(node.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(t, 0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{"some-other-host"},
}
if err := tc.WriteTopology(node.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
expected := "coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || errors.Cause(err).Error() != expected {
t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(t, 0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatalf("opening cluster: %v", err)
}
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(t, 0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
nodeIDs: []string{"node0", "node2"},
}
if err := tc.WriteTopology(node0.Path, top); err != nil {
t.Fatalf("writing topology: %v", err)
}
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatalf("opening cluster: %v", err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
if err := tc.addNode(); err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node2 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(t, 0)
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
t.Fatalf("creating field: %v", err)
}
// Each tc.SetBit starts and commits its own Tx.
if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
if err := tc.SetBit("i", "f", 1, ShardWidth+1, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.holder.Field("i", "f")
node0View := node0Field.view("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum, err := node0Fragment.Checksum()
if err != nil {
t.Fatal(err)
}
idx0 := node0.holder.Index("i")
if idx0 == nil {
t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
}
// addNode needs to block until the resize process has completed.
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
// INVAR: after node1.State() is normal, the rebalancing should have been done.
expectedTop := &Topology{
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.nodeIDs, node0.Topology.nodeIDs)
} else if !reflect.DeepEqual(node1.Topology.nodeIDs, expectedTop.nodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.nodeIDs, node1.Topology.nodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.holder.Field("i", "f")
node1View := node1Field.view("standard")
node1Fragment := node1View.Fragment(1)
idx1 := node1.holder.Index("i")
if idx1 == nil {
t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
}
// Ensure checksums are the same.
if chksum, err := node1Fragment.Checksum(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
func TestAE(t *testing.T) {
t.Run("AbortDoesn'tBlockUninitialized", func(t *testing.T) {
c := newCluster()
@ -944,136 +666,4 @@ func TestAE(t *testing.T) {
t.Fatalf("abort should not have blocked this long")
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(t, 2)
oldNode := c.nodes[0]
newNode := c.nodes[1]
// Update coordinator to the same value.
if c.updateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.updateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}
func TestCluster_confirmNodeDownUp(t *testing.T) {
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
// Close the server when test finishes
defer server.Close()
u, err := url.Parse(server.URL)
if err != nil {
t.Error("bad test setup")
}
uri := URI{}
host, port, _ := net.SplitHostPort(u.Host)
uri.Scheme = u.Scheme
uri.Host = host
iport, err := strconv.ParseUint(port, 0, 16)
if err != nil {
t.Error(err)
}
uri.Port = uint16(iport)
c := newCluster()
c.logger = logger.NewVerboseLogger(os.Stdout)
if c.confirmNodeDown(uri) {
t.Errorf("expected node to be up")
}
}
func TestCluster_confirmNodeDownTimeout(t *testing.T) {
sleep := 50 * time.Millisecond
retries := 5
if testing.Short() {
t.Skip()
}
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(sleep * time.Duration(retries))
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
// Close the server when test finishes
defer server.Close()
u, err := url.Parse(server.URL)
if err != nil {
t.Error("bad test setup")
}
uri := URI{}
host, port, _ := net.SplitHostPort(u.Host)
uri.Scheme = u.Scheme
uri.Host = host
iport, err := strconv.ParseUint(port, 0, 16)
if err != nil {
t.Error(err)
}
uri.Port = uint16(iport)
c := newCluster()
c.confirmDownSleep = sleep
c.confirmDownRetries = retries
c.logger = logger.NewVerboseLogger(os.Stdout)
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}
func TestCluster_confirmNodeDownDown(t *testing.T) {
if testing.Short() {
t.Skip()
}
uri := URI{}
uri.Scheme = "http"
uri.Host = "DoesntMatter"
uri.Port = 6666
c := newCluster()
c.confirmDownSleep = 50 * time.Millisecond
c.confirmDownRetries = 5
c.logger = logger.NewVerboseLogger(os.Stdout)
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}
func TestCluster_GetNonPrimaryReplicas(t *testing.T) {
c := newCluster()
c.ReplicaN = 3
topo := NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c)
c.Topology = topo
nNodes := 4
for i := 0; i < nNodes; i++ {
nodeID := fmt.Sprintf("node%d", i)
c.nodes = append(c.nodes, &Node{
ID: nodeID,
URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
c.Topology.addID(nodeID)
}
partitionID := 256
nonPrimes := topo.GetNonPrimaryReplicas(partitionID)
m := len(nonPrimes)
if m != c.ReplicaN-1 {
t.Fatalf("expected 2 non primes, got %v", m)
}
}

14
cmd/badloader/badloader.go
View file

@ -19,13 +19,17 @@ import (
"compress/gzip"
"context"
"time"
//"fmt"
"fmt"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
"io"
"io/ioutil"
gohttp "net/http"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
pnet "github.com/pilosa/pilosa/v2/net"
//"log"
"os"
//"path/filepath"
@ -140,15 +144,15 @@ func main() {
vv("total elapsed '%v'", time.Since(t0))
}
var globURI *pilosa.URI
var globURI *pnet.URI
func init() {
var err error
globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
panicOn(err)
}
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
return globURI
}

111
cmd/pilosa-chk/chk.go
View file

@ -1,111 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"fmt"
"log"
"os"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/hash"
"github.com/zeebo/blake3"
)
// pilosa-chk : read boltdb files and print checksums and counts on the keys. With
// -v and -ops and -bits you can display every last bit if you want.
//
// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
// forcing a customer to upgrade or downgrade their installed version.
func main() {
var dir string
var showOpsLog bool
var showBits bool
var showFrags bool
var dirChecksum bool
home := os.Getenv("HOME")
flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
flag.BoolVar(&showFrags, "v", false, "show the checksum hash for each fragment in each index. Warning: long output")
flag.BoolVar(&showOpsLog, "ops", false, "show the ops log for each fragment. Warning: very long output. Implies -v")
flag.BoolVar(&showBits, "bits", false, "show the hot bits for each fragment. Warning: very, very long output. Implies -v")
flag.BoolVar(&dirChecksum, "dirsum", false, "compute a directory hash")
flag.Parse()
if showBits {
showFrags = true
}
if showOpsLog {
showFrags = true
}
fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
if dirChecksum {
fmt.Printf("path '%v' has dirhash %v\n", dir, hash.HashOfDir(dir))
return
}
fmt.Printf(" the blake-3 hash includes the value of each mapping and the field or partitionID.\n")
holder := pilosa.NewHolder(dir, nil)
holder.OpenTranslateStore = boltdb.OpenTranslateStore
err := holder.Open()
if err != nil {
log.Fatal(err)
}
fmt.Printf("\ncalculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
var indexes []*pilosa.Index
final := pilosa.NewAllTranslatorSummary()
const verbose = true
const checkKeys = false
const applyKeyRepairs = false
for _, idx := range holder.Indexes() {
asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID", 10)
if err != nil {
log.Fatal(err)
}
final.Append(asum)
indexes = append(indexes, idx)
}
final.Sort()
hasher := blake3.New()
fmt.Printf("\nsummary of col/row translations%v:\n", dir)
for _, sum := range final.Sums {
//fmt.Printf("index: %v partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
_, _ = hasher.Write([]byte(sum.Checksum))
}
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
fmt.Printf("all-checksum = blake3-%x\n", buf)
if showFrags {
for _, idx := range indexes {
fmt.Printf("==============================\n")
fmt.Printf("index: %v\n", idx.Name())
fmt.Printf("==============================\n")
idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, nil, verbose)
}
}
}

36
cmd/pilosa-fsck/Makefile
View file

@ -1,36 +0,0 @@
.PHONY: install build release
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
VARIANT = Molecula
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
SHARD_WIDTH = 20
COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)"
GOOS = $(shell go env GOOS)
# Install pilosa-fsck
install:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS)
# Compile pilosa-fsck
build:
go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS)
REL = release-pilosa-fsck.$(COMMIT).$(GOOS)
release:
mkdir $(REL)
cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - )
go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck
tar cf - $(REL) | gzip > $(REL).tar.gz
rm -rf $(REL)
mv $(REL).tar.gz ../..
clean:
find . -name pilosa-fsck | xargs rm -f
rm -f release-pilosa-fsck*.tar.gz

985
cmd/pilosa-fsck/fsck.go
View file

@ -1,985 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/dustin/go-humanize"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/server"
"github.com/pkg/errors"
"github.com/zeebo/blake3"
)
// pilosa-fsck :
// an external customer tool (originally for Q2) to do 2 jobs:
// Given a set of cluster backups (and their .id and .topology files)
// mounted on the same file system, we can:
// 1) scan for fragment differences between the primary and its replicas (default); or
// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given).
//
// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
// forcing a customer to upgrade or downgrade their installed version.
// FsckConfig configures the dumpcols() and/or read() runs.
type FsckConfig struct {
Fix bool // -fix
FixCol bool // -fixcol
Colkeydump bool // -col
JustThisIndex string // -index
// -col column key dump only options:
// Dir string
// PartitionID int
// ShowHeader bool
// ShowKey bool
// ShowID bool
// not flags, just the Args() left after all other flags. Should be the list
// of pilosa (holder) directories for the cluster.
Dirs []string
Verbose bool // -v
Quiet bool // -q
// manual workaround for not having PilosaConfigPath, if really need be.
ReplicaN int // -replicas
PilosaConfigPath string // -config
ParallelReaders int // -readers
topo *pilosa.Topology
}
// call DefineFlags before myflags.Parse()
func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) {
fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol")
fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.")
//fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis")
fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet")
fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.")
fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.")
fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)")
fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.")
fs.Usage = func() {
fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo())
fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
-fix
(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
-replicas R
(required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
the number of replicas maintained in the cluster. Must be the same as the
[cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
-index index_name
(optional) restrict to just this index. Otherwise we default to all indexes.
-readers PR
how many parallel readers to use to scan at once. PR==0 means do everything
possible in parallel. PR==1 means serialize everything through a single reader.
Adjust PR to control memory consumption if needed. As a practical limit, setting
PR > 10000 will have no effect. (default is 10).
-q
be very quiet during analysis and repair
`)
fmt.Fprintf(os.Stderr, `
Welcome to pilosa-fsck. This is a scan and repair
tool that is modeled after the classic unix file
system utility fsck.
WARNING: DO NOT RUN ON A LIVE SYSTEM.
The most important point to remember is that analysis
and repair must be done *offline*.
Just as fsck must be run on an unmounted disk,
pilosa-fsck must be run on a backup. It must
not be run on the directories where a live Pilosa system
is serving queries. Instead, take a backup first.
A backup is a set of N Pilosa data directories that have been
copied from your live system. They must all
be visible and mounted on one filesystem together.
pilosa-fsck can be run in scan-mode (without -fix),
or in repair-mode with -fix. The console output
supplies a log documenting the analysis
and showing what data changes would have been made.
REQUIRED COMMAND LINE ARGUMENTS
The paths to all the top-level Pilosa
data directories in a cluster must be given on the command
line. The -replicas R flag is also always required. It
must be correct for your cluser. Here R is the same as
the [cluster] stanza "replicas = R" line from your
pilosa.conf.
Example:
Suppose you are ready to run pilosa-fsck:
you have taken a backup of your four node Pilosa
cluster and stored it all on one filesystem with
all nodes visible and uncompressed. This
is a pre-requisite to running pilosa-fsck.
Let's suppose we have replication R = 3 set.
In this example, have stored our backed-up directories in
/backup/molecula
and the four node backups are in
subdirectories node1/ node2/ node3/ node4/ under this:
/backup/molecula/node1/
/backup/molecula/node1/.pilosa/.id
/backup/molecula/node1/.pilosa/.topology
/backup/molecula/node1/.pilosa/myindex
/backup/molecula/node2/
/backup/molecula/node2/.pilosa/.id
/backup/molecula/node2/.pilosa/.topology
/backup/molecula/node2/.pilosa/myindex
/backup/molecula/node3/
/backup/molecula/node3/.pilosa/.id
/backup/molecula/node3/.pilosa/.topology
/backup/molecula/node3/.pilosa/myindex
/backup/molecula/node4/
/backup/molecula/node4/.pilosa/.id
/backup/molecula/node4/.pilosa/.topology
/backup/molecula/node4/.pilosa/myindex
NOTE: your .pilosa directories need not be named .pilosa. They can
be something else, such as when the -d flag to pilosa server was used.
The .id file, the .topology file, and the index directories must be
found directly underneath.
Then a typical invocation to scan a cluster backup for issues:
$ cd /backup/molecula/
$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log
A typical invocation to repair the replication in the same backup:
$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log
In both cases, the .id and .topology files must
be present in the backups.
Without -fix, no modifications will be made to the backups. Only
by running with -fix will repairs be made. The user can safely
always run with -fix to repair only if needed.
A zero error code will be returned to the shell if no repairs were needed.
A zero error code will be also be returned to the shell if
repairs were needed and they were accomplished under -fix.
A non-zero error code indicates that repairs were needed but
were not made.
`)
}
}
// call c.ValidateConfig() after myflags.Parse()
func (c *FsckConfig) ValidateConfig() error {
if c.Fix {
c.FixCol = true
}
if c.ReplicaN == 0 && c.PilosaConfigPath == "" {
return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)")
}
if c.ReplicaN == 0 && c.PilosaConfigPath != "" {
if !FileExists(c.PilosaConfigPath) {
return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath)
}
by, err := ioutil.ReadFile(c.PilosaConfigPath)
if err != nil {
return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err)
}
srvcfg, err := server.ParseConfig(string(by))
if err != nil {
//vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err)
// fall back to manual parsing of config
lines := strings.Split(string(by), "\n")
clusterStart := -1
for i, line := range lines {
if strings.Contains(line, `[cluster]`) {
clusterStart = i
}
if i > clusterStart {
if strings.Contains(line, "replicas") {
split := strings.Split(line, "=")
ns := strings.TrimSpace(split[1])
n, err := strconv.Atoi(ns)
if err != nil {
return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err)
}
c.ReplicaN = n
}
}
}
} else {
c.ReplicaN = srvcfg.Cluster.ReplicaN
}
if c.ReplicaN == 0 {
return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath)
}
//vv("c.ReplicaN = %v", c.ReplicaN)
}
return nil
}
var ProgramName = "pilosa-fsck"
func main() {
myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError)
cfg := &FsckConfig{}
cfg.DefineFlags(myflags)
cfg.Verbose = true
err := myflags.Parse(os.Args[1:])
if err != nil {
fmt.Fprintf(os.Stderr, "\n%v\n", err.Error())
os.Exit(1)
}
err = cfg.ValidateConfig()
if err != nil {
fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err)
os.Exit(1)
}
dirs := myflags.Args()
nDir := len(dirs)
if nDir <= 0 && !cfg.Colkeydump {
fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName)
os.Exit(1)
}
cmdline := strings.Join(os.Args, " ")
// make sure all the dir are distinct
dup := make(map[string]bool)
for _, dir := range dirs {
if dup[dir] {
fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline)
os.Exit(1)
} else {
dup[dir] = true
}
}
fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo())
cwd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd)
fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline)
t0 := time.Now()
fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0))
defer func() {
fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
}()
cfg.Dirs = dirs
fixNeeded, err := cfg.Run()
if err != nil {
fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
if fixNeeded && !cfg.Fix {
fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0))
fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n")
os.Exit(1)
}
}
func (cfg *FsckConfig) Run() (fixNeeded bool, err error) {
// if cfg.Colkeydump {
// cfg.dumpcols()
//}
perNodeIndexMaps, clusterNodes, ats, err := cfg.read()
if err != nil {
return false, err
}
if cfg.FixCol {
err := cfg.RepairTranslationStores(ats)
if err != nil {
return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err)
}
}
//vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes)
fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats)
if err != nil {
return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err)
}
fixNeeded = ats.RepairNeeded || fixme
for _, report := range reports {
fmt.Printf("%v\n", report)
}
if len(reports) == 0 {
fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " "))
}
return
}
var _ = (&FsckConfig{}).dumpAts
func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) {
fmt.Printf("# dumpAts: RepairNeeded=%v\n", ats.RepairNeeded)
for _, sum := range ats.Sums {
fmt.Printf("# sum = '%#v'\n", sum)
}
}
type group struct {
elem []*pilosa.TranslatorSummary
partitionID int
}
func (g *group) String() (s string) {
for i, e := range g.elem {
s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String())
}
return
}
func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) {
indexMap := make(map[string]bool)
for _, sum := range ats.Sums {
if !indexMap[sum.Index] {
indexMap[sum.Index] = true
indexes = append(indexes, sum.Index)
}
}
sort.Strings(indexes)
return
}
func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) {
verbose := cfg.Verbose
// group by index first. then repair.
indexes := indexesFromAts(ats)
for _, index := range indexes {
if !cfg.DoingIndex(index) {
continue
}
m := make(map[int]*group)
for _, sum := range ats.Sums {
if !sum.IsColKey || sum.Index != index {
continue
}
grp := m[sum.PartitionID]
if grp == nil {
grp = &group{
partitionID: sum.PartitionID,
}
m[sum.PartitionID] = grp
}
grp.elem = append(grp.elem, sum)
}
for partitionID, group := range m {
_ = partitionID
prim := -1
keyCount := 0
for k, e := range group.elem {
if e.IsPrimary {
prim = k
}
keyCount += e.KeyCount
}
if prim == -1 {
panic(fmt.Sprintf("no primary found for group '%v'", group.String()))
}
primary := group.elem[prim]
primaryChecksum := primary.Checksum
for _, e := range group.elem {
if e.IsPrimary {
continue
}
// is e a replica? not necessarily! have to check.
if !e.IsReplica {
//if verbose {
// since this will happen even on a fix point, where it is already empty,
// we don't report it again.
//fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath)
//}
err := os.RemoveAll(e.StorePath)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath))
}
store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath))
}
err = store.Close()
if err != nil {
return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath))
}
continue
}
// INVAR: e is a replica for this paritionID.
// Copy from primary if checksums are different.
if e.Checksum != primaryChecksum {
from := group.elem[prim].StorePath
dest := e.StorePath
if verbose {
fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest)
}
err := cp(from, dest)
if err != nil {
return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err)
}
}
}
}
}
return nil
}
/*
func (cfg *FsckConfig) dumpcols() {
verbose := cfg.Verbose
quiet := cfg.Quiet
_, _ = verbose, quiet
dir := cfg.Dir
index := cfg.Index
partitionID := cfg.PartitionID
showKey := cfg.ShowKey
showID := cfg.ShowID
if !quiet {
fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir)
}
holder := pilosa.NewHolder(dir, nil)
holder.OpenTranslateStore = boltdb.OpenTranslateStore
err := holder.Open()
if err != nil {
log.Fatal(err)
}
if cfg.ShowHeader {
fmt.Println("# columnKey columId")
}
id_key := make(map[uint64]string)
key_id := make(map[string]uint64)
for _, idx := range holder.Indexes() {
fmt.Printf("# Looking '%v'\n", idx.Name())
if idx.Name() == index {
store := idx.TranslateStore(partitionID)
fmt.Printf("# Key By ID partitionID = %v\n", partitionID)
err := store.KeyWalker(func(key string, col uint64) {
key_id[key] = col
if showKey {
fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID)
}
})
panicOn(err)
}
}
for _, idx := range holder.Indexes() {
if idx.Name() == index {
store := idx.TranslateStore(partitionID)
//fmt.Printf("# ID ByKey\n")
err := store.IDWalker(func(key string, col uint64) {
id_key[col] = key
if showID {
fmt.Printf("# '%v' %v\n", key, col)
}
})
panicOn(err)
}
}
fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key))
fmt.Println("id_key")
for k, v := range id_key {
l, ok := key_id[v]
if ok {
if k != l {
fmt.Printf("# X: %v %v %v\n", k, l, v)
}
} else {
fmt.Printf("# key not in id %v\n", v)
}
}
fmt.Println("key_id")
for k, v := range key_id {
l, ok := id_key[v]
if ok {
if k != l {
fmt.Printf("# T: %v %v %v\n", k, l, v)
}
} else {
fmt.Printf("# id not in key %v\n", v)
}
}
}
*/
func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) {
final = pilosa.NewAllTranslatorSummary()
dirs := cfg.Dirs
for _, dir := range dirs {
idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir)
if err != nil {
return nil, nil, nil, err
}
final.Append(atsNode)
clusterNodes = append(clusterNodes, nodeID)
perNodeIndexMaps = append(perNodeIndexMaps, idx2frag)
}
return
}
func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) {
verbose := cfg.Verbose
quiet := cfg.Quiet
if !quiet {
fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir)
}
jmphasher := &pilosa.Jmphasher{}
partitionN := pilosa.DefaultPartitionN
replicaN := cfg.ReplicaN
topo, err := loadTopology(dir, jmphasher, partitionN, replicaN)
if err != nil {
return nil, "", nil, err
}
cfg.topo = topo
//vv("topo = '%#v'", topo)
nodeIDs := topo.GetNodeIDs()
//vv("nodeIDs = '%#v'", nodeIDs)
nNodes := len(nodeIDs)
nDir := len(cfg.Dirs)
if nDir != nNodes {
return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs)
}
holder := pilosa.NewHolder(dir, nil)
holder.OpenTranslateStore = boltdb.OpenTranslateStore
nodeID, err = holder.LoadNodeID()
panicOn(err)
//vv("nodeID = '%v'", nodeID)
err = holder.Open()
if err != nil {
log.Fatal(err)
}
if !quiet {
fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
}
var indexes []*pilosa.Index
const checkKeys = true
atsNode = pilosa.NewAllTranslatorSummary()
for _, idx := range holder.Indexes() {
if !cfg.DoingIndex(idx.Name()) {
continue
}
//vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol)
asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders)
if err != nil {
log.Fatal(err)
}
atsNode.Append(asum)
indexes = append(indexes, idx)
}
atsNode.Sort()
hasher := blake3.New()
if !quiet {
fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir)
}
for _, sum := range atsNode.Sums {
if !quiet {
fmt.Printf("# index: %v partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
}
_, _ = hasher.Write([]byte(sum.Checksum))
}
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
if !quiet {
fmt.Printf("# all-checksum = blake3-%x\n", buf)
}
// fragment analysis
showBits := false
showOpsLog := false
idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node.
for _, idx := range indexes {
if verbose {
fmt.Printf("# ==============================\n")
fmt.Printf("# index: %v\n", idx.Name())
fmt.Printf("# ==============================\n")
}
frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose)
frgsum.Dir = dir
frgsum.NodeID = nodeID
idx2frag[idx.Name()] = frgsum
}
_ = holder.Close()
//vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple.
return
}
func (cfg *FsckConfig) DoingIndex(index string) bool {
if cfg.JustThisIndex == "" {
// scan all indexes
return true
}
if index == cfg.JustThisIndex {
// scan just this one
return true
}
return false
}
// from cluster.go:1924
func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) {
buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology"))
if err != nil {
return nil, err
}
var pb internal.Topology
err = proto.Unmarshal(buf, &pb)
if err != nil {
return nil, err
}
return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil)
}
func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) {
verbose := cfg.Verbose
quiet := cfg.Quiet
_, _ = verbose, quiet
allIndex := make(map[string]bool)
for _, mp := range perNodeIndexMaps {
for index := range mp {
allIndex[index] = true
}
}
if !quiet {
vv("allIndex = '%#v'", allIndex)
}
for index := range allIndex {
if !quiet {
vv("on index '%v'", index)
}
nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary)
for _, mp := range perNodeIndexMaps {
sum := mp[index]
if sum == nil {
continue
}
nodes2fragsum[sum.NodeID] = sum
}
fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats)
if err != nil {
return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err)
}
fixNeeded = fixNeeded || fixme
reports = append(reports, report)
}
return fixNeeded, reports, nil
}
func (cfg *FsckConfig) analyzeThisIndex(
index string,
nodes2fragsum map[string]*pilosa.IndexFragmentSummary,
ats *pilosa.AllTranslatorSummary,
) (fixNeeded bool, report string, err error) {
verbose := cfg.Verbose
quiet := cfg.Quiet
_, _ = verbose, quiet
var removedBytes int64
var copiedBytes int64
var changedFiles int64
var totalFiles int64
var overwrittenBytes int64
var totalBytes int64
if !quiet {
vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'",
index, len(nodes2fragsum), nodes2fragsum)
}
for node, sum := range nodes2fragsum {
if !quiet {
fmt.Printf("# on node '%v'\n", node)
}
// do they disagree on who is the primary?
// for each fragment, do they disagree on the checksum?
// Q: which nodes are supposed to have data, and which
// nodes are not supposed to have data?
// loopFragSum:
for relpath, fragsum := range sum.RelPath2fsum {
fragsum.NodeID = node
totalFiles++
//vv("checking %v on node %v", relpath, node)
replicas, nonReplicas := cfg.topo.GetReplicasForPrimary(fragsum.Primary)
_, _ = replicas, nonReplicas
//vv("replicas = '%#v'", replicas)
//vv("nonReplicas = '%#v'", nonReplicas)
err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum)
if err != nil {
return fixNeeded, "", err
}
// find the primary's checksum
primaryChecksum := ""
var primaryFragSum *pilosa.FragSum
for node, isPrimary := range replicas {
if isPrimary {
primarySum := nodes2fragsum[node]
primaryFragSum = primarySum.RelPath2fsum[relpath]
if primaryFragSum == nil {
// This seems clear indication that we have the topology wrong.
// When the topology is right, there are NO errors of this kind.
//
msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas)
vv(msg)
fmt.Fprintf(os.Stderr, "%v\n", msg)
panic(msg) // stop. the fixes are going to be wrong.
} else {
primaryChecksum = primaryFragSum.Checksum
primaryFragSum.NodeID = node
primaryFragSum.ScanDone = true
}
break
}
}
if primaryChecksum == "" {
return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum)
}
// is this a non-replica?
_, isNon := nonReplicas[fragsum.NodeID]
if isNon {
removedBytes += FileSize(fragsum.AbsPath)
changedFiles++
//vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID)
if !quiet {
fmt.Printf("rm %v #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum)
}
if cfg.Fix {
err := os.Remove(fragsum.AbsPath)
if err != nil {
return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err)
}
}
} else {
presz := FileSize(fragsum.AbsPath)
totalBytes += presz
checksum := fragsum.Checksum
if checksum != primaryChecksum {
copiedBytes += FileSize(primaryFragSum.AbsPath)
changedFiles++
overwrittenBytes += presz
if !quiet {
fmt.Printf("cp %v %v #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum)
}
if cfg.Fix {
err := cp(primaryFragSum.AbsPath, fragsum.AbsPath)
if err != nil {
return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'",
primaryFragSum.AbsPath, fragsum.AbsPath, err)
}
}
}
}
fragsum.ScanDone = true
}
}
nDir := len(nodes2fragsum)
keyCount, idCount := cfg.getKeyIDCounts(index, ats)
fixNeeded = changedFiles > 0 || ats.RepairNeeded
var actionTaken string
var wouldBe string
if cfg.Fix || cfg.FixCol {
if fixNeeded {
actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*"
wouldBe = "sync repairs made:"
} else {
wouldBe = ""
actionTaken = "NO REPAIR NEEDED."
}
} else {
if fixNeeded {
wouldBe = "sync actions that would be taken under -fix:"
actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted."
} else {
wouldBe = ""
actionTaken = "NO REPAIR NEEDED."
}
}
var fragUpdate string
if changedFiles > 0 {
fragUpdate = fmt.Sprintf(`
# %v
# copied bytes: %v
# file bytes overwritten: %v
# new bytes added: %v
# new bytes is %0.01f%% of %v total bytes
# removed %v bytes from non-replicas
# changed file count %v (%0.01f%%; total files=%v)
#
`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles))
}
report = fmt.Sprintf(`
# ========================================================
# pilosa-fsck final report
#
# run with -fix: %v
#
# index examined: '%v'
#
# nodes examined: %v
# -replicas %v replication factor used
#
# feature data examined: %v bytes
# feature files examined: %v files
#
# key-translation-stores examined: %v
# key-count: %v over all replicas
# id-count: %v over all replicas
#
# %v
# %v
# ========================================================
`,
cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate)
return
}
func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error {
for node := range replicas {
if nodes2fragsum[node] == nil {
return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum)
}
}
return nil
}
func cp(fromPath, toPath string) (err error) {
tmpTo := toPath + ".fsck.tmp"
toFd, err := os.Create(tmpTo)
if err != nil {
return err
}
defer toFd.Close()
fromFd, err := os.Open(fromPath)
if err != nil {
return err
}
defer fromFd.Close()
_, err = io.Copy(toFd, fromFd)
if err != nil {
return err
}
err = toFd.Close()
if err != nil {
return err
}
return os.Rename(tmpTo, toPath)
}
func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) {
for _, sum := range ats.Sums {
if sum.Index == index {
keyCount += sum.KeyCount
idCount += sum.IDCount
}
}
return
}

445
cmd/pilosa-fsck/fsck_test.go
View file

@ -1,445 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"io/ioutil"
"reflect"
"strconv"
"testing"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
)
func Test_Repair(t *testing.T) {
// a) setup 1 primary + 3 replicas of disagree-ing cluster dirs.
nNodes := 4
nReplicas := 3
name := t.Name()
var nodeid []string
for i := 0; i < nNodes; i++ {
// work around a bug in the test.MustRunCluster that corrupts
// the .topology file if we only join name with one "_" underscore.
nodeid = append(nodeid, name+"__"+strconv.Itoa(i))
}
c := test.MustRunCluster(t, nNodes,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[0]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[1]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[2]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[3]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
)
// note: do not defer c.Close() here. We manually close below.
var nodes []*test.Command
var dirs []string
for i := 0; i < nNodes; i++ {
nd := c.GetNode(i)
nodes = append(nodes, nd)
dirs = append(dirs, nd.Server.Holder().Path())
}
ctx := context.Background()
index := []string{"rick", "morty"}
fieldName := []string{"f", "flying_car"}
idx := make([]*pilosa.Index, len(index))
field := make([]*pilosa.Field, len(index))
var err error
for i := range index {
idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
if idx[i].CreatedAt() == 0 {
t.Fatal("index createdAt is empty")
}
field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if field[i].CreatedAt() == 0 {
t.Fatal("field createdAt is empty")
}
}
rowID := uint64(1)
timestamp := int64(0)
for i := range index {
// Generate some keyed records.
rowIDs := []uint64{}
timestamps := []int64{}
N := 10
for j := 1; j <= N; j++ {
rowIDs = append(rowIDs, rowID)
timestamps = append(timestamps, timestamp)
}
var colKeys []string
switch i {
case 0:
// Keys are sharded so ordering is not guaranteed.
colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
colKeys = colKeys[:N]
case 1:
colKeys = []string{"col11", "col12"}
N = len(colKeys)
rowIDs = rowIDs[:N]
timestamps = timestamps[:N]
}
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: index[i],
IndexCreatedAt: idx[i].CreatedAt(),
Field: fieldName[i],
FieldCreatedAt: field[i].CreatedAt(),
// even though this says Shard: 0, that won't matter. The column keys
// get hashed and that decides the actual shard.
Shard: 0,
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
qcx := nodes[0].API.Txf().NewQcx()
if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
//qcx.Reset()
pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID)
// Query node0.
if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys)
}
// Query node1.
if err := test.RetryUntil(5*time.Second, func() error {
if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil {
return err
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
return fmt.Errorf("unexpected column keys: %#v", keys)
}
return nil
}); err != nil {
t.Fatal(err)
}
}
// end of setup.
// partitionID in use: 6, 31, 57, 133, 185, 235
targetPartition := 31 // which partitionID we mess with.
targetNode := nodes[0] // this is the first replica.
targetIndex := index[0]
// 0 first replica
// 1 second replica
// 2 -- not a replica
// 3 primary
cfg := &FsckConfig{
Fix: false,
FixCol: false,
Quiet: true,
//Verbose: true,
ReplicaN: nReplicas,
Dirs: dirs,
ParallelReaders: 5,
}
panicOn(cfg.ValidateConfig())
// for this test, mess up a replica that is not the primary.
h := targetNode.API.Holder()
idx[0] = h.Index(index[0])
store := idx[0].TranslateStore(targetPartition)
fwd, rev := getFwdRev(store, targetPartition)
//vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev)
// # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001
presz := len(rev)
delete(rev, fwd["col5"])
postsz := len(rev)
if postsz == presz {
panic("did not delete any key!")
}
bolt := store.(*boltdb.TranslateStore)
//vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path))
//bolt.DumpBolt("pre-corruption")
if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil {
t.Fatal(err)
}
//vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path))
//bolt.DumpBolt("post-corruption")
//fwd3, rev3 := getFwdRev(store, targetPartition)
//vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3)
targetIndex1 := "morty"
targetPartition1 := 226 // for "col11"
// # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}'
//# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}'
idx[1] = h.Index(index[1])
store1 := idx[1].TranslateStore(targetPartition1)
fwd1, rev1 := getFwdRev(store1, targetPartition1)
//vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1)
presz1 := len(rev1)
delete(rev1, fwd1["col11"])
postsz1 := len(rev1)
if postsz1 == presz1 {
panic("did not delete any key!")
}
bolt1 := store1.(*boltdb.TranslateStore)
if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil {
t.Fatal(err)
}
// done corrupting.
for _, nd := range nodes {
nd.Command.Close()
}
//panicOn(bolt.Open())
//bolt.DumpBolt("post-corruption, after Close. bolt:")
//bolt.Close()
//chksums := getChecksums(dirs, cfg, targetPartition)
//vv("post corruption, pre repair chksums = '%#v'", chksums)
// first we check that the corruption can be detected
// by our test with the checksums.
chk, err := check(dirs, cfg, targetIndex, targetPartition)
_ = chk
//vv("pre-fix, chk='%v'; err='%v'", chk, err)
if err == nil {
panic("expected to see checksums not match! but no corruption detected.")
}
chk1, err := check(dirs, cfg, targetIndex1, targetPartition1)
_ = chk1
//vv("pre-fix, chk1='%v'; err='%v'", chk1, err)
if err == nil {
panic("expected to see checksums not match! but no corruption detected.")
}
// b) running in reporting mode only should report that a fix is needed.
fixNeeded, err := cfg.Run()
panicOn(err)
if !fixNeeded {
panic("fix should be needed now, before repair")
}
// c) run the fix.
cfg.Fix = true
cfg.FixCol = true
fixNeeded, err = cfg.Run()
panicOn(err)
if !fixNeeded {
panic("fix should be marked needed if repair was made")
}
// d) check that the replicas all look like the primary.
//chksums = getChecksums(dirs, cfg, targetPartition)
//vv("after repair chksums = '%#v'", chksums)
chk, err = check(dirs, cfg, targetIndex, targetPartition)
_ = chk
//vv("chk = '%v' after repair; err='%v'", chk, err)
panicOn(err)
chk1, err = check(dirs, cfg, targetIndex1, targetPartition1)
_ = chk1
//vv("chk = '%v' after repair; err='%v'", chk, err)
panicOn(err)
// e) run again, should see no fix needed.
fixNeeded, err = cfg.Run()
panicOn(err)
if fixNeeded {
panic("should see no fix needed after the prior repair")
}
}
func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) {
fwd = make(map[string]uint64)
rev = make(map[uint64]string)
_ = store.KeyWalker(func(key string, col uint64) {
//vv("partition %v, key '%v' -> %x", partitionID, key, col)
fwd[key] = col
})
_ = store.IDWalker(func(key string, col uint64) {
//vv("partition %v, id %x -> '%v'", partitionID, col, key)
rev[col] = key
})
return
}
func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) {
//vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition)
//defer vv("returning from check()")
firstChecksum := ""
firstDir := ""
firstStorePath := ""
quiet := cfg.Quiet
defer func() {
cfg.Quiet = quiet
}()
cfg.Quiet = true
for i := range dirs {
dir := dirs[i]
_, _, ats, err := cfg.readOneDir(dir)
panicOn(err)
indexes := indexesFromAts(ats)
//vv("indexes = '%#v'", indexes)
for _, index := range indexes {
if index != targetIndex {
continue
}
for _, s := range ats.Sums {
//vv(" s= '%#v'", s)
if s.Index != index {
//vv("skipping s.Index '%v' != index '%v'", s.Index, index)
continue
}
if s.PartitionID != targetPartition {
continue
}
//vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+
//"index '%v'; s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'",
//s.PartitionID, targetPartition, s.Index, index,
//s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum)
if s.IsPrimary || s.IsReplica {
chksum := s.Checksum
if firstChecksum == "" {
firstChecksum = chksum
firstDir = dir
firstStorePath = s.StorePath
} else {
//vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum)
if chksum != firstChecksum {
return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath)
}
}
}
}
}
}
return firstChecksum, nil
}
var _ = getChecksums
func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) {
for i := range dirs {
dir := dirs[i]
_, _, ats, err := cfg.readOneDir(dir)
panicOn(err)
for _, s := range ats.Sums {
if s.PartitionID != targetPartition {
continue
}
chksum = append(chksum, s.Checksum)
}
}
return
}
/* on shardwidth 20
# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001
# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2'
# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001
# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5'
# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001
# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10'
# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001
# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7'
# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001
# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3'
# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001
# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9'
*/
var _ = fileChecksum
func fileChecksum(path string) string {
by, err := ioutil.ReadFile(path)
panicOn(err)
return hash.Blake3sum16(by)
}

1
cmd/pilosa-fsck/release-pilosa-fsck/.gitignore vendored
View file

@ -1 +0,0 @@
pilosa-fsck

252
cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md
View file

@ -1,252 +0,0 @@
Design for pilosa-fsck
======================
Problem Background
------------------
Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster.
Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data,
and Row-Key data are replicated. Only the first two, Roaring data and Column-Key
data are relevant here. Broadly, the Roaring bitmap data
forms the central features -- the bits -- of a large, sparse bitmap matrix.
The Column-Keys are the labels for the columns at the top margin of this matrix.
For speed, the Roaring bitmap data is stored separately from the
Key data. The Roaring data is stored in sharded files
within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/...
The Key translation data is stored in sharded BoltDB databases within
the PILOSA-DATA-DIR/index_name/_key directory.
The current approach to Roaring file replication involves an
eventually consistent mechanism that uses an Anti-Entropy agent to
fix partial or incomplete replication from the primary shard to all
replica shards.
Unfortunately, the Anti-Entropy agent approach has proved inadequate on two
fronts. First, it does not provide for immediately consistent reads in the
event that the primary is lost. Second, the Anti-Entropy agent itself experienced
out-of-memory issues that have yet to be resolved.
Therefore, work is now underway to replace this replication
approach with a more consistent design.
However, in the meantime, for our customers in production with Molecula
Pilosa, we wish to provide a means to re-establish correct replication.
Thus even in the event of a node failure followed by a read from a replica, the
returned read will be correct.
The pilosa-fsck tool can therefore be seen as a temporary, stop-gap
measure to address immediate issues while the cluster replication
mechanism is replaced.
The second factor motivating the creation of pilosa-fsck was the discovery
of a bug in the Key-translation process. Unfortunately this was a hard
to reproduce bug. It happened only on the customer's premises,
and only after running the system for a long time, with a
large amount of data, and with various eccentric node failures
and recoveries.
However, we were able to reproduce a plausible explanation.
Non-primary replicas were creating keys when they should have been
forwarding the request to the primary. Correcting this bug is impetus
for the v2.1.4 release of Molecula Pilosa.
A fine point here: since we were not able to precisely reproduce the customer's
issue in the development environment, we cannot guarantee with 100%
certainty that we have actually addressed the bug that the customer
was seeing.
Therefore we also desired an additional insurance
policy. We wished to be able to empower customers to proactively discover any
future Key-translation issues that happen in their on-premise systems.
To do this, we proposed providing select customers with the pilosa-fsck
tool which can analyze their offline backups for issues.
Optionally, these issues can also be repaired in-place in the
offline backup on which pilosa-fsck is run.
The -fix flag repairs both kinds of replication issues.
Solution Approach: mechanism of action
--------------------------------------
The pilosa-fsck is run offline on a full set of backups taken from
all nodes in a Pilosa cluster. It runs on a single computer that
must be separate from the production or staging Pilosa environments.
When run, pilosa-fsck analyzes the differences between the
primary and its replicas. Both the Roaring
files and the Key translation databases are analyzed.
The computer running pilosa-fsck must have the same or more
memory as the Pilosa nodes in the cluster, as it will
"pretend" to be each Pilosa node in turn. However, as each
node's backup is closed before the next node's backup is
opened, we do not require substantially more memory than a single
production node. Short Blake3 cryptographic checksums are
computed for each Roaring fragment and each Key translation
database. These are held in memory (and printed to the log)
for comparing nodes. This comparison forms the heart of
the consistency checks, and is the basis for any subsequent
repair.
We recommend capturing both stdout and stderr to a log.
Use `&> log` or `2>&1 > log` at the end of the
pilosa-fsck invocation to save a log of the run to disk.
In a typical cluster, the Replication factor R may be less
than the number of nodes N in the cluster. For example, while
N may be 4, the R may be only 3. In this example, within
each replicated shard, one node will be the primary for
that shard, two nodes will be non-primary replicas, and one
node will be a non-replica. Note that the designation
of primary changes for different Roaring shards within an index,
even on a single node.
The essence of the the -fix repair operation that pilosa-fsck
can do is this: it will copy from the primary to the
the non-primary replicas. Further, it will remove data from
any non-replica node if it was mistakenly present.
The pilosa-fsck output log will contain
a sequence of command line 'cp' and 'rm' commands.
These commands are merely a record (with
accompanying justifcation in the comment following the
command) of what actions would be performed to repair
the Roaring file data.
Only with -fix will the repair actions actually happen
during the pilosa-fsck run.
Details: running pilosa-fsck
----------------------------
Errors in invocation are reported on stderr and the program will exit with a non-zero
error code if invocation errors are present. A non-zero error code
is returned if a repair is needed and -fix was not given.
A -fix run will return a zero error code to the shell if the fix was
successfully made; or if no fix was required.
The log of the run is printed to stdout.
The -h flag to pilosa-fsck prints a summary of its operation
and a guide to laying out the backup directories.
The help is reproduced below.
~~~
$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct 5 2020 1:28PM, 9dacbccf)
Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa
-fix
(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster.
-replicas R
(required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is
the number of replicas maintained in the cluster. Must be the same as the
[cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node.
-q
be very quiet during analysis and repair
Welcome to pilosa-fsck. This is a scan and repair
tool that is modeled after the classic unix file
system utility fsck.
WARNING: DO NOT RUN ON A LIVE SYSTEM.
The most important point to remember is that analysis
and repair must be done *offline*.
Just as fsck must be run on an unmounted disk,
pilosa-fsck must be run on a backup. It must
not be run on the directories where a live Pilosa system
is serving queries. Instead, take a backup first.
A backup is a set of N Pilosa data directories that have been
copied from your live system. They must all
be visible and mounted on one filesystem together.
pilosa-fsck can be run in scan-mode (without -fix),
or in repair-mode with -fix. The console output
supplies a log documenting the analysis
and showing what data changes would have been made.
REQUIRED COMMAND LINE ARGUMENTS
The paths to all the top-level Pilosa
data directories in a cluster must be given on the command
line. The -replicas R flag is also always required. It
must be correct for your cluser. Here R is the same as
the [cluster] stanza "replicas = R" line from your
pilosa.conf.
Example:
Suppose you are ready to run pilosa-fsck:
you have taken a backup of your four node Pilosa
cluster and stored it all on one filesystem with
all nodes visible and uncompressed. This
is a pre-requisite to running pilosa-fsck.
Let's suppose we have replication R = 3 set.
In this example, have stored our backed-up directories in
/backup/molecula
and the four node backups are in
subdirectories node1/ node2/ node3/ node4/ under this:
/backup/molecula/node1/
/backup/molecula/node1/.pilosa/.id
/backup/molecula/node1/.pilosa/.topology
/backup/molecula/node1/.pilosa/myindex
/backup/molecula/node2/
/backup/molecula/node2/.pilosa/.id
/backup/molecula/node2/.pilosa/.topology
/backup/molecula/node2/.pilosa/myindex
/backup/molecula/node3/
/backup/molecula/node3/.pilosa/.id
/backup/molecula/node3/.pilosa/.topology
/backup/molecula/node3/.pilosa/myindex
/backup/molecula/node4/
/backup/molecula/node4/.pilosa/.id
/backup/molecula/node4/.pilosa/.topology
/backup/molecula/node4/.pilosa/myindex
NOTE: your .pilosa directories need not be named .pilosa. They can
be something else, such as when the -d flag to pilosa server was used.
The .id file, the .topology file, and the index directories must be
found directly underneath.
Then a typical invocation to scan a cluster backup for issues:
$ cd /backup/molecula/
$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log
A typical invocation to repair the replication in the same backup:
$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log
In both cases, the .id and .topology files must
be present in the backups.
Without -fix, no modifications will be made to the backups. Only
by running with -fix will repairs be made. The user can safely
always run with -fix to repair only if needed.
A zero error code will be returned to the shell if no repairs were needed.
A zero error code will be also be returned to the shell if
repairs were needed and they were accomplished under -fix.
A non-zero error code indicates that repairs were needed but
were not made.
~~~

BIN
cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz
View file

Binary file not shown.

21
cmd/pilosa-fsck/release-pilosa-fsck/example.sh
View file

@ -1,21 +0,0 @@
#!/bin/bash
set +x
export PATH=.:${PATH}
# unpack the sample Molecula Pilosa cluster.
tar xf backups.tar.gz
# check if repair is needed.
pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
# yes, so do the repairs. This can be done first (only) as well.
#
pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa
# check again if you like
#
pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa

177
cmd/pilosa-fsck/vprint.go
View file

@ -1,177 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("# %s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) int64 {
fi, err := os.Stat(name)
if err != nil {
return 0
}
return fi.Size()
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

53
cmd/random-query/main.go
View file

@ -27,24 +27,24 @@ import (
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/pql"
)
// RandomQueryConfig
type RandomQueryConfig struct {
// user facing flags
HostPort string // -hostport
TreeDepth int // -d
QueryCount int // -n
Verbose bool // -v
VeryVerbose bool // -V
TimeFromArg string // --time.from
TimeToArg string // --time.to
TimeFrom time.Time // parsed time
TimeTo time.Time // parsed time
TimeRange int64 // hours between parsed times
HostPort string // -hostport
TreeDepth int // -d
QueryCount int // -n
Verbose bool // -v
VeryVerbose bool // -V
TimeFromArg string // --time.from
TimeToArg string // --time.to
TimeFrom time.Time // parsed time
TimeTo time.Time // parsed time
TimeRange int64 // hours between parsed times
IndexMap map[string]*Features
@ -73,7 +73,7 @@ type wrapper struct {
}
func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
return w.api.Schema(ctx), nil
return w.api.Schema(ctx, false)
}
func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
@ -234,11 +234,11 @@ NewSetup:
}
type Features struct {
Slc []IndexFieldRow
Ranges []IndexFieldRange
Slc []IndexFieldRow
Ranges []IndexFieldRange
Distinctables []IndexFieldRange
SlcWeight int
RangeWeight int
SlcWeight int
RangeWeight int
}
// Pick either a feature entry or a random query on a range, weighted
@ -274,7 +274,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
// anyway.
if fea.HasTime && cfg.Rnd.Int63n(20) != 0 {
startHours := (cfg.Rnd.Int63n(cfg.TimeRange - 1))
endHours := cfg.Rnd.Int63n(cfg.TimeRange - startHours) + 1 + startHours
endHours := cfg.Rnd.Int63n(cfg.TimeRange-startHours) + 1 + startHours
startTime := cfg.TimeFrom.Add(time.Duration(startHours) * time.Hour)
endTime := cfg.TimeFrom.Add(time.Duration(endHours) * time.Hour)
fromTo = fmt.Sprintf(", from=%s, to=%s",
@ -288,11 +288,11 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree {
}
type IndexFieldRange struct {
Index string
Field string
Index string
Field string
Min, Max, Scale int64
ScaleDiv float64
Range uint64
ScaleDiv float64
Range uint64
}
// We want to pick one of (1) a single-operation filter, (2) a
@ -316,8 +316,8 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
v2 = v2 + uint64(i.Min)
var v1s, v2s string
if i.Scale != 0 {
v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1)) / i.ScaleDiv)
v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2)) / i.ScaleDiv)
v1s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v1))/i.ScaleDiv)
v2s = fmt.Sprintf("%.*f", i.Scale, float64(int64(v2))/i.ScaleDiv)
} else {
v1s = strconv.FormatInt(int64(v1), 10)
v2s = strconv.FormatInt(int64(v2), 10)
@ -332,7 +332,7 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree {
if cfg.Rnd.Int63n(2) == 1 {
v1s = v2s
}
return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r - 4], v1s)}
return &Tree{S: fmt.Sprintf("Row(%s %s %s)", i.Field, binaryOps[r-4], v1s)}
}
}
@ -463,8 +463,8 @@ func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) {
}
type Tree struct {
Chd []*Tree
S string
Chd []*Tree
S string
Args []string // Extra args to pass after children, such as a field for Distinct.
}
@ -496,6 +496,7 @@ func (tr *Tree) StringIndent(ind int) (s string) {
}
const pilosaTimeFmt = "2006-01-02T15:04"
func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) {
features := cfg.IndexMap[index]
if depth == 0 {

2
cmd/random-query/main_test.go
View file

@ -116,7 +116,7 @@ func Test_RandomQuery(t *testing.T) {
timestamps = timestamps[:N]
}
// Import data with keys to the coordinator (node0) and verify that it gets
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: indexes[i],

6
cmd/root_test.go
View file

@ -189,11 +189,7 @@ bind = "127.0.0.1:10101"
[cluster]
replicas = 2
partitions = 128
hosts = [
"127.0.0.1:10101",
"127.0.0.1:10111",
]`
partitions = 128`
if _, err := file.Write([]byte(config)); err != nil {
t.Fatalf("writing config file: %v", err)
}

51
cmd/server_test.go
View file

@ -35,7 +35,13 @@ func TestServerHelp(t *testing.T) {
}
}
// I have no idea why the linter in ci is complaining about this being unused.
func nextPort() string { //nolint:unused
return fmt.Sprintf(`"localhost:%d"`, 0)
}
func TestServerConfig(t *testing.T) {
t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
actualDataDir, err := ioutil.TempDir("", "")
failErr(t, err, "making data dir")
logFile, err := ioutil.TempFile("", "")
@ -43,7 +49,7 @@ func TestServerConfig(t *testing.T) {
tests := []commandTest{
// TEST 0
{
args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
args: []string{"server", "--data-dir", actualDataDir, "--bind", "localhost:42454", "--bind-grpc", "localhost:30112", "--translation.map-size", "100000"},
env: map[string]string{
"PILOSA_DATA_DIR": "/tmp/myEnvDatadir",
"PILOSA_LONG_QUERY_TIME": "1m30s",
@ -54,17 +60,13 @@ func TestServerConfig(t *testing.T) {
},
cfgFileContent: `
data-dir = "/tmp/myFileDatadir"
bind = "localhost:0"
bind-grpc = "localhost:0"
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
max-writes-per-request = 3000
long-query-time = "1m10s"
[cluster]
disabled = true
replicas = 2
hosts = [
"localhost:19444",
]
long-query-time = "1m10s"
[profile]
block-rate = 100
@ -75,7 +77,6 @@ func TestServerConfig(t *testing.T) {
v.Check(cmd.Server.Config.DataDir, actualDataDir)
v.Check(cmd.Server.Config.Bind, "localhost:42454")
v.Check(cmd.Server.Config.Cluster.ReplicaN, 2)
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:42454", "localhost:10110"})
v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*90))
v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
@ -100,21 +101,15 @@ func TestServerConfig(t *testing.T) {
"PILOSA_PROFILE_MUTEX_FRACTION": "444",
},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
data-dir = "` + actualDataDir + `"
[cluster]
disabled = true
hosts = [
"localhost:19444",
]
[profile]
block-rate = 100
mutex-fraction = 10
`,
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
v.Check(cmd.Server.Config.Translation.MapSize, 100000)
v.Check(cmd.Server.Config.Profile.BlockRate, 4832)
@ -130,10 +125,6 @@ func TestServerConfig(t *testing.T) {
bind = "localhost:19444"
bind-grpc = "localhost:29444"
data-dir = "` + actualDataDir + `"
[cluster]
hosts = [
"localhost:19444",
]
[anti-entropy]
interval = "11m0s"
[metric]
@ -146,7 +137,6 @@ func TestServerConfig(t *testing.T) {
`,
validation: func() error {
v := validator{}
v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"})
v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11))
v.Check(cmd.Server.Config.LogPath, logFile.Name())
v.Check(cmd.Server.Config.Metric.Service, "statsd")
@ -198,6 +188,7 @@ func TestServerConfig(t *testing.T) {
}
}
func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
t.Skip("pilosa hosts config (cmd.Server.Config.Cluster.Hosts and brethren) is test only and will go away with high probability. skip for now.")
actualDataDir, err := ioutil.TempDir("", "")
failErr(t, err, "making data dir")
@ -207,11 +198,9 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
args: []string{"server", "--long-query-time", "1m10s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
data-dir = "` + actualDataDir + `"
[gossip]
port = "14321"
`,
validation: func() error {
v := validator{}
@ -225,10 +214,8 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
args: []string{"server", "--cluster.long-query-time", "1m20s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
[gossip]
port = "14321"
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
`,
validation: func() error {
v := validator{}
@ -242,10 +229,8 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) {
args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"},
env: map[string]string{},
cfgFileContent: `
bind = "localhost:0"
bind-grpc = "localhost:0"
[gossip]
port = "14321"
bind = ` + nextPort() + `
bind-grpc = ` + nextPort() + `
`,
validation: func() error {
v := validator{}

7
cmd/slurp/slurp.go
View file

@ -32,6 +32,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
pnet "github.com/pilosa/pilosa/v2/net"
)
// slurp: slurp is a load-tester for importing bulk data.
@ -191,7 +192,7 @@ func main() {
flag.StringVar(&tarSrcPath, "src", "q2.tar.gz", "data to import")
flag.Parse()
uri, err := pilosa.NewURIFromAddress(host)
uri, err := pnet.NewURIFromAddress(host)
panicOn(err)
globURI = uri
@ -253,9 +254,9 @@ func stopProfile(host, outfile string) {
}
var globURI *pilosa.URI
var globURI *pnet.URI
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
func GetImportRoaringURI(index string, shard uint64) *pnet.URI {
return globURI
}

2
ctl/import.go
View file

@ -261,7 +261,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, useColumnKeys, useRowK
func (cmd *ImportCommand) importBits(ctx context.Context, useColumnKeys, useRowKeys bool, bits []pilosa.Bit) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// If keys are used, all bits are sent to the primary translate store (i.e. coordinator).
// If keys are used, all bits are sent to the primary translate store.
if useColumnKeys || useRowKeys {
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.client.ImportK(ctx, cmd.Index, cmd.Field, bits, pilosa.OptImportOptionsClear(cmd.Clear)); err != nil {

76
ctl/server.go
View file

@ -18,86 +18,83 @@ import (
"fmt"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/storage"
"github.com/spf13/cobra"
)
// BuildServerFlags attaches a set of flags to the command for a server instance.
func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags := cmd.Flags()
flags.StringVar(&srv.Config.Name, "name", srv.Config.Name, "Name of the node in the cluster.")
flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.")
flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.")
flags.StringVar(&srv.Config.AdvertiseGRPC, "advertise-grpc", srv.Config.AdvertiseGRPC, "Address to advertise externally for gRPC.")
flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.IntVar(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
flags.Uint64Var(&srv.Config.MaxMapCount, "max-map-count", srv.Config.MaxMapCount, "Limits the maximum number of active mmaps. Pilosa will fall back to reading files once this is exhausted. Set below your system's vm.max_map_count.")
flags.Uint64Var(&srv.Config.MaxFileCount, "max-file-count", srv.Config.MaxFileCount, "Soft limit on the maximum number of fragment files Pilosa keeps open simultaneously.")
flags.DurationVarP((*time.Duration)(&srv.Config.LongQueryTime), "long-query-time", "", time.Duration(srv.Config.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
flags.DurationVar((*time.Duration)(&srv.Config.LongQueryTime), "long-query-time", time.Duration(srv.Config.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
flags.IntVar(&srv.Config.QueryHistoryLength, "query-history-length", srv.Config.QueryHistoryLength, "Number of queries to remember in history.")
// TLS
SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
// Handler
flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
flags.StringSliceVar(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", []string{}, "Comma separated list of allowed origin URIs (for CORS/Web UI).")
// Cluster
flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")
flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.")
flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.")
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Duration(srv.Config.Cluster.LongQueryTime), "RENAMED TO 'long-query-time': Duration that will trigger log and stat messages for slow queries.") // negative duration indicates invalid value because 0 is meaningful
flags.IntVar(&srv.Config.Cluster.ReplicaN, "cluster.replicas", 1, "Number of hosts each piece of data should be stored on.")
flags.DurationVar((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", time.Duration(srv.Config.Cluster.LongQueryTime), "RENAMED TO 'long-query-time': Duration that will trigger log and stat messages for slow queries.") // negative duration indicates invalid value because 0 is meaningful
flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.")
// Translation
flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
flags.StringVar(&srv.Config.Translation.PrimaryURL, "translation.primary-url", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
flags.IntVar(&srv.Config.Translation.MapSize, "translation.map-size", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringVarP(&srv.Config.Gossip.AdvertiseHost, "gossip.advertise-host", "", srv.Config.Gossip.AdvertiseHost, "Host on which memberlist should advertise.")
flags.StringVarP(&srv.Config.Gossip.AdvertisePort, "gossip.advertise-port", "", srv.Config.Gossip.AdvertisePort, "Port on which memberlist should advertise.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")
flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.")
flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.")
flags.IntVarP(&srv.Config.Gossip.Nodes, "gossip.nodes", "", srv.Config.Gossip.Nodes, "Number of random nodes to send gossip messages to per GossipInterval.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.Interval), "gossip.interval", "", (time.Duration)(srv.Config.Gossip.Interval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.")
flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ToTheDeadTime), "gossip.to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.ToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.")
// Etcd
// Etcd.Name used Config.Name for it's value.
// Etcd.Dir defaults to a directory under the pilosa data directory.
flags.StringVar(&srv.Config.Etcd.LClientURL, "etcd.listen-client-address", srv.Config.Etcd.LClientURL, "Listen client address.")
flags.StringVar(&srv.Config.Etcd.AClientURL, "etcd.advertise-client-address", srv.Config.Etcd.AClientURL, "Advertise client address. If not provided, uses the listen client address.")
flags.StringVar(&srv.Config.Etcd.LPeerURL, "etcd.listen-peer-address", srv.Config.Etcd.LPeerURL, "Listen peer address.")
flags.StringVar(&srv.Config.Etcd.APeerURL, "etcd.advertise-peer-address", srv.Config.Etcd.APeerURL, "Advertise peer address. If not provided, uses the listen peer address.")
flags.StringVar(&srv.Config.Etcd.ClusterURL, "etcd.cluster-url", srv.Config.Etcd.ClusterURL, "Cluster URL to join.")
// Etcd.ClusterName uses Cluster.Name for its value.
flags.StringVar(&srv.Config.Etcd.InitCluster, "etcd.initial-cluster", srv.Config.Etcd.InitCluster, "Initial cluster name1=apurl1,name2=apurl2")
// AntiEntropy
flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
flags.DurationVar((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.")
// Metric
flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
flags.StringVar(&srv.Config.Metric.Service, "metric.service", srv.Config.Metric.Service, "Where to send stats: can be expvar (in-memory served at /debug/vars), prometheus, statsd or none.")
flags.StringVar(&srv.Config.Metric.Host, "metric.host", srv.Config.Metric.Host, "URI to send metrics when metric.service is statsd.")
flags.DurationVar((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.")
flags.BoolVar((&srv.Config.Metric.Diagnostics), "metric.diagnostics", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.")
// Tracing
flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
flags.StringVar(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
flags.StringVar(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", srv.Config.Tracing.SamplerType, "Jaeger sampler type (remote, const, probabilistic, ratelimiting) or 'off' to disable tracing completely.")
flags.Float64Var(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
// Profiling
flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
// Transactional storage engine
// Note: the default for --tx must be kept "" empty string. Otherwise we
// cannot detect and honor the PILOSA_TXSRC env var over-ride.
flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --txsrc option on the command line.", pilosa.DefaultTxsrc))
// Storage
// Note: the default for --storage.backend must be kept "" empty string.
// Otherwise we cannot detect and honor the PILOSA_STORAGE_BACKEND env var
// over-ride.
// TODO: the comment above was carried over from the PILOSA_TXSRC flag, but
// we should confirm that this still applies.
flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk")
// RowcacheOn
flags.BoolVarP((&srv.Config.RowcacheOn), "rowcache-on", "", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)")
flags.BoolVar((&srv.Config.RowcacheOn), "rowcache-on", srv.Config.RowcacheOn, "turn on the rowcache for all backends (may speed some queries)")
// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.
srv.Config.RBFConfig.DefineFlags(flags)
@ -110,5 +107,4 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
}

11
ctl/server_test.go
View file

@ -35,14 +35,3 @@ func TestBuildServerFlags(t *testing.T) {
t.Fatal("log-path flag is required")
}
}
func TestServerDefaultTxsrcFlags(t *testing.T) {
cm := &cobra.Command{}
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
Server := server.NewCommand(stdin, stdout, stderr)
BuildServerFlags(cm, Server)
if cm.Flags().Lookup("txsrc").DefValue != "" {
t.Fatal("cannot set the txsrc default in ctl/server.go, otherwise we won't know to let the environment override the lack of --txsrc on the command line. We want explicit command line --txsrc to override the env value.")
}
}

68
dbshard.go
View file

@ -25,6 +25,8 @@ import (
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -61,7 +63,7 @@ type DBWrapper interface {
}
type DBRegistry interface {
OpenDBWrapper(path string, doAllocZero bool, rbfcfg *rbfcfg.Config) (DBWrapper, error)
OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error)
}
type DBShard struct {
@ -243,7 +245,8 @@ type DBPerShard struct {
isBlueGreen bool
RBFConfig *rbfcfg.Config
StorageConfig *storage.Config
RBFConfig *rbfcfg.Config
}
func newIndex2Shards() (r map[txtype]map[string]*shardSet) {
@ -252,7 +255,7 @@ func newIndex2Shards() (r map[txtype]map[string]*shardSet) {
}
type shardSet struct {
shards map[uint64]bool
shardsMap map[uint64]bool
shardsVer int64 // increment with each change.
// give out readonly to repeated consumers if
@ -269,11 +272,11 @@ func (a *shardSet) unionInPlace(b *shardSet) {
}
func (a *shardSet) equals(b *shardSet) bool {
if len(a.shards) != len(b.shards) {
if len(a.shardsMap) != len(b.shardsMap) {
return false
}
for shardInA := range a.shards {
_, ok := b.shards[shardInA]
for shardInA := range a.shardsMap {
_, ok := b.shardsMap[shardInA]
if !ok {
return false
}
@ -282,9 +285,17 @@ func (a *shardSet) equals(b *shardSet) bool {
}
func (a *shardSet) shards() []uint64 {
s := make([]uint64, 0, len(a.shardsMap))
for si := range a.shardsMap {
s = append(s, si)
}
return s
}
func (ss *shardSet) String() (r string) {
r = "["
for k := range ss.shards {
for k := range ss.shardsMap {
r += fmt.Sprintf("%v, ", k)
}
r += "]"
@ -292,9 +303,9 @@ func (ss *shardSet) String() (r string) {
}
func (ss *shardSet) add(shard uint64) {
_, already := ss.shards[shard]
_, already := ss.shardsMap[shard]
if !already {
ss.shards[shard] = true
ss.shardsMap[shard] = true
ss.shardsVer++
}
}
@ -315,7 +326,7 @@ func (ss *shardSet) CloneMaybe() map[uint64]bool {
// must make a fully new copy here.
ss.readonly = make(map[uint64]bool)
for k, v := range ss.shards {
for k, v := range ss.shardsMap {
ss.readonly[k] = v
}
ss.readonlyVer = ss.shardsVer
@ -324,12 +335,12 @@ func (ss *shardSet) CloneMaybe() map[uint64]bool {
func newShardSet() *shardSet {
return &shardSet{
shards: make(map[uint64]bool),
shardsMap: make(map[uint64]bool),
}
}
func newShardSetFromMap(m map[uint64]bool) *shardSet {
return &shardSet{
shards: m,
shardsMap: m,
shardsVer: 1,
}
}
@ -399,9 +410,8 @@ func (per *DBPerShard) LoadExistingDBs() (err error) {
}
func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) {
if holder.cfg == nil || holder.cfg.RBFConfig == nil {
panic("must have holder.cfg.RBFConfig set here")
if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil {
panic("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
}
useOpenList := 0
@ -421,17 +431,18 @@ func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Ho
}
d = &DBPerShard{
types: types,
HolderDir: holderDir,
holder: holder,
dbh: NewDBHolder(),
Flatmap: make(map[flatkey]*DBShard),
txf: txf,
useOpenList: useOpenList,
hasRoaring: hasRoaring,
isBlueGreen: len(types) > 1,
index2shards: newIndex2Shards(),
RBFConfig: holder.cfg.RBFConfig,
types: types,
HolderDir: holderDir,
holder: holder,
dbh: NewDBHolder(),
Flatmap: make(map[flatkey]*DBShard),
txf: txf,
useOpenList: useOpenList,
hasRoaring: hasRoaring,
isBlueGreen: len(types) > 1,
index2shards: newIndex2Shards(),
StorageConfig: holder.cfg.StorageConfig,
RBFConfig: holder.cfg.RBFConfig,
}
return
}
@ -645,13 +656,14 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
registry = globalRoaringReg
case rbfTxn:
registry = globalRbfDBReg
registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig)
case boltTxn:
registry = globalBoltReg
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
path := dbs.pathForType(ty)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.RBFConfig)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig)
panicOn(err)
h := idx.Holder()
w.SetHolder(h)
@ -921,7 +933,7 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor
// The blue is the destination -- this is always types[0].
// The green source is always types[1]. The mnemonic is blue_geen.
// The blue is first, so it is in types[0]. The green
// is second, in types[1]. For example, with PILOSA_TXSRC=bolt_roaring
// is second, in types[1]. For example, with PILOSA_STORAGE_BACKEND=bolt_roaring
// we have bolt as blue, and roaring as green. The contents of
// bolt must be empty or exactly match roaring. If bolt
// starts empty, it will be populated from roaring by

42
dbshard_internal_test.go
View file

@ -71,12 +71,9 @@ func TestShardPerDB_SetBit(t *testing.T) {
// test that we find all *local* shards
func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetShardsForIndex_LocalOnly")
panicOn(err)
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
defer os.RemoveAll(tmpdir)
v2s := NewFieldView2Shards()
stdShardSet := newShardSet()
@ -88,11 +85,9 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
}
for _, src := range []string{"roaring", "bolt", "rbf"} {
os.Setenv("PILOSA_TXSRC", src)
// must make Holder AFTER setting src.
holder := NewHolder(tmpdir, nil)
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = src
holder := NewHolder(tmpdir, cfg)
index := "rick"
idx := makeSampleRoaringDir(tmpdir, index, src, 1, holder, v2s)
@ -215,10 +210,9 @@ rick.index.txstores@@@/store-rbfdb@@/shard.0223-rbfdb@
`,
}
func makeSampleRoaringDir(root, index, txsrc string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
func makeSampleRoaringDir(root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
shards := []uint64{0, 93, 215, 217, 219, 221, 223}
fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
fns := strings.Split(sampleRoaringDirList[backend], "\n")
firstDone := false
for i, fn := range fns {
@ -226,11 +220,11 @@ func makeSampleRoaringDir(root, index, txsrc string, minBytes int, h *Holder, vi
continue
}
var shard uint64
if txsrc != "roaring" {
if backend != "roaring" {
// only have shards for the non-roaring
shard = shards[i]
}
switch txsrc {
switch backend {
case "bolt", "rbf":
idx = helperCreateDBShard(h, index, shard)
@ -327,19 +321,23 @@ func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shar
func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "Test_DBPerShard_GetFieldView2Shards_map_from_RBF")
panicOn(err)
defer os.RemoveAll(tmpdir)
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
os.Setenv("PILOSA_TXSRC", "rbf")
// must make Holder AFTER setting src.
holder := NewHolder(tmpdir, nil)
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = "rbf"
holder := NewHolder(tmpdir, cfg)
defer holder.Close()
index := "rick"
field := "f"
idx, err := holder.createIndex(index, IndexOptions{})
cim := &CreateIndexMessage{
Index: index,
CreatedAt: 0,
Meta: IndexOptions{},
}
idx, err := holder.createIndex(cim, false)
panicOn(err)
exp := NewFieldView2Shards()

2
dbshard_test.go
View file

@ -77,7 +77,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
// Keys are sharded so ordering is not guaranteed.
colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
// Import data with keys to the coordinator (node0) and verify that it gets
// Import data with keys to the primary and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: indexName,

2
debugstats/stats.go
View file

@ -17,7 +17,6 @@ package debugstats
import (
"fmt"
"math"
//"os"
"runtime"
"sort"
"sync"
@ -67,7 +66,6 @@ func (p SortByTot) Swap(i, j int) {
}
func (c *CallStats) Report(title string) (r string) {
//txsrc := os.Getenv("PILOSA_TXSRC")
r = fmt.Sprintf("CallStats: (%v)\n", title)
c.mu.Lock()
defer c.mu.Unlock()

7
diagnostics_internal_test.go
View file

@ -27,6 +27,8 @@ import (
)
func TestDiagnosticsClient(t *testing.T) {
t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()
@ -112,6 +114,8 @@ func TestDiagnosticsVersion_Compare(t *testing.T) {
}
func TestDiagnosticsVersion_Check(t *testing.T) {
t.Skip("does a listen on :0, skip for now. TODO(jea) restore this.")
// Mock server.
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
@ -146,6 +150,8 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
}
}
var _ = compareJSON
func compareJSON(a, b []byte) (bool, error) {
var j1, j2 interface{}
if err := json.Unmarshal(a, &j1); err != nil {
@ -158,6 +164,7 @@ func compareJSON(a, b []byte) (bool, error) {
}
func BenchmarkDiagnostics(b *testing.B) {
// Mock server.
server := httptest.NewServer(nil)
defer server.Close()

481
disco/disco.go Normal file
View file

@ -0,0 +1,481 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package disco
import (
"context"
"fmt"
"io"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
)
var (
ErrTooManyResults error = fmt.Errorf("too many results")
ErrNoResults error = fmt.Errorf("no results")
ErrKeyDeleted error = fmt.Errorf("key deleted")
ErrIndexExists error = fmt.Errorf("index already exists")
ErrIndexDoesNotExist error = fmt.Errorf("index does not exist")
ErrFieldExists error = fmt.Errorf("field already exists")
ErrFieldDoesNotExist error = fmt.Errorf("field does not exist")
ErrViewExists error = fmt.Errorf("view already exists")
ErrViewDoesNotExist error = fmt.Errorf("view does not exist")
)
type Peer struct {
URL string
ID string
}
func (p *Peer) String() string {
return fmt.Sprintf(`{"ID": "%s", "URL": "%s"}`, p.ID, p.URL)
}
type DisCo interface {
io.Closer
Start(ctx context.Context) (InitialClusterState, error)
IsLeader() bool
ID() string
Leader() *Peer
Peers() []*Peer
DeleteNode(ctx context.Context, id string) error
}
type (
InitialClusterState string
// ClusterState represents the state returned in the /status endpoint.
ClusterState string
)
const (
InitialClusterStateNew InitialClusterState = "new"
InitialClusterStateExisting InitialClusterState = "existing"
ClusterStateUnknown ClusterState = "UNKNOWN" // default cluster state. It is returned when we are not able to get the real actual state.
ClusterStateStarting ClusterState = "STARTING" // cluster is starting and some internal services are not ready yet.
ClusterStateDegraded ClusterState = "DEGRADED" // cluster is running but we've lost some # of hosts >0 but < replicaN. Only read queries are allowed.
ClusterStateNormal ClusterState = "NORMAL" // cluster is up and running.
ClusterStateResizing ClusterState = "RESIZING" // cluster is replicating data to other nodes.
ClusterStateDown ClusterState = "DOWN" // cluster is unable to serve queries.
)
type NodeState string
const (
NodeStateUnknown NodeState = "UNKNOWN"
NodeStateStarting NodeState = "STARTING"
NodeStateStarted NodeState = "STARTED"
NodeStateResizing NodeState = "RESIZING"
)
type Stator interface {
// Started will mark the actual node as already started.
// It must be called after all initialization processes
// are up and running.
Started(ctx context.Context) error
// ClusterState considers the state of all nodes and gives
// a general cluster state. The output calculation is as follows:
// - If any of the nodes are still starting: "STARTING"
// - If all nodes are up and running: "NORMAL"
// - If number of DOWN nodes is lower than number of replicas: "DEGRADED"
// - If number of unresponsive nodes is greater than (or equal to) the number of replicas: "DOWN"
// - If any of the nodes started a resize operation, or a new
// node was specifically added or removed from the cluster: "RESIZING"
ClusterState(context.Context) (ClusterState, error)
// NodeState returns the specific state of a node given its ID.
NodeState(context.Context, string) (NodeState, error)
// NodeStates will return all the states by node ID of the actual nodes in the cluster.
NodeStates(context.Context) (map[string]NodeState, error)
}
// Schema is a map of all indexes, each of those being a map of fields, then
// views.
type Schema map[string]*Index
// Index is a struct which contains the data encoded for the index as well as
// for each of its fields.
type Index struct {
Data []byte
Fields map[string]*Field
}
// Field is a struct which contains the data encoded for the field as well as
// for each of its views.
type Field struct {
Data []byte
Views map[string]struct{}
}
// Schemator is the source of truth for different schema elements.
// All nodes will store and retrieve information from the same source,
// having the same information at the same time.
type Schemator interface {
// Schema return the actual pilosa schema. If the schema is not present, an error is returned.
Schema(ctx context.Context) (Schema, error)
// Index gets a specific index data by name.
Index(ctx context.Context, name string) ([]byte, error)
CreateIndex(ctx context.Context, name string, val []byte) error
DeleteIndex(ctx context.Context, name string) error
Field(ctx context.Context, index, field string) ([]byte, error)
CreateField(ctx context.Context, index, field string, val []byte) error
DeleteField(ctx context.Context, index, field string) error
View(ctx context.Context, index, field, view string) (bool, error)
CreateView(ctx context.Context, index, field, view string) error
DeleteView(ctx context.Context, index, field, view string) error
}
// Metadator is in charge of storing specific metadata per node.
// This metadata can be retrieved by any node using the specific peerID.
type Metadator interface {
Metadata(ctx context.Context, peerID string) ([]byte, error)
SetMetadata(ctx context.Context, metadata []byte) error
}
// Resizer triggers resizing the node and changes cluster state into RESIZING.
// We can also return some kind of handler from Resize function (e.g. key-value)
type Resizer interface {
// Resize will trigger a resize event. Node state will change to RESIZE state.
// The returned function can be used to send info about the resize process to other nodes.
Resize(ctx context.Context) (func([]byte) error, error)
// DoneResize will mark the resize event as done. This will be called when all the resize actions are done.
DoneResize() error
// Watch will give information about a resize event in another node, using its peerID.
// onUpdate function will be called per each event sent by the node in RESIZE state.
Watch(ctx context.Context, peerID string, onUpdate func([]byte) error) error
}
// Sharder is an interface used to maintain the set of availableShards bitmaps
// per field.
type Sharder interface {
Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error)
AddShard(ctx context.Context, index, field string, shard uint64) error
AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error)
RemoveShard(ctx context.Context, index, field string, shard uint64) error
}
// NopDisCo represents a DisCo that doesn't do anything.
var NopDisCo DisCo = &nopDisCo{}
type nopDisCo struct{}
// Close no-op.
func (n *nopDisCo) Close() error {
return nil
}
// Start is a no-op implementation of the DisCo Start method.
func (n *nopDisCo) Start(ctx context.Context) (InitialClusterState, error) {
return InitialClusterStateNew, nil
}
// ID is a no-op implementation of the DisCo ID method.
func (n *nopDisCo) ID() string {
return ""
}
// IsLeader is a no-op implementation of the DisCo IsLeader method.
func (n *nopDisCo) IsLeader() bool {
return false
}
// Leader is a no-op implementation of the DisCo Leader method.
func (n *nopDisCo) Leader() *Peer {
return nil
}
// Peers is a no-op implementation of the DisCo Peers method.
func (n *nopDisCo) Peers() []*Peer {
return nil
}
// DeleteNode a no-op implementation of the DisCo DeleteNode method.
func (n *nopDisCo) DeleteNode(context.Context, string) error {
return nil
}
// NopStator represents a Stator that doesn't do anything.
var NopStator Stator = &nopStator{}
type nopStator struct{}
// ClusterState is a no-op implementation of the Stator ClusterState method.
func (n *nopStator) ClusterState(context.Context) (ClusterState, error) {
return ClusterStateUnknown, nil
}
func (n *nopStator) Started(ctx context.Context) error {
return nil
}
func (n *nopStator) NodeState(context.Context, string) (NodeState, error) {
return NodeStateUnknown, nil
}
func (n *nopStator) NodeStates(context.Context) (map[string]NodeState, error) {
return nil, nil
}
// NopMetadator represents a Metadator that doesn't do anything.
var NopMetadator Metadator = &nopMetadator{}
type nopMetadator struct{}
func (*nopMetadator) Metadata(context.Context, string) ([]byte, error) {
return nil, nil
}
func (*nopMetadator) SetMetadata(context.Context, []byte) error {
return nil
}
// NopResizer represents a Resizer that doesn't do anything.
var NopResizer Resizer = &nopResizer{}
type nopResizer struct{}
func (*nopResizer) Resize(context.Context) (func([]byte) error, error) { return nil, nil }
func (*nopResizer) DoneResize() error { return nil }
func (*nopResizer) Watch(context.Context, string, func([]byte) error) error { return nil }
// NopSharder represents a Sharder that doesn't do anything.
var NopSharder Sharder = &nopSharder{}
type nopSharder struct{}
// Shards is a no-op implementation of the Sharder Shards method.
func (n *nopSharder) Shards(ctx context.Context, index, field string) (*roaring.Bitmap, error) {
return nil, nil
}
// AddShard is a no-op implementation of the Sharder AddShard method.
func (n *nopSharder) AddShard(ctx context.Context, index, field string, shard uint64) error {
return nil
}
// AddShards is a no-op implementation of the Sharder AddShards method.
func (n *nopSharder) AddShards(ctx context.Context, index, field string, shards *roaring.Bitmap) (*roaring.Bitmap, error) {
return nil, nil
}
// RemoveShard is a no-op implementation of the Sharder RemoveShard method.
func (n *nopSharder) RemoveShard(ctx context.Context, index, field string, shard uint64) error {
return nil
}
// NopSchemator represents a Schemator that doesn't do anything.
var NopSchemator Schemator = &nopSchemator{}
type nopSchemator struct{}
// Schema is a no-op implementation of the Schemator Schema method.
func (*nopSchemator) Schema(ctx context.Context) (Schema, error) { return nil, nil }
// Index is a no-op implementation of the Schemator Index method.
func (*nopSchemator) Index(ctx context.Context, name string) ([]byte, error) { return nil, nil }
// CreateIndex is a no-op implementation of the Schemator CreateIndex method.
func (*nopSchemator) CreateIndex(ctx context.Context, name string, val []byte) error { return nil }
// DeleteIndex is a no-op implementation of the Schemator DeleteIndex method.
func (*nopSchemator) DeleteIndex(ctx context.Context, name string) error { return nil }
// Field is a no-op implementation of the Schemator Field method.
func (*nopSchemator) Field(ctx context.Context, index, field string) ([]byte, error) { return nil, nil }
// CreateField is a no-op implementation of the Schemator CreateField method.
func (*nopSchemator) CreateField(ctx context.Context, index, field string, val []byte) error {
return nil
}
// DeleteField is a no-op implementation of the Schemator DeleteField method.
func (*nopSchemator) DeleteField(ctx context.Context, index, field string) error { return nil }
// View is a no-op implementation of the Schemator View method.
func (*nopSchemator) View(ctx context.Context, index, field, view string) (bool, error) {
return false, nil
}
// CreateView is a no-op implementation of the Schemator CreateView method.
func (*nopSchemator) CreateView(ctx context.Context, index, field, view string) error {
return nil
}
// DeleteView is a no-op implementation of the Schemator DeleteView method.
func (*nopSchemator) DeleteView(ctx context.Context, index, field, view string) error { return nil }
// InMemSchemator represents a Schemator that manages the schema in memory. The
// intention is that this would be used for testing.
var InMemSchemator Schemator = &inMemSchemator{
schema: make(Schema),
}
type inMemSchemator struct {
mu sync.RWMutex
schema Schema
}
// Schema is an in-memory implementation of the Schemator Schema method.
func (s *inMemSchemator) Schema(ctx context.Context) (Schema, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.schema, nil
}
// Index is an in-memory implementation of the Schemator Index method.
func (s *inMemSchemator) Index(ctx context.Context, name string) ([]byte, error) {
s.mu.RLock()
defer s.mu.RUnlock()
idx, ok := s.schema[name]
if !ok {
return nil, ErrIndexDoesNotExist
}
return idx.Data, nil
}
// CreateIndex is an in-memory implementation of the Schemator CreateIndex method.
func (s *inMemSchemator) CreateIndex(ctx context.Context, name string, val []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
if idx, ok := s.schema[name]; ok {
// The current logic in pilosa doesn't allow us to return ErrIndexExists
// here, so for now we just update the Data value if the index already
// exists.
idx.Data = val
return nil
}
s.schema[name] = &Index{
Data: val,
Fields: make(map[string]*Field),
}
return nil
}
// DeleteIndex is an in-memory implementation of the Schemator DeleteIndex method.
func (s *inMemSchemator) DeleteIndex(ctx context.Context, name string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.schema, name)
return nil
}
// Field is an in-memory implementation of the Schemator Field method.
func (s *inMemSchemator) Field(ctx context.Context, index, field string) ([]byte, error) {
s.mu.RLock()
defer s.mu.RUnlock()
idx, ok := s.schema[index]
if !ok {
return nil, ErrIndexDoesNotExist
}
fld, ok := idx.Fields[field]
if !ok {
return nil, ErrFieldDoesNotExist
}
return fld.Data, nil
}
// CreateField is an in-memory implementation of the Schemator CreateField method.
func (s *inMemSchemator) CreateField(ctx context.Context, index, field string, val []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
idx, ok := s.schema[index]
if !ok {
return ErrIndexDoesNotExist
}
if fld, ok := idx.Fields[field]; ok {
// The current logic in pilosa doesn't allow us to return ErrFieldExists
// here, so for now we just update the Data value if the field already
// exists.
fld.Data = val
return nil
}
idx.Fields[field] = &Field{
Data: val,
Views: make(map[string]struct{}),
}
return nil
}
// DeleteField is an in-memory implementation of the Schemator DeleteField method.
func (s *inMemSchemator) DeleteField(ctx context.Context, index, field string) error {
s.mu.Lock()
defer s.mu.Unlock()
idx, ok := s.schema[index]
if !ok {
return ErrIndexDoesNotExist
}
delete(idx.Fields, field)
return nil
}
// View is an in-memory implementation of the Schemator View method.
func (s *inMemSchemator) View(ctx context.Context, index, field, view string) (bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
idx, ok := s.schema[index]
if !ok {
return false, ErrIndexDoesNotExist
}
fld, ok := idx.Fields[field]
if !ok {
return false, ErrFieldDoesNotExist
}
_, ok = fld.Views[view]
return ok, nil
}
// CreateView is an in-memory implementation of the Schemator CreateView method.
func (s *inMemSchemator) CreateView(ctx context.Context, index, field, view string) error {
s.mu.Lock()
defer s.mu.Unlock()
idx, ok := s.schema[index]
if !ok {
return ErrIndexDoesNotExist
}
fld, ok := idx.Fields[field]
if !ok {
return ErrFieldDoesNotExist
}
// The current logic in pilosa doesn't allow us to return ErrViewExists
// here, so for now we just update the value if the view already exists.
fld.Views[view] = struct{}{}
return nil
}
// DeleteView is an in-memory implementation of the Schemator DeleteView method.
func (s *inMemSchemator) DeleteView(ctx context.Context, index, field, view string) error {
s.mu.Lock()
defer s.mu.Unlock()
idx, ok := s.schema[index]
if !ok {
return ErrIndexDoesNotExist
}
fld, ok := idx.Fields[field]
if !ok {
return ErrFieldDoesNotExist
}
delete(fld.Views, view)
return nil
}

5
docs/README.md
View file

@ -1,5 +0,0 @@
Pilosa docs are maintained here, to stay in sync with the codebase. The format is [Blackfriday](https://github.com/russross/blackfriday) markdown, with some Hugo [front matter](https://gohugo.io/content-management/front-matter/).
Please visit [our website](https://www.pilosa.com/docs/) to view the docs complete with styles, diagrams, and comprehensive search. Internal links will only work on the website.
Have you found a discrepancy, typo, or other problem? Please submit an [issue](https://github.com/pilosa/pilosa/issues/new) or a pull request!

327
docs/administration.md
View file

@ -1,327 +0,0 @@
+++
title = "Administration"
weight = 13
nav = [
"Installing in production",
"Importing and Exporting Data",
"Versioning",
"Resizing the Cluster",
"Backup/restore",
]
+++
## Administration Guide
### Installing in production
#### Hardware
Pilosa is a standalone, compiled Go application, so there is no need to worry about running and configuring a Java VM. Pilosa can run on very small machines and works well with even a medium sized dataset on a personal laptop. If you are reading this section, you are likely ready to deploy a cluster of Pilosa servers handling very large datasets or high velocity data. These are guidelines for running a cluster; specific needs may differ.
#### Memory
Pilosa holds all row/column bitmap data in main memory. While this data is compressed more than a typical database, available memory is a primary concern. In a production environment, we recommend choosing hardware with a large amount of memory >= 64GB. Prefer a small number of hosts with lots of memory per host over a larger number with less memory each. Larger clusters tend to be less efficient overall due to increased inter-node communication.
#### CPUs
Pilosa is a concurrent application written in Go and can take full advantage of multicore machines. The main unit of parallelism is the [shard](../data-model/#shard), so a single query will only use a number of cores up to the number of shards stored on that host. Multiple queries can still take advantage of multiple cores as well, so tuning in this area is dependent upon the expected workload.
#### Disk
Even though the main dataset is in memory Pilosa backs up to disk frequently. We recommend SSDs—especially if you have a write-heavy application.
#### Network
Pilosa is designed to be a distributed application, with data replication replicated across the cluster. As such, every write and read needs to communicate with several nodes. Therefore fast internode communication is essential. If using a service like AWS we recommend that all nodes exist in the same region and availability zone. The inherent latency of spreading a Pilosa cluster across physical regions is not usually worth the redundancy protection. Since Pilosa is designed to be an indexing service there should already be a system of record, or ability to rebuild a cluster quickly from backups.
#### Overview
While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines, as the internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time.
### Open File Limits
Pilosa requires a large number of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa.
On Mac OS X, `ulimit` does not behave predictably. The Mac OS X system has a utility called csrutil that prevents you from changing the open file limit easily. One workaround that may work for you involves disabling the csrutil program. To disable the csrutil program, restart your laptop and when the start up screen pops up, hold down command + R to enter Recovery Mode. Open a terminal and enter `csrutil disable`, then restart your computer as you normally would. Now that the csrutil is disabled, you can change the open file limit. The open file limit can be changed by creating the following files and changing their ownership:
Copy the contents of [this](https://github.com/wilsonmar/mac-setup/blob/master/configs/limit.maxfiles.plist) file into a new file on your system located at /Library/LaunchDaemons/limit.maxfiles.plist, then run:
```
sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
```
Copy the contents of [this](https://github.com/wilsonmar/mac-setup/blob/master/configs/limit.maxproc.plist) file into a new file on your system located at /Library/LaunchDaemons/limit.maxproc.plist, then run:
```
sudo chown root:wheel /Library/LaunchDaemons/limit.maxproc.plist
```
To ensure the open file limit has successfully changed, run `ulimit -a`. Your open files should be set to a number greater than 256 (in the range of 524288) and your max users processes should be greater than 709 (in the range of 2048).
### Importing and Exporting Data
#### Importing
The import API expects a csv of the format `Row,Column`.
When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results.
```
pilosa import --sort -i project -f stargazer project-stargazer.csv
```
We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at:
* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md)
* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md)
* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md)
##### Importing Integer Values
If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`.
```
pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
```
##### Importing Boolean Values
If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true).
For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row.
```
0,3
0,9
1,1
1,2
1,4
1,8
```
<div class="note">
<p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
</div>
#### Clearing Data via Import
By using the `--clear` flag with the import command, Pilosa will clear the values provided in the import payload.
For example, importing a file with the following contents along with the `--clear` flag will result in data being cleared from row 0, column 9; row 1, columns 2 and 8; and row 3, column 12. Clearing a value that doesn't exists is allowed.
```
0,9
1,2
1,8
3,12
```
#### Exporting
Exporting data to csv can be performed on a live instance of Pilosa. You need to specify the index and the field. The API also expects the shard number, but the `pilosa export` sub command will export all shards within a field. The data will be in csv format `Row,Column` and sorted by column.
```request
curl "http://localhost:10101/export?index=repository&field=stargazer&shard=0" \
--header "Accept: text/csv"
```
```response
2,10
2,30
3,426
4,2
...
```
### Versioning
Pilosa follows [Semantic Versioning](http://semver.org/).
MAJOR.MINOR.PATCH:
* MAJOR version when you make incompatible API changes,
* MINOR version when you add functionality in a backwards-compatible manner, and
* PATCH version when you make backwards-compatible bug fixes.
#### PQL versioning
The Pilosa server should support PQL versioning using HTTP headers. On each request, the client should send a Content-Type header and an Accept header. The server should respond with a Content-Type header that matches the client Accept header. The server should also optionally respond with a Warning header if a PQL version is in a deprecation period, or an HTTP 400 error if a PQL version is no longer supported.
#### Upgrading
To upgrade Pilosa:
1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`.
2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases).
3. Shut down the Pilosa cluster.
4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node.
5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make.
6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes.
##### Version 1.4
Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release.
### Resizing the Cluster
If you need to increase (or decrease) the capacity of a Pilosa server, you can add or remove nodes to a running cluster at any time. Note that you can only add or remove one node at a time; if you attempt to add multiple nodes at once, those requests will be enqueued and processed serially. Also note that during any resize process, the cluster goes into state `RESIZING` during which all read/write requests are denied. When the cluster returns to state `NORMAL` then read/write operations can resume. The amount of time that the cluster stays in state `RESIZING` depends on the amount of data that needs to be moved during the resize process.
#### Adding a Node
You can add a new, empty node to an existing cluster by starting `pilosa server` on the new node with the correct configuration options. Specifically, you must specify the [cluster coordinator](../configuration/#cluster-coordinator) to be the same as the coordinator on the existing nodes. You must also specify at least one valid [gossip seed](../configuration/#gossip-seeds) (preferably multiple for redundancy). When the new node starts, the coordinator node will receive a `nodeJoin` event indicating that a new node is joining the cluster. At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the additional capacity of the new node. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the new node is included in future queries.
If the node is being added to a cluster which contains no data (for example, during startup of a new cluster), the coordinator will bypass the `RESIZING` state and allow the node to join the cluster immediately.
#### Removing a Node
In order to remove a node from a cluster, your cluster must be configured to have a [cluster replicas](../configuration/#cluster-replicas) value of at least 2; if you're removing a node that no longer exists (for example a node that has died), there must be at least one additional replica of the data owned by the dead node in order for the cluster to correctly rebalance itself.
To remove node `localhost:10102` from a cluster having coordinator `localhost:10101`, first determine the ID of the node to be removed. If the node to be removed is still available, you can find the ID by issuing a `/status` request to the node. The node's ID is in the `localID` field:
``` request
curl localhost:10101/status
```
``` response
{
"state":"NORMAL",
"nodes":[
{"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
{"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
{"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
],
"localID": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"
}
```
If the node to be removed is no longer available, you can get the IDs of the nodes in the cluster by issuing a `/status` request to any available node:
``` request
curl localhost:10101/status
```
``` response
{
"state":"NORMAL",
"nodes":[
{"id":"24824777-62ec-4151-9fbd-67e4676e317d","uri":{"scheme":"http","host":"localhost","port":10101}}
{"id":"40a891fa-243b-4d71-ae24-4f5c78a0f4b1","uri":{"scheme":"http","host":"localhost","port":10102}}
{"id":"9fab09cc-3c26-4202-9622-d167c84684d9","uri":{"scheme":"http","host":"localhost","port":10103}}
],
"localID": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"
}
```
Once you have the ID of the node that you want to remove from the cluster, issue the following request:
```
curl localhost:10101/cluster/resize/remove-node \
-X POST \
-d '{"id": "40a891fa-243b-4d71-ae24-4f5c78a0f4b1"}'
```
At this point, the coordinator will put the cluster into state `RESIZING` and kick off a resize job that instructs all of the nodes in the cluster how to rebalance data to accomodate the reduced capacity of the cluster. Once the resize job is complete, the coordinator will put the cluster back to state `NORMAL` and ensure that the removed node is no longer included in future queries.
Note that you can't directly remove the coordinator node. If you need to remove the coordinator node from the cluster, you must first [make one of the other nodes the coordinator](#changing-the-coordinator).
#### Aborting a Resize Job
If at any point you need to abort an active resize job, you can issue a `POST` request to the `/cluster/resize/abort` endpoint on the coordinator node.
For example, if your coordinator node is `localhost:10101`, then you can run:
```
curl localhost:10101/cluster/resize/abort -X POST
```
This will immediately abort the resize job and return the cluster to state `NORMAL`. Because data is never removed from a node during a resize job (only once a resize job has successfully completed), aborting a resize job will return the cluster back to the state it was in before the resize began.
#### Changing the Coordinator
In order to assign a different node to be the coordinator, you can issue a `/cluster/resize/set-coordinator` request to any node in the cluster. The payload should indicate the ID of the node to be made coordinator.
```
curl localhost:10101/cluster/resize/set-coordinator \
-X POST \
-d '{"id": "9fab09cc-3c26-4202-9622-d167c84684d9"}'
```
### Backup/restore
Pilosa continuously writes out the in-memory bitmap data to disk. This data is organized by Index->Field->Views->Fragment->numbered shard files. These data files can be routinely backed up to restore nodes in a cluster.
Depending on the size of your data you have two options. For a small dataset you can rely on the periodic anti-entropy sync process to replicate existing data back to this node.
For larger datasets and to make this process faster you could copy the relevant data files from the other nodes to the new one before startup.
Note: This will only work when the replication factor is >= 2
#### Using Index Sync
- Shutdown the cluster.
- Modify config file to replace existing node address with new node.
- Restart all nodes in the cluster.
- Wait for auto Index sync to replicate data from existing nodes to new node.
#### Copying data files manually
- To accomplish this you will first need:
- List of all indexes on your cluster
- List of all fields in your indexes
- Max shard per index, listed in the `/internal/shards/max` endpoint
- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each shard
- Using the list of shards owned by this node you will then need to manually:
- setup a directory structure similar to the other nodes with a path for each Index/Field
- copy each owned shard for an existing node to this new node
- Modify the cluster config file to replace the previous node address with the new node address.
- Restart the cluster
- Wait for the first sync (10 minutes) to validate Index connections
### Diagnostics
Each Pilosa cluster is configured by default to share anonymous usage details with Pilosa Corp. These metrics allow us to understand how Pilosa is used by the community and improve the technology to suit your needs. Diagnostics are sent to Pilosa every hour. Each of the metrics are detailed below as well as opt-out instructions.
- **Version:** Version string of the build.
- **Host:** Host URI.
- **Cluster:** List of nodes in the cluster.
- **NumNodes:** Number of nodes in the cluster.
- **NumCPU:** Number of cores per node
- **BSIEnabled:** Bit Sliced Index Fields in use.
- **TimeQuantumEnabled:** Time Quantum Fields in use.
- **NumIndexes:** Number of indexes in the Cluster.
- **NumFields:** Number of fields in the Cluster.
- **NumShards:** Number of shards in the Cluster.
- **NumViews:** Number of views in the Cluster.
- **OpenFiles:** Open file handle count.
- **GoRoutines:** Go routine count.
You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option.
### Metrics
Pilosa can be configured to emit metrics pertaining to its internal processes in one of three formats: Expvar, StatsD, or Prometheus. Metric recording is disabled by default.
The metrics configuration options are:
- [Host](../configuration/#metric-host): specify host that receives metric events
- [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics
- [Service](../configuration/#metric-service): declare type StatsD or Expvar
#### Tags
StatsD Tags adhere to the DataDog format (key:value), and we tag the following:
- NodeID
- Index
- Field
- View
- Shard
#### Events
We currently track the following events
- **Index:** The creation of a new index.
- **Field:** The creation of a new field.
- **MaxShard:** The creation of a new Shard.
- **SetBit:** Count of set bits.
- **ClearBit:** Count of cleared bits.
- **ImportBit:** During a bulk data import this represents the count of bits created.
- **SetRowAttrs:** Count of attributes set per row.
- **SetColumnAttrs:** Count of attributes set per column.
- **Bitmap:** Count of Bitmap queries.
- **TopN:** Count of TopN queries.
- **Union:** Count of Union queries.
- **Intersection:** Count of Intersection queries.
- **Difference:** Count of Difference queries.
- **Xor:** Count of Xor queries.
- **Not:** Count of Not queries.
- **Count:** Count of Count queries.
- **Range:** Count of ranged Row queries.
- **Snapshot:** Event count when the snapshot process is triggered.
- **BlockRepair:** Count of data blocks that were out of sync and repaired.
- **GarbageCollection:** Event count when garbage collection occurs.
- **Goroutines:** Number of running goroutines.
- **OpenFiles:** Number of open file handles associated with running Pilosa process ID.

415
docs/api-reference.md
View file

@ -1,415 +0,0 @@
+++
title = "API Reference"
weight = 10
nav = []
+++
## API Reference
### List all index schemas
`GET /index`
Is equivalent to `GET /schema` and returns the same response.
### List index schema
`GET /index/{index-name}`
Returns the schema of the specified index in JSON.
``` request
curl -XGET localhost:10101/index/user
```
``` response
{
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
}
],
"shardWidth": 1048576
}
```
### Create index
`POST /index/{index-name}`
Creates an index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object with the following options:
* `keys` (bool): Enables using column keys instead of column IDs.
* `trackExistence` (bool): Enables or disables existence tracking on the index. Required for [Not](../query-language/#not) queries. It is `true` by default.
``` request
curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
```
``` response
{"success":true,"name":"user","createdAt":1591179042178854000}
```
### Remove index
`DELETE /index/index-name`
Removes the given index.
``` request
curl -XDELETE localhost:10101/index/user
```
``` response
{"success":true}
```
### Query index
`POST /index/{index-name}/query`
Sends a [query](../query-language/) to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
``` request
curl localhost:10101/index/user/query \
-X POST \
-d 'Row(language=5)'
```
``` response
{
"results": [
{
"attrs": {},
"columns": [
100
]
}
]
}
```
In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
The query is executed for all [shards](../data-model/#shard) by default. To use specified shards only, set the `shards` query argument to a comma-separated list of slice indices.
``` request
curl "localhost:10101/index/user/query?columnAttrs=true&shards=0,1" \
-X POST \
-d 'Row(language=5)'
```
``` response
{
"columnAttrs": [
{
"attrs": {
"name": "Klingon"
},
"id": 100
}
],
"results": [
{
"attrs": {},
"columns": [
100
]
}
]
}
```
By default, all bits and attributes (*for `Row` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
### Import Data
`POST /index/{index-name}/field/{field-name}/import`
Supports high-rate data ingest to a particular shard of a particular field. The
official client libraries use this endpoint for their import functionality - it
is not usually necessary to use this endpoint directly. See the documentation for
imports for
<a href="https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md">Go</a>,
<a href="https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md">Java</a>,
and <a href="https://github.com/pilosa/python-pilosa/tree/master/docs/imports.md">Python</a>.
The request payload is protobuf encoded with the following schema. The RowKeys
and/or ColumnKeys fields are used if the pilosa field or index are configured
for keys respectively. Otherwise, the RowIDs and ColumnIDs fields are used. They
must have the same number of items, and each index into those two lists
represents a particular bit to be set. Timestamps are optional, but if they
exist must also contain the same number of items as rows and columns. The
column IDs must all be in the shard specified in the request.
Some endpoints and data structures include a `CreatedAt` fields.
This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
but to serve as a unique identifier for use in cache invalidation.
The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
This is true except in cases where an index or field gets deleted and then recreated,
or if Pilosa is restored from a backup.
So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
and the ingester will know that it needs to drop its cache.
```
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated int64 Timestamps = 6;
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
int64 IndexCreatedAt = 9;
int64 FieldCreatedAt = 10;
}
```
### Create field
`POST /index/{index-name}/field/{field-name}`
Creates a field in the given index with the given name.
The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which must contain a `type`:
* `type` (string): Sets the field type and type options.
* `keys` (bool): Enables using column keys instead of column IDs (optional).
Valid `type`s and correspondonding options are listed below:
* `set`
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `ranked`.
* `cacheSize` (int): Number of rows to keep in the cache. Default is 50,000.
* `int`
* `min` (int): Minimum integer value allowed for the field.
* `max` (int): Maximum integer value allowed for the field.
* `bool`
* (boolean fields take no arguments)
* `time`
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
* `mutex`
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this field. Default is `ranked`.
* `cacheSize` (int): Number of rows to keep in the cache. Default is 50,000.
The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
``` request
curl localhost:10101/index/user/field/quantity \
-X POST \
-d '{"options": {"type": "int", "min": -1000, "max":2000}}'
```
``` response
{"success":true,"name":"quantity","createdAt":1591180110914425000}
```
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
``` request
curl localhost:10101/index/user/field/language -X POST
```
``` response
{"success":true,"name":"language","createdAt":1591180128294321000}
```
``` request
curl localhost:10101/index/repository/field/stats \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
```
``` response
{"success":true,"name":"stats","createdAt":1591180737881627000}
```
### Remove field
`DELETE /index/{index-name}/field/{field-name}`
Removes the given field.
``` request
curl -XDELETE localhost:10101/index/user/field/language
```
``` response
{"success":true}
```
### List all index schemas
`GET /schema`
Returns the schema of all indexes in JSON.
``` request
curl -XGET localhost:10101/schema
```
``` response
{
"indexes": [
{
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "language",
"createdAt": 1591180128294321000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "quantity",
"createdAt": 1591180110914425000,
"options": {
"type": "int",
"base": 0,
"bitDepth": 0,
"min": -1000,
"max": 2000,
"keys": false,
"foreignIndex": ""
}
}
],
"shardWidth": 1048576
}
]
}
```
### Duplicate schema into empty Pilosa cluster
`POST /schema`
To duplicate one Pilosa cluster's schema to another, it's possible to
pass the output of `GET /schema` as the request body of `POST /schema`
and all the indexes and fields in the schema will be created in
Pilosa. As of this writing, the behavior of POSTing a schema to a
non-empty Pilosa cluster is undefined. These semantics will likely be
ironed out in a future version.
``` request
# after (e.g.) curl -XGET localhost:10101/schema > schema.json
curl -XPOST localhost:10101/schema --data-binary @schema.json
```
Response: `204 No Content`
### Get version
`GET /version`
Returns the version of the Pilosa server.
``` request
curl -XGET localhost:10101/version
```
``` response
{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
```
### Get status
`GET /status`
Returns the status of the cluster.
```request
curl -XGET localhost:10101/status
```
```response
{
"state": "NORMAL",
"nodes": [
{
"id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
"uri": {
"scheme": "http",
"host": "localhost",
"port": 10101
},
"grpc-uri": {
"scheme": "http",
"host": "localhost",
"port": 20101
},
"isCoordinator": true,
"state": "READY"
}
],
"localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
}
```
### Get active queries
`GET /queries`
Returns the set of active queries. Supports pretty printing in `text/plain` format or JSON output in `application/json` format.
Also includes the amount of time that the query has been running (in nanoseconds when using JSON).
```request
curl -XGET localhost:10101/queries
```
```response
182.412µs All()
```
```request
curl -XGET -H "Accept: application/json" localhost:10101/queries
```
```response
[{"query":"All()","age":135123}]
```
### Recalculate Caches
`POST /recalculate-caches`
Recalculates the caches on demand. The cache is recalculated every 10
seconds by default. This endpoint can be used to recalculate the cache
before the 10 second interval. This should probably only be used in
integration tests and not in a typical production workflow. Note that
in a multi-node cluster, the cache is only recalculated on the node
that receives the request.
``` request
curl -XPOST localhost:10101/recalculate-caches
```
Response: `204 No Content`

25
docs/architecture.md
View file

@ -1,25 +0,0 @@
+++
title = "Architecture"
weight = 6
nav = []
+++
## Architecture
### Roaring bitmap storage format
Bitmaps are persisted to disk using a file format very similar to the [Roaring Bitmap format spec](https://github.com/RoaringBitmap/RoaringFormatSpec). Pilosa's format uses 64-bit IDs, so it is not binary-compatible with the spec. Some parts of the format are simpler, and an additional section is included. Specific differences include:
* The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3.
* The cookie includes file format version in bytes 2-3 (currently equal to zero).
* The descriptive header includes, for each container, a 64-bit key, a 16-bit cardinality, and a 16-bit container type (which only uses two bits now). This makes the runFlag bitset unnecessary. This is in contrast to the spec, which stores a 16-bit key and a 16-bit cardinality.
* The offset header section is always included.
* RLE runs are serialized as [start, last], not [start, length].
* After the container storage section is an operation log, of unspecified length.
![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png)
*Pilosa Roaring storage format diagram*
All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.
Check out this [blog post](/blog/adding-rle-support/) for some more details about Roaring in Pilosa.

18
docs/client-libraries.md
View file

@ -1,18 +0,0 @@
+++
title = "Client Libraries"
weight = 12
nav = [
"Go",
"Python",
"Java",
]
+++
## Client Libraries
We have the following official client libraries. You can find more information in their repositories:
* [Go client repository](https://github.com/pilosa/go-pilosa)
* [Java client repository](https://github.com/pilosa/java-pilosa)
* [Python client repository](https://github.com/pilosa/python-pilosa)
Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients.

648
docs/configuration.md
View file

@ -1,648 +0,0 @@
+++
title = "Configuration"
weight = 7
nav = [
"Command line flags",
"Environment variables",
"Config file",
"All Options",
]
+++
## Configuration
Pilosa can be configured through command line flags, environment variables, and/or a configuration file; configured options take precedence in that order. So if an option is specified in a command line flag, it will take precedence over the same option specified in the environment, which will take precedence over that same option specified in the configuration file.
All options are available in all three configuration types with the exception of the `--config` option which specifies the location of the config file, and therefore will not be used if it is present in the config file.
The syntax for each option is slightly different between each of the configuration types, but follows a simple formula. See the following three sections for an explanation of each configuration type.
### Command line flags
Pilosa uses GNU/POSIX style flags. Most flags you specify as `--flagname=value` although some have a short form that is a single character and can be specified with a single dash like `-f value`. Running `pilosa server --help` will give an overview of the available flags as well as their short forms (if applicable).
### Environment variables
Every command line flag has a corresponding environment variable. The environment variable is the flag name in all caps, prefixed by `PILOSA_`, and with dots and dashes replaced by underscores. For example: `--scope.flag-name` becomes `PILOSA_SCOPE_FLAG_NAME`.
### Config file
The config file is in the [toml format](https://github.com/toml-lang/toml) and has exactly the same options available as the flags and environment variables. Any flag which contains a dot (".") denotes nesting within the config file, so the two flags `--cluster.coordinator` and `--cluster.replicas=1` look like this in the config file:
```toml
[cluster]
coordinator = true
replicas = 1
```
### All Options
#### Advertise
* Description: Address advertised by the server to other nodes in the cluster and to clients via the `/status` endpoint. Host defaults to the IP address represented by `bind` and port to 10101. If `bind` is set to `0.0.0.0` and `advertise` is not specified, then Pilosa will try to determine a reasonable, external IP address to use for `advertise`.
* Flag: `--advertise="192.168.1.100:10101"`
* Env: `PILOSA_BIND="192.168.1.100:10101"`
* Config:
```toml
advertise = 192.168.1.100:10101
```
#### Anti Entropy Interval
* Description: Interval at which the cluster will run its anti-entropy routine which ensures that all replicas of each fragment are in sync.
* Flag: `--anti-entropy.interval="10m0s"`
* Env: `PILOSA_ANTI_ENTROPY_INTERVAL="10m0s"`
* Config:
```toml
[anti-entropy]
interval = "10m0s"
```
#### Bind
* Description: host:port on which the Pilosa server will listen for requests. Host defaults to localhost and port to 10101. If `bind` is set to `0.0.0.0` then Pilosa will listen on all available interfaces.
* Flag: `--bind="localhost:10101"`
* Env: `PILOSA_BIND="localhost:10101"`
* Config:
```toml
bind = localhost:10101
```
#### CORS (Cross-Origin Resource Sharing) Allowed Origins
* Description: List of allowed origin URIs for CORS
* Flag: `--handler.allowed-origins="https://myapp.com,https://myapp.org"`
* Env: `PILOSA_HANDLER_ALLOWED_ORIGINS="https://myapp.com,https://myapp.org"`
* Config:
```toml
[handler]
allowed-origins = ["https://myapp.com", "https://myapp.org"]
```
#### Data Dir
* Description: Directory to store Pilosa data files.
* Flag: `--data-dir="~/.pilosa"`
* Env: `PILOSA_DATA_DIR="~/.pilosa"`
* Config:
```toml
data-dir = "~/.pilosa"
```
#### Log Path
* Description: Path of log file.
* Flag: `--log-path="/path/to/logfile"`
* Env: `PILOSA_LOG_PATH="/path/to/logfile"`
* Config:
```toml
log-path = "/path/to/logfile"
```
#### Verbose
* Description: Enable verbose logging.
* Flag: `--verbose`
* Env: `PILOSA_VERBOSE`
* Config:
```toml
verbose = true
```
#### Long Query Time
* Description: Duration that will trigger log and stat messages for slow queries.
* Flag: `long-query-time="1m0s"`
* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
* Config:
```toml
long-query-time = "1m0s"
```
#### Max Map Count
* Description: Maximum number of active memory maps Pilosa will use for fragment
files (actual total usage may be slightly higher). Best practice is to set
this ~10% lower than your system's maximum map count (obtained via `sysctl
vm.max_map_count` on Linux). If you plan on having lots of fragments per host,
it's a good idea to raise both the system's max map count, and Pilosa's. The
number of fragments is a function of the number of shards, fields, and time
quantums. Using, for example, YMDH time quantum fields with a wide range of
timestamps will create lots of fragments. When Pilosa exhausts the
max-map-count it falls back to reading files directly into memory. This can be
a bit slower, and cause slower restarts, but is generally fine.
* Flag: `--max-map-count=1000000`
* Env: `PILOSA_MAX_MAP_COUNT=1000000`
* Config:
```toml
max-map-count = 1000000
```
#### Max Writes Per Request
* Description: Maximum number of mutating commands allowed per request. This includes Set, Clear, SetRowAttrs, and SetColumnAttrs.
* Flag: `--max-writes-per-request=5000`
* Env: `PILOSA_MAX_WRITES_PER_REQUEST=5000`
* Config:
```toml
max-writes-per-request = 5000
```
#### Max File Count
* Description: A soft limit on the maximum number of files that Pilosa will keep
open simultaneously. When past this limit, Pilosa will only keep files open
for as long as it needs to write updates. This will negatively affect
performance in cases where Pilosa is doing lots of small updates.
* Flag: `--max-file-count=1000000`
* Env: `PILOSA_MAX_FILE_COUNT=1000000`
* Config:
```toml
max-file-count = 1000000
```
#### Gossip Advertise Host
* Description: Host on which memberlist should advertise. Defaults to `advertise` host.
* Flag: `--gossip.advertise-host=192.168.1.100`
* Env: `PILOSA_GOSSIP_ADVERTISE_HOST=192.168.1.100
* Config:
```toml
[gossip]
advertise-host = 192.168.1.100
```
#### Gossip Advertise Port
* Description: Port on which memberlist should advertise. Defaults to `advertise` port.
* Flag: `--gossip.advertise-port=15001`
* Env: `PILOSA_GOSSIP_ADVERTISE_PORT=15001`
* Config:
```toml
[gossip]
advertise-port = 15001
```
#### Gossip Port
* Description: Port to which Pilosa should bind for internal communication. If more than one Pilosa server is running on the same host, the gossip port for each server must be unique.
* Flag: `--gossip.port=11101`
* Env: `PILOSA_GOSSIP_PORT=11101`
* Config:
```toml
[gossip]
port = 11101
```
#### Gossip Seeds
* Description: This specifies which internal host(s) should be used to initialize membership in the cluster. Typically this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip.seeds` for all three nodes can be configured to be the address of `node0`. Multiple seeds should be comma-separated in the flag and env forms.
* Flag: `--gossip.seeds="localhost:11101,localhost:11110"`
* Env: `PILOSA_GOSSIP_SEEDS="localhost:11101,localhost:11110"`
* Config:
```toml
[gossip]
seeds = ["localhost:11101", "localhost:11110"]
```
#### Gossip Key
* Description: Path to the file which contains the key to encrypt gossip communication. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 encryption. You can read from `/dev/random` device on UNIX-like systems to create the key file; e.g., `head -c 32 /dev/random > gossip.key32` creates a key file to use AES-256.
* Flag: `--gossip.key="/var/secret/gossip.key32"`
* Env: `PILOSA_GOSSIP_KEY="/var/secret/gossip.key32"`
* Config:
```toml
[gossip]
key = "/var/secret/gossip.key32"
```
#### Cluster Long Query Time
* Description (DEPRICATED, see Long Query Time): Duration that will trigger log and stat messages for slow queries.
* Flag: `cluster.long-query-time="1m0s"`
* Env: `PILOSA_CLUSTER_LONG_QUERY_TIME="1m0s"`
* Config:
```toml
[cluster]
long-query-time = "1m0s"
```
#### Cluster Coordinator
* Description: Indicates whether the node should act as the coordinator for the cluster. Only one node per cluster should be the coordinator.
* Flag: `cluster.coordinator`
* Env: `PILOSA_CLUSTER_COORDINATOR`
* Config:
```toml
[cluster]
coordinator = true
```
#### Cluster Replicas
* Description: Number of hosts each piece of data should be stored on.
* Flag: `cluster.replicas=1`
* Env: `PILOSA_CLUSTER_REPLICAS=1`
* Config:
```toml
[cluster]
replicas = 1
```
#### Cluster Type
* Description: Determine how the cluster handles membership and state sharing. Choose from [static, gossip].
* static - Messaging between nodes is disabled. This is primarily used for testing.
* gossip - Messages are transmitted over TCP. Cluster status and node state are kept in sync via internode gossip.
* Flag: `cluster.type="gossip"`
* Env: `PILOSA_CLUSTER_TYPE="gossip"`
* Config:
```toml
[cluster]
type = "gossip"
```
#### Profile CPU
* Description: If this is set to a path, collect a cpu profile and store it there.
* Flag: `--profile.cpu="/path/to/somewhere"`
* Env: `PILOSA_PROFILE_CPU="/path/to/somewhere"`
* Config:
```toml
[profile]
cpu = "/path/to/somewhere"
```
#### Profile CPU Time
* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set.
* Flag: `--profile.cpu-time="30s"`
* Env: `PILOSA_PROFILE_CPU_TIME="30s"`
* Config:
```toml
[profile]
cpu-time = "30s"
```
#### Metric Service
* Description: Which stats service to use for collecting [metrics](../administration/#metrics). Choose from [statsd, expvar, prometheus, none].
* Flag: `--metric.service=statsd`
* Env: `PILOSA_METRIC_SERVICE=statsd`
* Config:
```toml
[metric]
service = "statsd"
```
#### Metric Host
* Description: Address of the StatsD service host.
* Flag: `--metric.host=localhost:8125`
* Env: `PILOSA_METRIC_HOST=localhost:8125`
* Config:
```toml
[metric]
host = "localhost:8125"
```
#### Metric Poll Interval
* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected.
* Flag: `metric.poll-interval="0m15s"`
* Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s`
* Config:
```toml
[metric]
poll-interval = "0m15s"
```
#### Metric Diagnostics
* Description: Enable [reporting](../administration/#diagnostics) of limited usage statistics to Pilosa developers. To disable, set to false.
* Flag: `metric.diagnostics`
* Env: `PILOSA_METRIC_DIAGNOSTICS`
* Config:
```toml
[metric]
diagnostics = true
```
#### TLS Certificate
* Description: Path to the TLS certificate to use for serving HTTPS. Usually has one of `.crt` or `.pem` extensions.
* Flag: `tls.certificate=/srv/pilosa/certs/server.crt`
* Env: `PILOSA_TLS_CERTIFICATE=/srv/pilosa/certs/server.crt`
* Config:
```toml
[tls]
certificate = "/srv/pilosa/certs/server.crt"
```
#### TLS Certificate Key
* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has the `.key` extension.
* Flag: `tls.key=/srv/pilosa/certs/server.key`
* Env: `PILOSA_TLS_KEY=/srv/pilosa/certs/server.key`
* Config:
```toml
[tls]
key = "/srv/pilosa/certs/server.key"
```
#### TLS CA Certificate
* Description: Path to the TLS certificate key to use for serving HTTPS. Usually has one of `.crt` or `.pem` extensions.
* Flag: `tls.ca-certificate=/srv/pilosa/certs/ca-chain.pem`
* Env: `PILOSA_TLS_CA_CERTIFICATE=/srv/pilosa/certs/ca-chain.pem`
* Config:
```toml
[tls]
ca-certificate = "/srv/pilosa/certs/ca-chain.pem"
```
#### TLS Skip Verify
* Description: Disables verification for checking TLS certificates. This configuration item is mainly useful for using self-signed certificates for a Pilosa cluster. Do not use in production since it makes man-in-the-middle attacks trivial.
* Flag: `tls.skip-verify`
* Env: `PILOSA_TLS_SKIP_VERIFY`
* Config:
```toml
[tls]
skip-verify = true
```
#### TLS Enable Client Certificate Verification
* Description: Enables verification of client certificates on incoming HTTPS requests for mutual TLS authentication.
* Flag: `tls.enable-client-verification`
* Env: `PILOSA_TLS_ENABLE_CLIENT_VERIFICATION`
* Config:
```toml
[tls]
enable-client-verification = true
```
#### Tracing Sampler Type
* Description: Jaeger sampler type (const, probabilistic, ratelimiting, or remote). Set to 'off' to disable tracing completely. Default is 'off'.
* Flag: `tracing.sampler-type`
* Env: `PILOSA_TRACING_SAMPLER_TYPE`
* Config:
```toml
[tracing]
sampler-type = "remote"
```
#### Tracing Sampler Parameter
* Description: Jaeger sampler parameter (number)
* Flag: `tracing.sampler-param`
* Env: `PILOSA_TRACING_SAMPLER_PARAM`
* Config:
```toml
[tracing]
sampler-param = 0.001
```
#### Tracing Agent Host/Port
* Description: Jaeger agent host:port
* Flag: `tracing.agent-host-port`
* Env: `PILOSA_TRACING_AGENT_HOST_PORT`
* Config:
```toml
[tracing]
agent-host-port = "localhost:6831"
```
#### Profile Block Rate
* Description: Block Rate is passed directly to Go's
[runtime.SetBlockProfileRate](https://golang.org/pkg/runtime/#SetBlockProfileRate). Goroutine blocking events will be sampled at 1
per `rate` nanoseconds. A value of "1" samples every event, and 0 disables
profiling.
* Flag: `--profile.block-rate=10000000`
* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000`
* Config:
```toml
[profile]
block-rate = 10000000
```
#### Profile Mutex Fraction
* Description: Mutex Fraction is passed directly to Go's
[runtime.SetMutexProfileFraction](https://golang.org/pkg/runtime/#SetMutexProfileFraction). 1/`fraction` of events will be sampled.
* Flag: `--profile.mutex-fraction=100`
* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100`
* Config:
```toml
[profile]
mutex-fraction = 100
```
#### Translation Map Size
* Description: Size in bytes of mmap to allocate for key translation
* Flag: `translation.map-size`
* Env: `PILOSA_TRANSLATION_MAP_SIZE`
* Config:
```toml
[translation]
map-size = 10737418240
```
### Example Cluster Configuration
A three node cluster running on different hosts could be minimally configured as follows:
#### Node 0
data-dir = "/home/pilosa/data"
bind = "node0.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
[cluster]
replicas = 1
coordinator = true
#### Node 1
data-dir = "/home/pilosa/data"
bind = "node1.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
[cluster]
replicas = 1
coordinator = false
#### Node 2
data-dir = "/home/pilosa/data"
bind = "node2.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
[cluster]
replicas = 1
coordinator = false
### Example Cluster Configuration (HTTPS)
The same cluster which uses HTTPS instead of HTTP can be configured as follows. Note that we explicitly specify `https` as the protocol in `bind` and `cluster.hosts` configuration. It is not required to use a gossip key but it is highly recommended:
#### Node 0
data-dir = "/home/pilosa/data"
bind = "https://node0.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"
#### Node 1
data-dir = "/home/pilosa/data"
bind = "https://node1.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"
#### Node 2
data-dir = "/home/pilosa/data"
bind = "https://node2.pilosa.com:10101"
[gossip]
port = 12000
seeds = ["node0.pilosa.com:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"
### Example Cluster Configuration (HTTPS, same host)
You can run a cluster on the same host using the configuration above with a few changes. Gossip port and bind address should be different for each node and a data directory should be accessed only by a single node.
#### Node 0
data-dir = "/home/pilosa/data0"
bind = "https://localhost:10100"
[gossip]
port = 12000
seeds = ["localhost:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = true
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"
#### Node 1
data-dir = "/home/pilosa/data1"
bind = "https://localhost:10101"
[gossip]
port = 12001
seeds = ["localhost:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"
#### Node 2
data-dir = "/home/pilosa/data2"
bind = "https://localhost:10102"
[gossip]
port = 12002
seeds = ["localhost:12000"]
key = "/home/pilosa/private/gossip.key32"
[cluster]
replicas = 1
coordinator = false
[tls]
certificate = "/home/pilosa/private/server.crt"
key = "/home/pilosa/private/server.key"

58
docs/console.md
View file

@ -1,58 +0,0 @@
+++
title = "Console"
weight = 9
nav = [
"Installation",
"Query",
"Cluster Admin",
]
+++
## Console
A web-based app called Pilosa Console is available in a separate package. This can be used for constructing queries and viewing the cluster status.
### Installation
Releases are [available on Github](https://github.com/pilosa/console/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
Installing on a Mac with Homebrew is simple; just run:
```
brew tap pilosa/homebrew-pilosa
brew install pilosa-console
```
You may also build from source by checking out the [repo on Github](https://github.com/pilosa/console) and running:
```
make install
```
### Query
The Query tab allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
The Console will keep a record of each query and its result with the latest query on top.
![Console screenshot](/img/docs/webui-console.png)
*Console query screenshot*
In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
- `:create index <indexname>`
- `:delete index <indexname>`
- `:use <indexname>`
- `:create field <fieldname>`
- `:delete field <fieldname>`
Field creation also supports options like `timeQuantum`. When creating a new field, add options by using the keys documented in [API reference](../api-reference/#create-field).
- `:create field <fieldname> cacheSize=10000`
### Cluster Admin
Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Fields.

217
docs/data-model.md
View file

@ -1,217 +0,0 @@
+++
title = "Data Model"
weight = 5
nav = [
"Overview",
"Index",
"Column",
"Row",
"Field",
"Time Quantum",
"Attribute",
"Shard",
]
+++
## Data Model
### Overview
The central component of Pilosa's data model is a boolean matrix. Each cell in the matrix is a single bit; if the bit is set, it indicates that a relationship exists between that particular row and column.
Rows and columns can represent anything (they could even represent the same set of things as in a [bigraph](https://en.wikipedia.org/wiki/Bigraph)). Pilosa can associate arbitrary key/value pairs (referred to as attributes) to rows and columns, but queries and storage are optimized around the core matrix.
Pilosa lays out data first in rows, so queries which get all the set bits in one or many rows, or compute a combining operation—such as Intersect or Union—on multiple rows, are the fastest. Pilosa categorizes rows into different *fields* and quickly retrieves the top rows in a field sorted by the number of columns set in each row.
Please note that Pilosa is most performant when row and column IDs are sequential starting from 0. You can deviate from this to some degree, but setting a bit with column ID 2<sup>63</sup> on a single-node cluster, for example, will not work well due to memory limitations.
![basic data model diagram](/img/docs/data-model.png)
*Basic data model diagram*
### Index
The purpose of the Index is to represent a data namespace. You cannot perform cross-index queries.
### Column
Column ids are sequential, increasing integers and they are common to all Fields within an Index. A single column often corresponds to a record in a relational table, although other configurations are possible, and sometimes preferable.
### Row
Row ids are sequential, increasing integers namespaced to each Field within an Index.
### Field
Fields are used to segment rows within an index, for example to define different functional groups. A Pilosa field might correspond to a single field in a relational table, where each row in a standard Pilosa field represents a single possible value of the relational field. Similarly, an integer field could represent all possible integer values of a relational field.
#### Relational Analogy
The Pilosa index is a flexible structure; it can represent any sort of high-cardinality binary matrix. We have explored a number of modeling patterns in Pilosa use cases; one accessible example is a direct analogy to the relational model, summarized here.
Entities:
Relational | Pilosa
-------------|----------------------------------------------
Database | N/A *(internal: Holder)*
Table | Index
Row | Column
Column | Field
Value | Row
Value (int) | Field.Value (see [BSI](#bsi-range-encoding))
Simple queries:
Relational | Pilosa
-----------------------------------------------|------------------------------------
`select ID from People where Name = 'Bob'` | `Row(Name="Bob")`
`select ID from People where Age > 30` | `Row(Age > 30)`
`select ID from People where Member = true` | `Row(Member=0)`
Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join:
```sql
select AVG(p.Age) from People p
inner join PersonCar pc on pc.PersonID=p.ID
inner join Cars c on pc.CarID=c.ID
where c.Make = 'Ford'
```
can be accomplished with a Pilosa query like this (note that [Sum](../query-language/#sum) returns a json object containing both the sum and count, from which the average is easily computed):
```pql
Sum(Row(Car-Make="Ford"), field=Age)
```
This is one major component of Pilosa's ability to combine relationships from multiple data stores.
#### Ranked
Ranked Fields maintain a sorted cache of column counts by Row ID (yielding the top rows by columns with a bit set in each). This cache facilitates the TopN query. The cache size defaults to 50,000 and can be set at Field creation.
![ranked field diagram](/img/docs/field-ranked.png)
*Ranked field diagram*
#### LRU
The LRU cache maintains the most recently accessed Rows.
![lru field diagram](/img/docs/field-lru.png)
*LRU field diagram*
### Time Quantum
Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to `YMD`, ranged Row queries down to the granularity of a day are supported.
### Attribute
Attributes are arbitrary key/value pairs that can be associated with either rows or columns. This metadata is stored in a separate BoltDB data structure.
Column-level attributes are common across an index. That is, each column attribute applies to all bits in the corresponding column, across all fields in an index. Row attributes apply to all bits in the corresponding row.
### Shard
Indexes are segmented into groups of columns called shards (previously known as slices). Each shard contains a fixed number of columns, which is the ShardWidth. ShardWidth is a constant that can only be modified at compile time, and before ingesting data. The default value is 2<sup>20</sup>.
Query operations run in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.
### Field Type
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`.
#### Set
Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
```go
[]byte{
0x00, 0x00, 0x00,
0x4d, 0x54, 0x4d, 0x54, // MTMT
0x00,
0xc2, 0xa0, // NO-BREAK SPACE
0x00,
}
```
(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
``` request
curl localhost:10101/index/repository/field/info \
-X POST \
-d '{"options": {"type": "set", "cacheType": "ranked", "cacheSize":100000}}'
```
``` response
{"success":true}
```
#### Int
Fields of type `int` are used to store integer values. Integer fields share the same columns as the other fields in the index, but values for the field must be integers that fall between the `min` and `max` values specified when creating the field. The following example creates an `int` field called "quantity" capable of storing values from -1000 to 2000:
``` request
curl localhost:10101/index/repository/field/quantity \
-X POST \
-d '{"options": {"type": "int", "min": -1000, "max":2000}}'
```
``` response
{"success":true}
```
##### BSI Range-Encoding
Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Row`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead.
Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows.
For example, the following `Set()` queries executed against BSI fields will result in the data described in the diagram below:
```
Set(1, A=1)
Set(2, A=2)
Set(3, A=3)
Set(4, A=7)
Set(2, B=1)
Set(3, B=6)
```
![BSI field diagram](/img/docs/field-bsi.png)
*BSI field diagram*
Check out this [blog post](/blog/range-encoded-bitmaps/) for some more details about BSI in Pilosa.
###### BSI Deprecated Format
The original implementation of BSI required a fixed bit depth when creating fields because the existence bit was written to the bit above the highest bit. The second version of BSI moves the existence bit to the beginning, adds a negative bit as the second bit, and shifts all remaining bits up by two.
Pilosa automatically converts all old data to the new format on startup, however, this can cause issues when upgrading Pilosa and then reverting back to an old version. This documentation section exists as a record for anyone who experiences unusual behavior in BSI between versions.
#### Time
Time fields are similar to `set` fields, but in addition to row and column information, they also store a per-bit time value down to a defined granularity. The following example creates a `time` field called "event" which stores timestamp information down to a day granularity.
``` request
curl localhost:10101/index/repository/field/event \
-X POST \
-d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
```
``` response
{"success":true}
```
With `time` fields, data views are generated for each of the defined time segments. For example, for a field with a time quantum of `YMD`, the following `Set()` queries will result in the data described in the diagram below:
```
Set(3, A=8, 2017-05-18T00:00)
Set(3, A=8, 2017-05-19T00:00)
```
![time quantum field diagram](/img/docs/field-time-quantum.png)
*Time quantum field diagram*
#### Mutex
Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field.
#### Boolean
A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values.

221
docs/examples.md
View file

@ -1,221 +0,0 @@
+++
title = "Examples"
weight = 4
nav = [
"Transportation",
]
+++
## Examples
### Transportation
#### Introduction
New York City released an extremely detailed data set of over 1 billion taxi rides taken in the city - this data has become a popular target for analysis by tech bloggers and has been very well studied. For this reason, we thought it would be interesting to import this data to Pilosa in order to compare with other data stores and techniques on the exact same data set.
Transportation in general is a compelling use case for Pilosa as it often involves multiple disparate data sources, as well as high rate, real time, and extremely large amounts of data (particularly if one wants to draw reasonable conclusions).
We've written a tool to help import the NYC taxi data into Pilosa - this tool is part of the [PDK](../pdk/) (Pilosa Development Kit), and takes advantage of a number of reusable modules that may help you import other data as well. Follow along and we'll explain the whole process step by step.
After initial setup, the PDK import tool does everything we need to define a Pilosa schema, map data to bitmaps accordingly, and import it into Pilosa.
#### Data Model
The NYC taxi data is comprised of a number of csv files listed here: http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml. These data files have around 20 columns, about half of which are relevant to the benchmark queries we're looking at:
* Distance: miles, floating point
* Fare: dollars, floating point
* Number of passengers: integer
* Dropoff location: latitude and longitude, floating point
* Pickup location: latitude and longitude, floating point
* Dropoff time: timestamp
* Pickup time: timestamp
We import these fields, creating one or more Pilosa fields from each of them:
field |mapping
------------|---------------------
cab_type |direct map of enum int → row ID
dist_miles |round(dist) → row ID
total_amount_dollars |round(dist) → row ID
passenger_count |direct map of integer value → row ID
drop_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID
drop_year |year(timestamp) → row ID
drop_month |month(timestamp) → row ID
drop_day |day(timestamp) → row ID
drop_time |time of day mapped to one of 48 half-hour buckets
pickup_grid_id |(lat, lon) → 100x100 rectangular grid → cell ID
pickup_year |year(timestamp) → row ID
pickup_month |month(timestamp) → row ID
pickup_day |day(timestamp) → row ID
pickup_time |time of day mapped to one of 48 half-hour buckets → row ID
We also created two extra fields that represent the duration and average speed of each ride:
field |mapping
--------------------|-------------
duration_minutes |round(drop_timestamp - pickup_timestamp) → row ID
speed_mph |round(dist_miles / (drop_timestamp - pickup_timestamp)) → row ID
#### Mapping
Each column that we want to use must be mapped to a combination of fields and row IDs according to some rule. There are many ways to approach this mapping, and the taxi dataset gives us a good overview of possibilities.
##### 0 columns → 1 field
**cab_type**: contains one row for each type of cab. Each column, representing one ride, has a bit set in exactly one row of this field. The mapping is a simple enumeration, for example yellow=0, green=1, etc. The values of the bits in this field are determined by the source of the data. That is, we're importing data from several disparate sources: NYC yellow taxi cabs, NYC green taxi cabs, and Uber cars. For each source, the single row to be set in the cab_type field is constant.
##### 1 column → 1 field
The following three fields are mapped in a simple direct way from single columns of the original data.
**dist_miles:** each row represents rides of a certain distance. The mapping is simple: as an example, row 1 represents rides with a distance in the interval [0.5, 1.5]. That is, we round the floating point value of distance to an integer, and use that as the row ID directly. Generally, the mapping from a floating point value to a row ID could be arbitrary. The rounding mapping is concise to implement, which simplifies importing and analysis. As an added bonus, it's human-readable. We'll see this pattern used several times.
In PDK parlance, we define a Mapper, which is simply a function that returns integer row IDs. PDK has a number of predefined mappers that can be described with a few parameters. One of these is LinearFloatMapper, which applies a linear function to the input, and casts it to an integer, so the rounding is handled implicitly. In code:
```go
lfm := pdk.LinearFloatMapper{
Min: -0.5,
Max: 3600.5,
Res: 3601,
}
```
`Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a "round to nearest integer" behavior. Other predefined mappers have their own specific parameters, usually two or three.
This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). <!-- TODO update so this makes sense -->
```go
pdk.ColumnMapper{
Field: "dist_miles",
Mapper: lfm,
Parsers: []pdk.Parser{pdk.FloatParser{}},
Fields: []int{fields["trip_distance"]},
},
```
These same objects are represented in the JSON definition file:
```go
{
"Fields": {
"Trip_distance": 10
},
"Mappers": [
{
"Name": "lfm0",
"Min": -0.5,
"Max": 3600.5,
"Res": 3600
}
],
"ColumnMappers": [
{
"Field": "dist_miles",
"Mapper": {
"Name": "lfm0"
},
"Parsers": [
{"Name": "FloatParser"}
],
"Fields": "Trip_distance"
}
]
}
```
Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of ColumnMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the ColumnMapper definitions to keep things human-readable.
**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The ColumnMapper definition is very similar to the previous one.
**passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID.
##### 1 column → multiple fields
When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis.
We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day".
We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day".
We do all of this for each timestamp of interest, one for pickup time and one for dropoff time. That gives us eight total fields for two timestamps: pickup_year, pickup_month, pickup_day, pickup_time, drop_year, drop_month, drop_day, drop_time.
##### Multiple columns → 1 field
The ride data also contains geolocation data: latitude and longitude for both pickup and dropoff. We just want to be able to produce a rough overview heatmap of ride locations, so we use a grid mapping. We divide the area of interest into a 100x100 grid in latitude-longitude space, label each cell in this grid with a single integer, and use that integer as the row ID.
We do all of this for each location of interest, one for pickup and one for dropoff. That gives us two fields for two locations: pickup_grid_id, drop_grid_id.
Again, there are many mapping options for location data. For example, we might convert to a different coordinate system, apply a projection, or aggregate locations into real-world regions such as neighborhoods. Here, the simple approach is sufficient.
##### Complex mappings
We also anticipate looking for trends in ride duration and speed, so we want to capture this information during the import process. For the field `duration_minutes`, we compute a row ID as `round((drop_timestamp - pickup_timestamp).minutes)`. For the field `speed_mph`, we compute row ID as `round(dist_miles / (drop_timestamp - pickup_timestamp).minutes)`. These mapping calculations are straightforward, but because they require arithmetic operations on multiple columns, they are a bit too complex to capture in the basic mappers available in PDK. Instead, we define custom mappers to do the work:
```go
durm := pdk.CustomMapper{
Func: func(fields ...interface{}) interface{} {
start := fields[0].(time.Time)
end := fields[1].(time.Time)
return end.Sub(start).Minutes()
},
Mapper: lfm,
}
```
#### Import process
After designing this schema and mapping, we capture it in a JSON definition file that can be read by the PDK import tool. Running `pdk taxi` runs the import based on the information in this file. For more details, see the [PDK](../pdk/) section, or check out the [code](https://github.com/pilosa/pdk/tree/master/usecase/taxi) itself.
#### Queries
Now we can run some example queries.
Count per cab type can be retrieved, sorted, with a single PQL call.
```request
TopN(cab_type)
```
```response
{"results":[[{"id":1,"count":1992943},{"id":0,"count":7057}]]}
```
High traffic location IDs can be retrieved with a similar call. These IDs correspond to latitude, longitude pairs, which can be recovered from the mapping that generates the IDs.
```request
TopN(pickup_grid_id)
```
```response
{"results":[[{"id":5060,"count":40620},{"id":4861,"count":38145},{"id":4962,"count":35268},...]]}
```
Average of `total_amount` per `passenger_count` can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average.
```python
import pilosa
client = pilosa.Client()
schema = client.schema()
taxi = schema.index("taxi")
passenger_count = taxi.field("passenger_count")
total_amount_dollars = taxi.field("total_amount_dollars")
queries = []
pcounts = range(10)
for i in pcounts:
queries.append(total_amount_dollars.topn(passenger_count.row(i))
query = taxi.batch_query(**queries)
results = client.query(query)
resp = requests.post(qurl, data=queries)
average_amounts = []
for pcount, result in zip(pcounts, resp.results):
wsum = sum([r.count * r.id for r in result.count_items])
count = sum([r.count for r in result.count_items])
average_amounts.append(float(wsum)/count)
```
<div class="note">
Note that the <a href="../data-model/#bsi-range-encoding">BSI</a>-powered <a href="../query-language/#sum">Sum</a> query now provides an alternative approach to this kind of query.
</div>
<!-- Disabled until we have the time to update the Jupyter notebook --YT
For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
-->

43
docs/faq.md
View file

@ -1,43 +0,0 @@
+++
title = "FAQ"
weight = 15
nav = []
+++
## FAQ
### What is Pilosa?
Pilosa is an in-memory, distributed index that is layered over persistent storage. It supports fast ad-hoc queries and segmentation. Pilosa does not require the underlying data to be moved, rather it can be populated in conjunction with data writes, or it can be backfilled asynchronously from any other data store or event processing system. This allows Pilosa to support sub-second queries against very large underlying data sets.
### Is Pilosa a database?
Pilosa is not a database in the traditional sense. While Pilosa does store data (both in-memory as well as persisted to disk), it wouldn't typically be used as a primary data store. Instead, one would likely use Pilosa as an index of the data stored in a traditional database or in a data warehouse.
### Where does Pilosa fit in my stack?
Pilosa was designed to index the relationships in your data. Pilosa runs along with your existing stack, integrating with one or more backing data stores. Pilosa can connect through a stream platform like Kafka or application integration via [PDK](../pdk/).
### How is Pilosa different from Elasticsearch since they are both indexes?
Elasticsearch is a search engine based on Lucene, and is therefore very good at indexing and searching large volumes of unstructured text. As it matures, Elasticsearch has continued to move into the analytics space, but its core data object is still the "document". Pilosa is specifically designed to index structured data and improve query speed. By representing data as the relationship between objects, and then storing those relationships in bitmaps, Pilosa can very efficiently search and compare many millions of data points while still maintaining a small memory footprint.
### How do I get my data into Pilosa?
There are typically two methods for getting data into Pilosa: importing large batches of data from an existing data set, and continuously updating Pilosa as data is added or updated.
In the first case, one would use the `pilosa import` command to bulk load structured data into Pilosa. In order to improve this process, one can use the Pilosa Development Kit (PDK) to map structured data in the original data set onto the Pilosa schema.
For the case where data is continually mutating, one would apply a parallel data writer at the point at which data is written to the persistent data store. This new writer would simultaneously write to Pilosa. An example use case would be one where Kafka was employed as the message broker in your data pipeline, you could introduce an additional Kafka consumer to read from the message log and write mutated data to Pilosa.
### What languages can I use with it?
There is currently [client support](../client-libraries/) for [Go](https://github.com/pilosa/go-pilosa), [Python](https://github.com/pilosa/python-pilosa), and [Java](https://github.com/pilosa/java-pilosa). If you want to use Pilosa with a different language, you can access Pilosa via the [Pilosa API](../api-reference/).
### Do you query Pilosa using SQL?
One can access Pilosa directly via the terminal using the [Pilosa Query Language](../query-language/) (PQL), but a typical implementation would use one of the Pilosa client libraries to integrate with an existing codebase. There is currently client support for Go, Python, and Java.
### Replication on each node?
Pilosa supports a replication factor greater than or equal to one. When replication is configured to be greater than one, then all mutations will be replicated to additional nodes in the cluster. For example, in a five-node cluster consisting of nodes A-B-C-D-E and with replication factor of three, then a write to node B will result in data being written to nodes B, C, and D. If the replication factor is greater than the number of nodes in the cluster, the data will be replicated to every node in the cluster only once.

970
docs/getting-started.md
View file

@ -1,970 +0,0 @@
+++
title = "Getting Started"
weight = 3
nav = [
"Starting Pilosa",
"Sample Project",
"Using Curl",
"Using Go",
"Using Java",
"Using Python",
"What's Next?",
]
+++
## Getting Started
Pilosa supports an HTTP interface which uses JSON by default.
Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. However, the best way to interface with the Pilosa server is through one of our three official client libraries. Pilosa currently supports [Go](https://github.com/pilosa/go-pilosa), [Java](https://github.com/pilosa/java-pilosa), and [Python](https://github.com/pilosa/python-pilosa).
<div class="note">
<p>Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. See <a href="/docs/administration/#open-file-limits">Open File Limits</a> for more details.</p>
</div>
### Starting Pilosa
Follow the steps in the [Installation](../installation/) document to install Pilosa.
Execute the following in a terminal to run Pilosa with the default configuration (Pilosa will be available at [localhost:10101](http://localhost:10101)):
```
pilosa server
```
Let's make sure Pilosa is running:
``` request
curl localhost:10101/status
```
``` response
{"state":"NORMAL","nodes":[{"id":"91715a50-7d50-4c54-9a03-873801da1cd1","uri":{"scheme":"http","host":"localhost","port
":10101},"isCoordinator":true}],"localID":"91715a50-7d50-4c54-9a03-873801da1cd1"}
```
### Sample Project
In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about 1,000 popular Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language and stargazers—people who have starred a project.
Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and stargazers. We can better organize the rows by grouping them into sets called Fields. So the "repository" index might have a "languages" field as well as a "stargazers" field. You can learn more about indexes and fields in the [Data Model](../data-model/) section of the documentation.
<div class="note">
<p>If at any time you want to verify the data structure, you can request the schema as follows:</p>
</div>
```request
curl localhost:10101/schema
```
```response
{
"indexes": [
{
"name": "repository",
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "language",
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "stargazer",
"options": {
"type": "time",
"timeQuantum": "YMDH",
"keys": false,
"noStandardView": false
}
}
],
"shardWidth": 1048576
}
]
}
```
<div class="note">
<p>Note: This is the response you should receive once completing this project. It has also been formatted using <a href="https://stedolan.github.io/jq/"><code>jq</code></a>. </p>
</div>
#### Using Curl
##### Create the Schema
Before we can import data or run queries, we need to create our indexes and the fields within them. Let's create the `repository` index first:
``` request
curl localhost:10101/index/repository -X POST
```
``` response
{"success":true}
```
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
``` request
curl localhost:10101/index/repository/field/stargazer \
-X POST \
-d '{"options": {"type": "time", "timeQuantum": "YMD"}}'
```
``` response
{"success":true}
```
Since our data contains time stamps which represent the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`.
Next up is the `language` field, which will contain IDs for programming languages:
``` request
curl localhost:10101/index/repository/field/language \
-X POST
```
``` response
{"success":true}
```
The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options.
##### Import Data From CSV Files
Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
```
Run the following commands to import the data into Pilosa:
```
pilosa import -i repository -f stargazer stargazer.csv
pilosa import -i repository -f language language.csv
```
If you are using a Docker container for Pilosa (with name `pilosa`), you should instead copy the `*.csv` file into the container and then import them:
```
docker cp stargazer.csv pilosa:/stargazer.csv
docker exec -it pilosa /pilosa import -i repository -f stargazer /stargazer.csv
docker cp language.csv pilosa:/language.csv
docker exec -it pilosa /pilosa import -i repository -f language /language.csv
```
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
##### Make Some Queries
Which repositories did user 14 star:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Row(stargazer=14)'
```
``` response
{
"results":[
{
"attrs":{},
"columns":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514]
}
]
}
```
What are the top 5 languages in the sample data:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'TopN(language, n=5)'
```
``` response
{
"results":[
[
{"id":5,"count":119},
{"id":1,"count":50},
{"id":4,"count":48},
{"id":9,"count":31},
{"id":13,"count":25}
]
]
}
```
Which repositories were starred by user 14 and 19:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Row(stargazer=14),
Row(stargazer=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"columns":[2,3,362,396,416,461,464,466,470,486]
}
]
}
```
Which repositories were starred by user 14 or 19:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Union(
Row(stargazer=14),
Row(stargazer=19)
)'
```
``` response
{
"results":[
{
"attrs":{},
"columns":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514]
}
]
}
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Intersect(
Row(stargazer=14),
Row(stargazer=19),
Row(language=1)
)'
```
``` response
{
"results":[
{
"attrs":{},
"columns":[2,362,416,461]
}
]
}
```
Set user 99999 as a stargazer for repository 77777:
``` request
curl localhost:10101/index/repository/query \
-X POST \
-d 'Set(77777, stargazer=99999)'
```
``` response
{"results":[true]}
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
#### Using Go
Pilosa follows the Go policy of supporting the two most recent major versions of Go.
##### Create the Environment
Interacting with Pilosa in your go program is best accomplished using our client, go-pilosa. To install go-pilosa, open a new terminal and download the library to your `GOPATH` using:
```
go get github.com/pilosa/go-pilosa
```
Create a project folder:
```
mkdir getting-started && cd getting-started
```
In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
```
We will also create a file called `startrace.go` as follows:
```
touch startrace.go
```
This file will be used in the following sections.
##### Create the Schema
Before we can import data or run queries, we need to create our schema. You can see two imports from the go-pilosa repo, go-pilosa for the client, and csv for the CSV reader. Create the schema by creating a client (which will communicate our schema to Pilosa), creating a schema locally (which will contain our indexes and fields), and syncing with Pilosa. This is all done in the `startrace.go` file:
```
package main
import (
"bytes"
"fmt"
"github.com/pilosa/go-pilosa"
"github.com/pilosa/go-pilosa/csv"
"io/ioutil"
"log"
)
func main() {
// Create the Schema
client := pilosa.DefaultClient()
schema, _ := client.Schema()
// This is where the index will go later
// This is where the fields will go later
err := client.SyncSchema(schema)
if err != nil {
log.Fatal(err)
}
}
```
Next, let's create the `repository` index:
```
repository := schema.Index("repository")
```
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```
stargazer := repository.Field("stargazer")
```
Next up is the `language` field, which will contain IDs for programming languages:
```
language := repository.Field("language")
```
Your `startrace.go` file should look like:
```
package main
import (
"bytes"
"fmt"
"github.com/pilosa/go-pilosa"
"github.com/pilosa/go-pilosa/csv"
"io/ioutil"
"log"
)
func main() {
// Create the Schema
client := pilosa.DefaultClient()
schema, _ := client.Schema()
repository := schema.Index("repository")
stargazer := repository.Field("stargazer")
language := repository.Field("language")
err := client.SyncSchema(schema)
if err != nil {
log.Fatal(err)
}
}
```
##### Import Data From CSV Files
Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
First, we will load our data into the `stargazer` field:
```
stargazerFile, err := ioutil.ReadFile("stargazer.csv")
if err != nil {
log.Fatal(err)
}
format := "2006-01-02T15:04"
iterator = csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, bytes.NewReader(stargazerFile), format)
err = client.ImportField(stargazer, iterator)
if err != nil {
log.Fatal(err)
}
```
Since our `stargazer` data contains time stamps, which represent the time users starred repos, we will be using the `csv.NewColumnIteratorWithTimeStampFormat` function from the go-pilosa/csv package. This function takes the format of the csv files (`csv.RowIDColumnID`), an `io.Reader` (`bytes.NewReader(stargazerFile)`), and the time quantum format (`format`) and translates the csv file into a format Pilosa can read. Time quantum is the resolution of the time we want to use.
Next, we will load our data into the `language` field:
```
languageFile, err := ioutil.ReadFile("language.csv")
if err != nil {
log.Fatal(err)
}
iterator := csv.NewColumnIterator(csv.RowIDColumnID, bytes.NewReader(languageFile))
err = client.ImportField(language, iterator)
if err != nil {
log.Fatal(err)
}
```
Since our `language` data doesn't contain time stamps, we will use the `csv.NewColumnIterator` function in place of `csv.NewColumnIteratorWithTimeStampFormat`.
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
For more information on imports in go-pilosa, please see the go-pilosa [site](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md).
##### Make Some Queries
Now that we have a working schema, we can query it.
Which repositories did user 14 star:
``` request
response, err := client.Query(stargazer.Row(14))
if err != nil {
log.Fatal(err)
}
fmt.Println("User 14 starred: ", response.Result().Row().Columns)
```
``` response
User 14 starred: [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514]
```
What are the top 5 languages in the sample data:
``` request
response, err = client.Query(language.TopN(5))
if err != nil {
log.Fatal(err)
}
fmt.Println("Top Languages: ", response.Result().CountItems())
```
``` response
Top Languages: [{5 119} {1 50} {4 48} {9 31} {13 25}]
```
Which repositories were starred by user 14 and 19:
``` request
response, err = client.Query(repository.Intersect(stargazer.Row(14), stargazer.Row(19)))
if err != nil {
log.Fatal(err)
}
fmt.Println("Both user 14 and 19 starred: ", response.Result().Row().Columns)
```
``` response
Both user 14 and 19 starred: [2 3 362 396 416 461 464 466 470 486]
```
Which repositories were starred by user 14 or 19:
``` request
response, err = client.Query(repository.Union(stargazer.Row(14), stargazer.Row(19)))
if err != nil {
log.Fatal(err)
}
fmt.Println("User 14 or 19 starred: ", response.Result().Row().Columns)
```
``` response
User 14 or 19 starred: [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514]
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
``` request
response, err = client.Query(repository.Intersect(stargazer.Row(14), stargazer.Row(19), language.Row(1)))
if err != nil {
log.Fatal(err)
}
fmt.Println("Both user 14 and 19 starred and were written in language 1: ", response.Result().Row().Columns)
```
``` response
Both user 14 and 19 starred and were written in language 1: [2 362 416 461]
```
Set user 99999 as a stargazer for repository 77777:
``` request
client.Query(stargazer.Set(99999, 77777))
response, err = client.Query(stargazer.Row(99999))
if err != nil {
log.Fatal(err)
}
fmt.Println("Set user 99999 as a stargazer for repository 77777")
```
``` response
Set user 99999 as a stargazer for repository 77777
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
For more information about go-pilosa, please see our Go client library at [go-pilosa](https://github.com/pilosa/go-pilosa) or checkout the go-pilosa [Data Model and Queries](https://github.com/pilosa/go-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
#### Using Java
Pilosa requires Java 8 or higher and Maven 3 or higher.
##### Create the Environment
Create a project folder:
```
mkdir getting-started && cd getting-started
```
In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
```
We will now create the java directory that will contain our `pom.xml` file and create the `pom.xml` file:
```
mkdir startrace && cd startrace
touch pom.xml
```
For this specific project, the `pom.xml` file needs to contain:
```
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.pilosa</groupId>
<artifactId>getting-started</artifactId>
<version>1.0.0</version>
<dependencies>
<dependency>
<groupId>com.pilosa</groupId>
<artifactId>pilosa-client</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!-- Build an executable JAR -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>main.java.StarTrace</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<!-- create an uber JAR -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
```
We will now create the java directory that will contain our `StarTrace.java` file and create the `StarTrace.java` file:
```
mkdir -p src/main/java && cd src/main/java
touch StarTrace.java
```
This file will be used in the following sections.
##### Create the Schema
Before we can import data or run queries, we need to create our schema. You can see the first six dependencies are imported from the java-pilosa library. Create the schema by creating a client which will communicate our schema to Pilosa, creating a schema which will contain our indexes and fields, and syncing with Pilosa. This is all done in the `StarTrace.java` file:
```
package main.java;
import com.pilosa.client.PilosaClient;
import com.pilosa.client.QueryResponse;
import com.pilosa.client.exceptions.PilosaException;
import com.pilosa.client.orm.*;
import com.pilosa.client.csv.FileRecordIterator;
import com.pilosa.client.TimeQuantum;
import java.io.IOException;
import java.text.SimpleDateFormat;
public class StarTrace {
public static void main(String []args) throws IOException {
// Create the Schema
PilosaClient client = PilosaClient.defaultClient();
Schema schema = client.readSchema();
// This is were the index will go later
// This is were the fields will go later
client.syncSchema(schema);
}
}
```
Next, let's create the `repository` index:
```
Index repository = schema.index("repository");
```
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```
FieldOptions stargazerOptions = FieldOptions.builder()
.fieldTime(TimeQuantum.YEAR_MONTH_DAY)
.build();
Field stargazer = repository.field("stargazer", stargazerOptions);
```
Since our data contains time stamps which represent the time users starred repos, we set the field type to `time` using `fieldTime()`. Time quantum is the resolution of the time we want to use, and we set it to `YEAR_MONTH_DAY` for `stargazer`.
Next up is the `language` field, which will contain IDs for programming languages:
```
Field language = repository.field("language");
```
The `language` field is a `set` field, but since the default field type is `set`, we don't need to specify it
Your `StarTrace.java` file should look like:
```
package main.java;
import com.pilosa.client.PilosaClient;
import com.pilosa.client.QueryResponse;
import com.pilosa.client.exceptions.PilosaException;
import com.pilosa.client.orm.*;
import com.pilosa.client.csv.FileRecordIterator;
import com.pilosa.client.TimeQuantum;
import java.io.IOException;
import java.text.SimpleDateFormat;
public class StarTrace {
public static void main(String []args) throws IOException {
// Create the Schema
PilosaClient client = PilosaClient.defaultClient();
Schema schema = client.readSchema();
Index repository = schema.index("repository");
FieldOptions stargazerOptions = FieldOptions.builder()
.fieldTime(TimeQuantum.YEAR_MONTH_DAY)
.build();
Field stargazer = repository.field("stargazer", stargazerOptions);
Field language = repository.field("language");
client.syncSchema(schema);
}
}
```
##### Import Data From CSV Files
Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
First, we will load our data into the `stargazer` field:
```
SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm");
FileRecordIterator iterator = FileRecordIterator.fromPath("stargazer.csv", stargazer, timestampFormat);
client.importField(stargazer, iterator);
```
Due to the time aspect of the `stargazer` csv file, we have to specify the time stamp format in the `fromPath` function. We set the variable `timestampFormat` to the format present in the csv file using the function `SimpleDateFormat()` and pass the variable to the `fromPath` function, which will take the csv file name, the field name, and the time stamp format and translate the csv file into a format Pilosa can read.
Next, we will load our data into the `language` field:
```
iterator = FileRecordIterator.fromPath("language.csv", language);
client.importField(language, iterator);
```
Since our `language` data doesn't have a time aspect, the time stamp format doesn't need to be specified.
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
For more information on imports in java-pilosa, please see the java-pilosa [site](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md).
##### Make Some Queries
Now that we have a working schema, we can query it.
Which repositories did user 14 star:
``` request
QueryResponse response = client.query(stargazer.row(14));
System.out.println("User 14 starred: " + response.getResult().getRow().getColumns());
```
``` response
User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
```
What are the top 5 languages in the sample data:
``` request
response = client.query(language.topN(5));
System.out.println("Top Languages: " + response.getResult().getCountItems());
```
``` response
Top Languages: [CountResultItem(id=5, count=119), CountResultItem(id=1, count=50), CountResultItem(id=4, count=48), CountResultItem(id=9, count=31), CountResultItem(id=13, count=25)]
```
Which repositories were starred by user 14 and 19:
``` request
response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19)));
System.out.println("Both user 14 and 19 starred: " + response.getResult().getRow().getColumns());
```
``` response
Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486]
```
Which repositories were starred by user 14 or 19:
``` request
response = client.query(repository.union(stargazer.row(14), stargazer.row(19)));
System.out.println("User 14 or 19 starred: " + response.getResult().getRow().getColumns());
```
``` response
User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514]
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
``` request
response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19), language.row(1)));
System.out.println("Both user 14 and 19 starred and were written in language 1: " + response.getResult().getRow().getColumns());
```
``` response
Both user 14 and 19 starred and were written in language 1: [2, 362, 416, 461]
```
Set user 99999 as a stargazer for repository 77777:
``` request
client.query(stargazer.set(99999, 77777));
System.out.println("Set user 99999 as a stargazer for repository 77777");
```
``` response
Set user 99999 as a stargazer for repository 77777
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
For more information about java-pilosa, please see our Java client library at [java-pilosa](https://github.com/pilosa/java-pilosa) or checkout the java-pilosa [Data Model and Queries](https://github.com/pilosa/java-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
#### Python Users
Pilosa requires Python 2.7 or higher or Python 3.4 or higher.
##### Create the Environment
Create a new project folder:
```
mkdir getting-started && cd getting-started
```
In this folder, we will download two CSV files to provide data to our fields later on. Download the `stargazer.csv` and `language.csv` files here:
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.csv
```
We will also download two text files. One is the `requirements.txt` that will install python-pilosa later on and the other is `languages.txt` which will provide context to the `language` field.
```
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/python/requirements.txt
curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/language.txt
```
We will now create the python environment:
```
python3 -m venv startrace
```
Next, we activate the python environment we created and install the single dependency, python-pilosa:
```
source startrace/bin/activate
pip install -r requirements.txt
```
We will also create a file called `startrace.py` as follows:
```
touch startrace.py
```
This file will be used in the following sections.
##### Create the Schema
Before we can import data or run queries, we need to create our schema. You can see the dependencies dealing with `pilosa` are from the python-pilosa library. Create the schema by creating a client which will communicate our schema to Pilosa, creating a schema which will contain our indexes and fields, and syncing with Pilosa. This is all done in the `startrace.py` file:
```
from __future__ import print_function
import os
import sys
import time
import pilosa
from pilosa import Client, Index, TimeQuantum
from pilosa.imports import csv_column_reader, csv_row_id_column_id
try:
# Python 2.7 and 3
from io import StringIO
except ImportError:
# Python 2.6 and 2.7
from StringIO import StringIO
# Create the Schema
client = pilosa.Client()
schema = client.schema()
# This is where the index will go later
# This is where the fields will go later
client.sync_schema(schema)
```
Next, let's create the `repository` index:
```
repository = schema.index("repository")
```
The index name must be 230 characters or fewer, start with a letter, and consist only of lowercase alphanumeric characters or `_-`. The same goes for field names.
Let's create the `stargazer` field which has user IDs of stargazers as its rows:
```
stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY)
```
Since our data contains time stamps which represent the time users starred repos, we establish the time aspect by using `time_quantum`. Time quantum is the resolution of the time we want to use, and we set it to `YEAR_MONTH_DAY` for `stargazer`.
Next up is the `language` field, which will contain IDs for programming languages:
```
language = repository.field("language")
```
The `language` field is a `set` field, but since the defualt field is `set`, we didn't need to specify any options.
Your `StarTrace.py` file should look like:
```
from __future__ import print_function
import os
import sys
import time
import pilosa
from pilosa import Client, Index, TimeQuantum
from pilosa.imports import csv_column_reader, csv_row_id_column_id
try:
# Python 2.7 and 3
from io import StringIO
except ImportError:
# Python 2.6 and 2.7
from StringIO import StringIO
# Create the Schema
client = pilosa.Client()
schema = client.schema()
repository = schema.index("repository")
stargazer = repository.field("stargazer", time_quantum=pilosa.TimeQuantum.YEAR_MONTH_DAY)
language = repository.field("language")
client.sync_schema(schema)
```
##### Import Data From CSV Files
Now that we have our index and our fields, we can import the data we downloaded earlier and be on our way to making our own queries.
First, we will load our data into the `stargazer` field:
```
time_func = lambda s: int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M")))
with open("stargazer.csv") as f:
stargazer_reader = csv_column_reader(f, timefunc=time_func)
client.import_field(stargazer, stargazer_reader)
```
Due to the time aspect of the `stargazer` csv file, we have to specify the time stamp format in the `csv_column_reader` function. We set the variable `time_func` to the format present in the csv file and call it in the `csv_column_reader` function, which will take the csv file and the time stamp format and translate the csv file into a format Pilosa can read
Next, we will load our data into the `language` field:
```
with open("language.csv") as f:
language_reader = csv_column_reader(f, csv_row_id_column_id)
client.import_field(language, language_reader)
```
The `language` is a `set` field, but since the default field type is `set`, we didn't need to specify it.
For more information on imports in python-pilosa, please see the python-pilosa [site](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md).
Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages.
##### Make Some Queries
Now that we have a working schema, we can query it.
Which repositories did user 14 star:
``` request
response = client.query(stargazer.row(14))
print("User 14 starred: ", response.result.row.columns)
```
``` response
User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
```
What are the top 5 languages in the sample data:
``` request
def load_language_names():
with open("languages.txt") as f:
return [line.strip() for line in f]
def print_topn(items):
lines = ["\t{i}. {s[0]}: {s[1]} stars".format(s=s, i=i + 1) for i, s in enumerate(items)]
print("\n".join(lines))
language_names = load_language_names()
top_languages = client.query(language.topn(5)).result.count_items
language_items = [(language_names[item.id], item.count) for item in top_languages]
print("Top languages: ")
print_topn(language_items)
```
``` response
Top languages:
1. Go: 119 stars
2. Shell: 50 stars
3. Makefile: 48 stars
4. HTML: 31 stars
5. JavaScript: 25 stars
```
Which repositories were starred by user 14 and 19:
``` request
repsonse = client.query(repository.intersect(stargazer.row(14), stargazer.row(19)))
print("Both user 14 and 19 starred: ", response.result.row.columns)
```
``` resposne
Both user 14 and 19 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514]
```
Which repositories were starred by user 14 or 19:
``` request
response = client.query(repository.union(stargazer.row(14), stargazer.row(19)))
print("User 14 or 19 starred: ", response.result.row.columns)
```
``` response
User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514]
```
Which repositories were starred by user 14 and 19 and also were written in language 1:
``` request
response = client.query(repository.intersect(stargazer.row(14), stargazer.row(19), language.row(1)))
print("Both user 14 and 19 starred and were written in language 1: ", response.result.row.columns)
```
``` response
Both user 14 and 19 starred and were written in language 1: [2, 362, 416, 461]
```
Set user 99999 as a stargazer for repository 77777:
``` request
client.query(stargazer.set(99999, 77777))
print("Set user 99999 as a stargazer for repository 77777")
```
``` response
Set user 99999 as a stargazer for repository 77777
```
Please note that while user ID 99999 may not be sequential with the other column IDs, it is still a relatively low number.
Don't try to use arbitrary 64-bit integers as column or row IDs in Pilosa - this will lead to problems such as poor performance and out of memory errors.
For more information about python-pilosa, please see our Python client library at [python-pilosa](https://github.com/pilosa/python-pilosa) or checkout the python-pilosa [Data Model and Queries](https://github.com/pilosa/python-pilosa/blob/master/docs/data-model-queries.md) section for more query options.
### What's Next?
You can jump to [Data Model](../data-model/) for an in-depth look at Pilosa's data model, or [Query Language](../query-language/) for more details about **PQL**, the query language of Pilosa. Check out the [Examples](../examples/) page for example implementations of real world use cases for Pilosa. Ready to get going in your favorite language? Have a peek at our small but expanding set of official [Client Libraries](../client-libraries/).

77
docs/glossary.md
View file

@ -1,77 +0,0 @@
+++
title = "Glossary"
weight = 14
nav = []
+++
## Glossary
<strong id="anti-entropy">[Anti-entropy](../configuration/#anti-entropy-interval):</strong> A periodic process that compares each [shard](#shard) and its [replicas](#replica) across the [cluster](#cluster) to repair inconsistencies.
<strong id="attribute">[Attribute](../data-model/#attribute):</strong> Attributes can be associated to both [rows](#row) and [columns](#column). This metadata is kept separately from the core binary matrix in a [BoltDB](https://github.com/boltdb/bolt) store.
<strong id="bit">[Bit](../data-model/#overview):</strong> Bits are the fundamental unit of data in Pilosa. A bit lives in a [field](#field), at the intersection of a [row](#row) and [column](#column).
<strong id="bitmap">[Bitmap](../data-model/#overview):</strong> The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap).
<strong id="bsi">[BSI](../data-model/#bsi-range-encoding):</strong> Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in `int` [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries.
<strong id="cluster">Cluster:</strong> A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries.
<strong id="column">[Column](../data-model/#column):</strong> Columns are the fundamental horizontal data axis within Pilosa. Columns are global to all [fields](#field) within an [index](#index).
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of five types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.
<strong id="gossip">[Gossip](https://en.wikipedia.org/wiki/Gossip_protocol):</strong> A protocol used by Pilosa for internal communication.
<strong id="groupby">[GroupBy](../query-language/#group-by):</strong> A [PQL](#pql) query, with functionality similar to a SQL `GROUP BY` clause, that returns the count of the intersection of every combination of rows taking one row each from the specified `Rows` calls. GroupBy can be thought of as a multi-dimensional version of the [TopN](#topn) query.
<strong id="index">[Index](../data-model/#index):</strong> An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Basic queries cannot operate across multiple indexes.
<strong id="jump-consistent-hash">[Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf):</strong> A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes.
<strong id="max">[Max](../query-language/#max):</strong> A [PQL](#pql) query that returns the maximum integer value stored in an [integer](#bsi) [field](#field).
<strong id="maxshard">MaxShard:</strong> The total number of [shards](#shard) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. MaxShard is zero-indexed, so if an index contains six shards, its MaxShard will be 5.
<strong id="min">[Min](../query-language/#min):</strong> A [PQL](#pql) query that returns the minimum integer value stored in an [integer](#bsi) [field](#field).
<strong id="node">Node:</strong> An individual running instance of Pilosa server which belongs to a [cluster](#cluster).
<strong id="partition">Partition:</strong> The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data.
<strong id="pql">[PQL](../query-language/):</strong> Pilosa Query Language.
<strong id="protobuf">[Protobuf](https://developers.google.com/protocol-buffers/):</strong> Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON.
<strong id="replica">[Replica](../configuration/#cluster-replicas):</strong> A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made.
<strong id="roaring-bitmap">[Roaring Bitmap](http://roaringbitmap.org):</strong> the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations.
<strong id="row">[Row](../data-model/#row):</strong> Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap).
<strong id="range">[Row (Ranged)](../query-language/#row-range):</strong> A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum).
<strong id="range-bsi">[Row (BSI)](../query-language/#row-bsi):</strong> A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field).
<strong id="rows">[Rows](../query-language/#rows):</strong> A [PQL](#pql) query that returns a list of row IDs in the given field which have at least one bit set. The field argument is mandatory, the others are optional. `Rows` is the primary argument used with the [GroupBy](#groupby) query.
<strong id="slice">[Slice](../data-model/#shard):</strong> Prior to Pilosa 1.0, shards were known as slices.
<strong id="shard">[Shard](../data-model/#shard):</strong> [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash).
<strong id="shardwidth">ShardWidth:</strong> This is the number of [columns](#column) in a [shard](#shard). `ShardWidth` defaults to 2<sup>20</sup> or about one million. It can be modified, but only at compile time, and before ingesting any data.
<strong id="sum">[Sum](../query-language/#sum):</strong> A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field).
<strong id="time-quantum">[Time quantum](../data-model/#time-quantum):</strong> Defines the granularity to be used for [ranged Row](#range) queries on time [fields](#field).
<strong id="toml">[TOML](https://github.com/toml-lang/toml):</strong> the language used for Pilosa's [configuration file](../configuration/).
<strong id="topn">[TopN](../query-language/#topn):</strong> A [PQL](#pql) query that returns a list of rows, sorted by the count of [columns](#column) set in the [row](#row), within a specified [field](#field).
<strong id="view">View:</strong> Views separate the different data layouts within a [Field](#field). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based field views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation.

382
docs/installation.md
View file

@ -1,382 +0,0 @@
+++
title = "Installation"
weight = 2
nav = [
"Installing on MacOS",
"Installing on Linux",
]
+++
## Installation
Pilosa is currently available for [MacOS](#installing-on-macos) and [Linux](#installing-on-linux).
### Installing on MacOS
There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) (recommended), download the binary, build from source, or use [Docker](#docker).
#### Use Homebrew
1. Update your Homebrew formulas:
```
brew update
```
2. Install Pilosa
```
brew install pilosa
```
3. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.4.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Download the Binary
1. Download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.4.0/pilosa-v1.4.0-darwin-amd64.tar.gz
```
Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v1.4.0-darwin-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v1.4.0-darwin-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.4.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make install
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.4.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### What's next?
Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
### Installing on Linux
There are three ways to install Pilosa on Linux: download the binary (recommended), build from source, or use [Docker](#docker).
#### Download the Binary
1. To install the latest version of Pilosa, download the latest release:
```
curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.4.0/pilosa-v1.4.0-linux-amd64.tar.gz
```
Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
2. Extract the binary:
```
tar xfz pilosa-v1.4.0-linux-amd64.tar.gz
```
3. Move the binary into your PATH so you can run `pilosa` from any shell:
```
cp -i pilosa-v1.4.0-linux-amd64/pilosa /usr/local/bin
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.4.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### Build from Source
<div class="note">
<p>For advanced instructions for building from source, view our <a href="https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md">Contributor's Guide.</a></p>
</div>
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
* [Git](https://git-scm.com/)
2. Clone the repo:
```
mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make install
```
4. Make sure Pilosa is installed successfully:
```
pilosa
```
If you see something like:
```
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Version: v1.4.0
Build Time: 2018-05-14T22:14:01+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
You're good to go!
#### What's next?
Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.
### Windows
Windows is currently not supported as a target deployment platform for Pilosa, but developing and running Pilosa is made possible by Docker. See the [Docker](#docker) documentation for using Docker for Windows and Docker Toolbox.
Windows Subsystem for Linux is currently not supported.
### Docker
1. Install Docker for your platform. On Linux, Docker is available via your package manager. On MacOS, you can use Docker for Mac or Docker Toolbox. On Windows, you can use Docker for Windows or Docker Toolbox.
2. **This step is necessary only if you are using Docker Toolbox**, otherwise skip to step 3:
a. Start the Docker support using `docker-machine start` in a terminal. The environment variables of the terminal should be updated accordingly, run `docker-machine env` to display the necessary commands.
b. Set up port forwarding in the VirtualBox GUI or on the command line. Guest port should be 10101. For the host port, 10101 is recommended. If the `VBoxManage` command is in your `PATH`, you can use the following command (assuming you use the default VM):
```
VBoxManage modifyvm "default" --natpf1 "pilosa,tcp,,10101,,10101"
```
3. Confirm that the Docker daemon is running in the background:
```
docker version
```
If you are getting a "command not found" or similar, check that `docker` command is in your path. If you don't see the server listed, start the Docker application.
4. Pull the official Pilosa image from Docker Hub:
```
docker pull pilosa/pilosa:latest
```
5. Make sure Pilosa is installed successfully, and make it accessible:
```
docker run -d --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest server --bind 0.0.0.0:10101
```
6. Check that it is accessible from outside the container.
Run the following in a separate terminal:
```
curl localhost:10101/schema
```
If that returns `{"indexes":null}` or similar, then Pilosa is accessible from outside the container. Otherwise check that you have correctly typed `-p 10101:10101` when running the Pilosa container and the port mappings in VirtualBox is correct (Docker Toolbox only).
7. When you want to terminate the Pilosa container, you can run the following:
```
docker stop pilosa
```
#### What's next?
Head over to the [Getting Started](../getting-started/) guide to create your first Pilosa index.

19
docs/introduction.md
View file

@ -1,19 +0,0 @@
+++
title = "Introduction"
weight = 1
nav = []
+++
## Introduction
Pilosa is an open source, distributed index.
[//]: # (TODO insert a graphic here?)
It is designed primarily for speed and horizontal scalability. If you have data with billions of objects that can have millions of possible attributes, and you want to explore those relationships, Pilosa can help you.
"What attributes are the most common?", "Which objects have these specific attributes?", "What groups of attributes often appear together?" Pilosa is designed to answer these types of queries in real time, suitable for use with high rate data streams, or to power a user interface.
Once you have Pilosa [installed](../installation/), the [getting started](../getting-started/) guide will show you the basics of interacting with Pilosa and give you some pointers for deeper exploration.

74
docs/pdk.md
View file

@ -1,74 +0,0 @@
+++
title = "PDK"
weight = 11
nav = [
"Examples and Executables",
"Library",
]
+++
## PDK
The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively.
### Examples and Executables
Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here.
#### Kafka
`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the
Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each
record from Kafka is assigned a Pilosa column, and each value in a record is
assigned a row or field. Pilosa field names are built from the "path" through
the record to arrive at that field. For example:
```json
{
"name": "jill",
"favorite_foods": ["corn chips", "chipotle dip"],
"location": {
"city": "Austin",
"state": "Texas",
"latitude": 3754,
"longitude": 4526
},
"active": true,
"age": 27
}
```
This JSON object would result in the following Pilosa schema:
| Field | Example Value | Type | Cache Size |
|----------------|---------------|--------|------------|
| name | "jill" | ranked | 100000 |
| favorite_foods | "corn chips" | ranked | 100000 |
| default | | ranked | 100000 |
| age | 27 | int | |
| location | | ranked | 1000 |
| latitude | 3754 | int | |
| longitude | 4526 | int | |
| location-city | "Austin" | ranked | 100000 |
| location-state | "Texas" | ranked | 100000 |
All set fields are created as ranked fields by default, with the cache size
listed above. Integer fields are created with a minimum size of zero and a
fixed maximum of 2147483647. Field names are a dash-separated concatenation of
all key values in the path - you can see this with fields like location-city.
Most of the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts,
kafka topics, kafka group, etc.), but there are a few options that give some
control over the way data is indexed, and ingestion performance.
* `--batch-size`: The batch size controls how many set bits or values are batched up to be imported *per field*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner.
* `--framer.collapse`: This is a list of strings which will be removed from the field names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be fields named "city" and "state" rather than "location-city" and "location-state".
* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it.
* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used.
* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb.
For more information on running `pdk kafka` and how Pilosa interfaces with Kafka, please see the [kafka directory](https://github.com/pilosa/pdk/tree/master/kafka) in the pdk repository.
### Library
For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation.

1060
docs/query-language.md
View file

File diff suppressed because it is too large Load diff

779
docs/tutorials.md
View file

@ -1,779 +0,0 @@
+++
title = "Tutorials"
weight = 4
nav = [
"Setting Up a Secure Cluster",
"Setting Up a Docker Cluster",
"Using Integer Field Values",
"Storing Row and Column Attributes",
]
+++
## Tutorials
<div class="note">
<!-- this is html because there is a problem putting a list inside a shortcode -->
Some of our tutorials work better as standalone repos, since you can <code>git clone</code> the instructions, code, and data all at once. Officially supported tutorials are listed here.<br />
<br />
<ul>
<li><a href="https://github.com/pilosa/cosmosa">Run Pilosa with Microsoft's Azure Cosmos DB</a></li>
</ul>
</div>
### Setting Up a Secure Cluster
#### Introduction
Pilosa supports encrypting all communication with nodes in a cluster using TLS, including [Mutual TLS Authentication](https://en.wikipedia.org/wiki/Mutual_authentication). In this tutorial, we will be setting up a three node Pilosa cluster running on the same computer. The same steps can be used for a multi-computer cluster but that requires setting up firewalls and other platform-specific configuration which is beyond the scope of this tutorial.
This tutorial assumes that you are using a UNIX-like system, such as Linux or MacOS. [Windows Subsystem for Linux (WSL)](https://msdn.microsoft.com/en-us/commandline/wsl/about) works equally well on Windows 10 systems.
#### Installing Pilosa and Creating the Directory Structure
If you haven't already done so, install Pilosa server on your computer. For Linux and WSL (Windows Subsystem for Linux) use the [Installing on Linux](../installation/#installing-on-linux) instructions. For MacOS use the [Installing on MacOS](../installation/#installing-on-macos). We do not support precompiled releases for other platforms, but you can always compile it yourself from source. See [Build from Source](../installation/#build-from-source).
After installing Pilosa, you may have to add it to your `$PATH`. Check that you can run Pilosa from the command line:
``` request
pilosa --help
```
``` response
Pilosa is a fast index to turbocharge your database.
This binary contains Pilosa itself, as well as common
tools for administering pilosa, importing/exporting data,
backing up, and more. Complete documentation is available
at https://www.pilosa.com/docs/.
Pilosa v1.4.0
Build Time: 2019-09-23T14:33:07+0000
Usage:
pilosa [command]
Available Commands:
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
generate-config Print the default configuration.
help Help about any command
holder Load Pilosa.
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
server Run Pilosa.
Flags:
-c, --config string Configuration file to read from.
-h, --help help for pilosa
Use "pilosa [command] --help" for more information about a command.
```
First, create a directory in which to put all of the files for this tutorial. Then switch to that directory:
```
mkdir $HOME/pilosa-tls-tutorial && cd $_
```
#### Creating the TLS Certificate and Gossip Key
Securing a Pilosa cluster consists of securing the communication between nodes using TLS and Gossip encryption.
The first step is acquiring the necessary TLS certificates. Operating your own public key infrastructure (PKI) is outside of the scope of this tutorial, but it is easy to get started with [certstrap](https://github.com/square/certstrap) for testing/development purposes. For production, you can use OpenSSL or any other software that provides PKI using X.509 certificates, including [Hashicorp Vault](https://learn.hashicorp.com/vault/secrets-management/sm-pki-engine). It is not recommended to use certstrap in production.
First, create a certificate authority (CA):
```
$ certstrap init --common-name ca
Created out/ca.key
Created out/ca.crt
Created out/ca.crl
```
The command above creates three files in the `out/` directory:
* `ca.key` is the CA private key file which must be kept as secret.
* `ca.crt` is the CA TLS certificate.
* `ca.crl` is the Certificate Revocation List (CRL).
Next, create and sign a wildcard certificate for pilosa:
```
$ certstrap request-cert --cn "*.pilosa.local"
Created out/*.pilosa.local.key
Created out/*.pilosa.local.csr
$ certstrap sign "*.pilosa.local" --CA ca
Created out/*.pilosa.local.crt from out/*.pilosa.local.csr signed by out/ca.key
```
The commands above create three files in the `out/` directory:
* `*.pilosa.local.key` is the private key file which must be kept as secret.
* `*.pilosa.local.csr` is the certificate signing request (CSR).
* `*.pilosa.local.crt` is the signed TLS certificate.
You can also create a separate client certificate signed by the same CA to test mutual TLS using curl:
```
$ certstrap request-cert --cn "curl"
Created out/curl.key
Created out/curl.csr
$ certstrap sign "curl" --CA ca
Created out/curl.crt from out/curl.csr signed by out/ca.key
```
Having created the TLS certificates, we can now create the gossip encryption key. The gossip encryption key file must be exactly 16, 24, or 32 bytes to select one of AES-128, AES-192, or AES-256 encryption. Reading random bytes from cryptographically secure `/dev/random` serves our purpose very well:
```
head -c 32 /dev/random > pilosa.local.gossip32
```
We now have a file called `pilosa.local.gossip32` in the current directory which contains 32 random bytes.
#### Creating the Configuration Files
Pilosa supports passing configuration items using command line options, environment variables, or a configuration file. For this tutorial, we will use three configuration files; one configuration file for each of our three nodes.
One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated.
Create `node1.config.toml` in the project directory and paste the following in it:
```toml
# node1.config.toml
data-dir = "node1_data"
bind = "https://01.pilosa.local:10501"
[cluster]
coordinator = true
[tls]
ca-certificate = "out/ca.crt"
certificate = "out/*.pilosa.local.crt"
key = "out/*.pilosa.local.key"
enable-client-verification = true
[gossip]
seeds = ["01.pilosa.local:15000"]
port = 15000
key = "pilosa.local.gossip32"
```
Create `node2.config.toml` in the project directory and paste the following in it:
```toml
# node2.config.toml
data-dir = "node2_data"
bind = "https://02.pilosa.local:10502"
[tls]
ca-certificate = "out/ca.crt"
certificate = "out/*.pilosa.local.crt"
key = "out/*.pilosa.local.key"
enable-client-verification = true
[gossip]
seeds = ["01.pilosa.local:15000"]
port = 16000
key = "pilosa.local.gossip32"
```
Create `node3.config.toml` in the project directory and paste the following in it:
```toml
# node3.config.toml
data-dir = "node3_data"
bind = "https://03.pilosa.local:10503"
[tls]
ca-certificate = "out/ca.crt"
certificate = "out/*.pilosa.local.crt"
key = "out/*.pilosa.local.key"
enable-client-verification = true
[gossip]
seeds = ["01.pilosa.local:15000"]
port = 17000
key = "pilosa.local.gossip32"
```
Here is some explanation of the configuration items:
* `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it.
* `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes.
* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings.
* `[tls]` section contains the TLS settings, including the path to the TLS certificate and the corresponding key. The `ca-certificate` setting is optional and will default to your system CAs. You may also disable server-to-server verification by setting `skip-verify` to `true`, which we don't recommend for production.
* `[gossip]` section contains settings for the gossip protocol. `seeds` contains the list of nodes from which to seed cluster membership. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. If all nodes of the cluster are running on the same computer, the gossip listen address should be different for each node. Otherwise, it can be set to the same value. Finally, the `key` points to the gossip encryption key we created earlier.
#### Final Touches Before Running the Cluster
Before running the cluster, let's make sure that `01.pilosa.local`, `02.pilosa.local` and `03.pilosa.local` resolve to an IP address. If you are running the cluster on your computer, it is adequate to add them to your `/etc/hosts`. Below is one of the many ways of doing that (mind the `>>`):
```
sudo sh -c 'printf "\n127.0.0.1 01.pilosa.local 02.pilosa.local 03.pilosa.local\n" >> /etc/hosts'
```
Ensure we can access the hosts in the cluster:
```
ping -c 1 01.pilosa.local
ping -c 1 02.pilosa.local
ping -c 1 03.pilosa.local
```
If any of the commands above return `ping: unknown host`, make sure your `/etc/hosts` contains the failed hostname.
#### Running the Cluster
Let's open three terminal windows and run each node in its own window. This will enable us to better observe what's happening on each node.
Switch to the first terminal window, change to the project directory and start the first node:
```
cd $HOME/pilosa-tls-tutorial
pilosa server -c node1.config.toml
```
Switch to the second terminal window, change to the project directory and start the second node:
```
cd $HOME/pilosa-tls-tutorial
pilosa server -c node2.config.toml
```
Switch to the third terminal window, change to the project directory and start the third node:
```
cd $HOME/pilosa-tls-tutorial
pilosa server -c node3.config.toml
```
Let's ensure that all three Pilosa servers are running and they are connected:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://01.pilosa.local:10501/status
```
``` response
{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]}
```
The `-k` flag is used to tell curl that it shouldn't bother checking the certificate the server provides, and the `--ipv4` flag avoids an issue on MacOS where the curl request takes a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL.
If everything is set up correctly, the cluster state should be `NORMAL`.
#### Running Queries
Having confirmed that our cluster is running normally, let's perform a few queries. First, we need to create an index and a field:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://01.pilosa.local:10501/index/sample-index \
-X POST
```
``` response
{"success":true}
```
This will create index `sample-index` with default options. Let's create the field now:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://01.pilosa.local:10501/index/sample-index/field/sample-field \
-X POST
```
``` response
{"success":true}
```
We just created field `sample-field` with default options.
Let's run a `Set` query:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://01.pilosa.local:10501/index/sample-index/query \
-X POST \
-d 'Set(100, sample-field=1)'
```
``` response
{"results":[true]}
```
Confirm that the value was indeed set:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://01.pilosa.local:10501/index/sample-index/query \
-X POST \
-d 'Row(sample-field=1)'
```
``` response
{"results":[{"attrs":{},"columns":[100]}]}
```
The same response should be returned when querying other nodes in the cluster:
``` request
curl --cacert out/ca.crt --cert out/curl.crt --key out/curl.key \
https://02.pilosa.local:10502/index/sample-index/query \
-X POST \
-d 'Row(sample-field=1)'
```
``` response
{"results":[{"attrs":{},"columns":[100]}]}
```
#### What's Next?
Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
### Setting Up a Docker Cluster
In this tutorial, we will be setting up a 2-node Pilosa cluster using Docker containers.
#### Running a Docker Cluster on a Single Server
The instructions below require Docker 1.13 or better.
Let's first be sure that the Pilosa image is up to date:
```
docker pull pilosa/pilosa:latest
```
Then, create a virtual network to attach our containers. We are going to name our network `pilosanet`:
```
docker network create pilosanet
```
Let's run the first Pilosa node and attach it to that virtual network. We set the first node as the cluster coordinator and use its address as the gossip seed. And also set the server address to `pilosa1`:
```
docker run -it --rm --name pilosa1 -p 10101:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
```
Let's run the second Pilosa node and attach it to the virtual network as well. Note that we set the address of the gossip seed to the address of the first node:
```
docker run -it --rm --name pilosa2 -p 10102:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
```
Let's test that the nodes in the cluster connected with each other:
``` request
curl localhost:10101/status
```
``` response
{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
```
And similarly for the second node:
``` request
curl localhost:10102/status
```
``` response
{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1"}
```
The corresponding [Docker Compose](https://docs.docker.com/compose/) file is below:
```yaml
version: '2'
services:
pilosa1:
image: pilosa/pilosa:latest
ports:
- "10101:10101"
environment:
- PILOSA_CLUSTER_COORDINATOR=true
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
networks:
- pilosanet
entrypoint:
- /pilosa
- server
- --bind
- "pilosa1:10101"
pilosa2:
image: pilosa/pilosa:latest
ports:
- "10102:10101"
environment:
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
networks:
- pilosanet
entrypoint:
- /pilosa
- server
- --bind
- "pilosa2:10101"
networks:
pilosanet:
```
#### Running a Docker Swarm
It is very easy to run a Pilosa Cluster on different servers using [Docker Swarm mode](https://docs.docker.com/engine/swarm/). All we have to do is create an overlay network instead of a bridge network.
The instructions in this section require Docker 17.06 or newer. Although it is possible to run a Docker swarm on MacOS or Windows, it is easiest to run it on Linux. The following instructions assume you are running on Linux.
We are going to use two servers: the manager node runs in the first server and a worker node in the second server.
Docker nodes require some ports to be accesible from the outside. Before proceeding, make sure the following ports are open on all nodes: TCP/2377, TCP/7946, UDP/7946, UDP/4789.
Let's initialize the swarm first. Run the following on the manager:
```
docker swarm init --advertise-addr=IP-ADDRESS
```
Virtual machines running on the cloud usually have at least two network interfaces: the external interface and the internal interface. Use the IP of the external interface.
The output of the command above should be similar to:
```
To add a manager to this swarm, run the following command:
docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
```
Let's make the worker node join the manager. Copy/paste the command above in a shell on the worker, replacing the token and IP address with the correct values. You may neeed to add `--advertise-addr=WORKER-EXTERNAL-IP-ADDRESS` parameter if the worker has more than one network interface:
```
docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
```
Run the following on the manager to check that the worker joined to the swarm:
```
docker node ls
```
Which should output:
ID|HOSTNAME|STATUS|AVAILABILITY|MANAGER STATUS|ENGINE VERSION
---|--------|------|------------|--------------|-------------
MANAGER-ID *|swarm1|Ready|Active|Leader|18.05.0-ce|
WORKER-ID|swarm2|Ready|Active||18.05.0-ce|
If you have created the `pilosanet` network before, delete it before carrying on, otherwise skip to the next step:
```
docker network rm pilosanet
```
Let's create the `pilosanet` network, but with `overlay` type this time. We should also make this network attachable in order to be able to attach containers to it. Run the following on the manager:
```
docker network create -d overlay pilosanet --attachable
```
We can now create the Pilosa containers. Let's start the coordinator node first. Run the following on one of the servers:
```
docker run -it --rm --name pilosa1 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
```
And the following on the other server:
```
docker run -it --rm --name pilosa2 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
```
These were the same commands we used in the previous section except the port mapping! Let's run another container on the same virtual network to read the status from the coordinator:
``` request
docker run -it --rm --network=pilosanet --name shell alpine wget -q -O- pilosa1:10101/status
```
``` response
{"state":"NORMAL","nodes":[{"id":"3e3b0abd-1945-441a-a01f-5a28272972f5","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"71ed27cc-9443-4f41-88fb-1c22f92bf695","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}],"localID":"3e3b0abd-1945-441a-a01f-5a28272972f5"}
```
You can add additional worker nodes to both the swarm and the Pilosa cluster using the steps above.
#### What's Next?
Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
Refer to the [Docker documentation](https://docs.docker.com) to see your options about running Docker containers. The [Networking with overlay networks](https://docs.docker.com/network/network-tutorial-overlay/) is a detailed overview of the Docket swarm mode and overlay networks.
### Using Integer Field Values
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support `Row`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
``` request
curl localhost:10101/index/patients \
-X POST
```
``` response
{"success":true}
```
In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `patients` index.
``` request
curl localhost:10101/index/patients/field/age \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 120}}'
```
``` response
{"success":true}
```
``` request
curl localhost:10101/index/patients/field/weight \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 500}}'
```
``` response
{"success":true}
```
``` request
curl localhost:10101/index/patients/field/tcells \
-X POST \
-d '{"options":{"type": "int", "min": 0, "max": 2000}}'
```
``` response
{"success":true}
```
Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Set(1, age=34)'
```
``` response
{"results":[true]}
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Set(1, weight=128)'
```
``` response
{"results":[true]}
```
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Set(1, tcells=1145)'
```
``` response
{"results":[true]}
```
In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
Assuming we have a file called `ages.csv` that is structured like this:
```
1,34
2,57
3,19
4,40
5,32
6,71
7,28
8,33
9,63
```
where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command:
```
pilosa import -i patients --field age ages.csv
```
Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
In order to find all patients over the age of 40, then simply run a `Row` query against the `age` field.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Row(age > 40)'
```
``` response
{"results":[{"attrs":{},"columns":[2,6,9]}]}
```
You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#row-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(field="age")'
```
``` response
{"results":[{"value":377,"count":9}]}
```
The results you get from the `Sum` query contain the sum of all values as well as the `count` of columns with a value. To get the average you can just divide `value` by `count`.
You can also provide a filter to the `Sum()` function to find the average age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Row(age > 40), field="age")'
```
``` response
{"results":[{"value":191,"count":3}]}
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
To find the minimum age of all patients, run a `Min` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(field="age")'
```
``` response
{"results":[{"value":19,"count":1}]}
```
The results you get from the `Min` query contain the minimum `value` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Min(Row(age > 40), field="age")'
```
``` response
{"results":[{"value":57,"count":1}]}
```
To find the maximum age of all patients, run a `Max` query:
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(field="age")'
```
``` response
{"results":[{"value":71,"count":1}]}
```
The results you get from the `Max` query contain the maximum `value` of all values as well as the `count` of columns with that value.
You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40.
``` request
curl localhost:10101/index/patients/query \
-X POST \
-d 'Max(Row(age < 40), field="age")'
```
``` response
{"results":[{"value":34,"count":1}]}
```
### Storing Row and Column Attributes
#### Introduction
Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data.
First, create an index called `books` to use for this tutorial:
``` request
curl localhost:10101/index/books \
-X POST
```
``` response
{"success":true}
```
Next, create a field in the `books` index called `members` which will represent library members who have read books.
``` request
curl localhost:10101/index/books/field/members \
-X POST \
-d '{}'
```
``` response
{"success":true}
```
Now, let's add some books to our index.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(2, name="No Name in the Street", year=1972)
SetColumnAttrs(3, name="The Tipping Point", year=2000)
SetColumnAttrs(4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(5, name="The Forever War", year=2008)'
```
``` response
{"results":[null,null,null,null,null]}
```
And add some members.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(members, 10001, fullName="John Smith")
SetRowAttrs(members, 10002, fullName="Sue Perkins")
SetRowAttrs(members, 10003, fullName="Jennifer Hawks")
SetRowAttrs(members, 10004, fullName="Pedro Vazquez")
SetRowAttrs(members, 10005, fullName="Pat Washington")'
```
``` response
{"results":[null,null,null,null,null]}
```
At this point we can query one of the `member` records by querying that row.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Row(members=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[]}]}
```
Now let's add some data to the matrix such that each pair represents a member who has read that book.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Set(3, members=10001)
Set(5, members=10001)
Set(1, members=10002)
Set(2, members=10002)
Set(4, members=10002)
Set(3, members=10003)
Set(4, members=10004)
Set(5, members=10004)
Set(1, members=10005)
Set(2, members=10005)
Set(3, members=10005)
Set(4, members=10005)
Set(5, members=10005)'
```
``` response
{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]}
```
Now pull the record for `Sue Perkins` again.
``` request
curl localhost:10101/index/books/query \
-X POST \
-d 'Row(members=10002)'
```
``` response
{"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}]}
```
Notice that the result set now contains a list of integers in the `columns` attribute. These integers match the column IDs of the books that Sue has read.
In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Row(members=10002)'
```
``` response
{
"results":[{"attrs":{"fullName":"Sue Perkins"},"columns":[1,2,4]}],
"columnAttrs":[
{"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}},
{"id":2,"attrs":{"name":"No Name in the Street","year":1972}},
{"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
]
}
```
The `book` attributes are included in the result set at the `columnAttrs` attribute.
Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members:
``` request
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Row(members=10002), Row(members=10004))'
```
``` response
{
"results":[{"attrs":{},"columns":[4]}],
"columnAttrs":[
{"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
]
}
```
Notice that we don't get row attributes on a complex query, but we still get the column attributes—in this case book information.

164
encoding/proto/proto.go
View file

@ -21,9 +21,12 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/internal"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pkg/errors"
)
@ -136,22 +139,6 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
s.decodeResizeInstructionComplete(msg, mt)
return nil
case *pilosa.SetCoordinatorMessage:
msg := &internal.SetCoordinatorMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling SetCoordinatorMessage")
}
s.decodeSetCoordinatorMessage(msg, mt)
return nil
case *pilosa.UpdateCoordinatorMessage:
msg := &internal.UpdateCoordinatorMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling UpdateCoordinatorMessage")
}
s.decodeUpdateCoordinatorMessage(msg, mt)
return nil
case *pilosa.NodeStateMessage:
msg := &internal.NodeStateMessage{}
err := proto.Unmarshal(buf, msg)
@ -168,6 +155,14 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
s.decodeRecalculateCaches(msg, mt)
return nil
case *pilosa.LoadSchemaMessage:
msg := &internal.LoadSchemaMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling LoadSchemaMessage")
}
s.decodeLoadSchemaMessage(msg, mt)
return nil
case *pilosa.NodeEvent:
msg := &internal.NodeEventMessage{}
err := proto.Unmarshal(buf, msg)
@ -184,7 +179,7 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
s.decodeNodeStatus(msg, mt)
return nil
case *pilosa.Node:
case *topology.Node:
msg := &internal.Node{}
err := proto.Unmarshal(buf, msg)
if err != nil {
@ -320,6 +315,25 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
*mt = s.decodeRowMatrix(msg)
return nil
case *pilosa.ResizeNodeMessage:
msg := &internal.ResizeNodeMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling ResizeNodeMessage")
}
decodeResizeNodeMessage(msg, mt)
return nil
case *pilosa.ResizeAbortMessage:
msg := &internal.ResizeAbortMessage{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling ResizeAbortMessage")
}
decodeResizeAbortMessage(msg, mt)
return nil
default:
panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
}
@ -349,19 +363,17 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeResizeInstruction(mt)
case *pilosa.ResizeInstructionComplete:
return s.encodeResizeInstructionComplete(mt)
case *pilosa.SetCoordinatorMessage:
return s.encodeSetCoordinatorMessage(mt)
case *pilosa.UpdateCoordinatorMessage:
return s.encodeUpdateCoordinatorMessage(mt)
case *pilosa.NodeStateMessage:
return s.encodeNodeStateMessage(mt)
case *pilosa.RecalculateCaches:
return s.encodeRecalculateCaches(mt)
case *pilosa.LoadSchemaMessage:
return s.encodeLoadSchemaMessage(mt)
case *pilosa.NodeEvent:
return s.encodeNodeEventMessage(mt)
case *pilosa.NodeStatus:
return s.encodeNodeStatus(mt)
case *pilosa.Node:
case *topology.Node:
return s.encodeNode(mt)
case *pilosa.QueryRequest:
return s.encodeQueryRequest(mt)
@ -393,6 +405,10 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeTransactionMessage(mt)
case *pilosa.AtomicRecord:
return s.encodeAtomicRecord(mt)
case *pilosa.ResizeNodeMessage:
return s.encodeResizeNodeMessage(mt)
case *pilosa.ResizeAbortMessage:
return s.encodeResizeAbortMessage(mt)
}
return nil
}
@ -573,7 +589,7 @@ func (s Serializer) encodeResizeInstruction(m *pilosa.ResizeInstruction) *intern
return &internal.ResizeInstruction{
JobID: m.JobID,
Node: s.encodeNode(m.Node),
Coordinator: s.encodeNode(m.Coordinator),
Primary: s.encodeNode(m.Primary),
Sources: s.encodeResizeSources(m.Sources),
TranslationSources: s.encodeTranslationResizeSources(m.TranslationSources),
NodeStatus: s.encodeNodeStatus(m.NodeStatus),
@ -680,7 +696,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *internal.FieldOp
}
// s.encodeNodes converts a slice of Nodes into its internal representation.
func (s Serializer) encodeNodes(a []*pilosa.Node) []*internal.Node {
func (s Serializer) encodeNodes(a []*topology.Node) []*internal.Node {
other := make([]*internal.Node, len(a))
for i := range a {
other[i] = s.encodeNode(a[i])
@ -689,17 +705,17 @@ func (s Serializer) encodeNodes(a []*pilosa.Node) []*internal.Node {
}
// s.encodeNode converts a Node into its internal representation.
func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node {
func (s Serializer) encodeNode(m *topology.Node) *internal.Node {
n := m.Clone()
return &internal.Node{
ID: n.ID,
URI: s.encodeURI(n.URI),
IsCoordinator: n.IsCoordinator,
State: n.State,
GRPCURI: s.encodeURI(n.GRPCURI),
ID: n.ID,
URI: s.encodeURI(n.URI),
State: string(n.State),
GRPCURI: s.encodeURI(n.GRPCURI),
}
}
func (s Serializer) encodeURI(u pilosa.URI) *internal.URI {
func (s Serializer) encodeURI(u pnet.URI) *internal.URI {
return &internal.URI{
Scheme: u.Scheme,
Host: u.Host,
@ -728,7 +744,7 @@ func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *inte
return &internal.CreateIndexMessage{
Index: m.Index,
CreatedAt: m.CreatedAt,
Meta: s.encodeIndexMeta(m.Meta),
Meta: s.encodeIndexMeta(&m.Meta),
}
}
@ -793,18 +809,6 @@ func (s Serializer) encodeResizeInstructionComplete(m *pilosa.ResizeInstructionC
}
}
func (s Serializer) encodeSetCoordinatorMessage(m *pilosa.SetCoordinatorMessage) *internal.SetCoordinatorMessage {
return &internal.SetCoordinatorMessage{
New: s.encodeNode(m.New),
}
}
func (s Serializer) encodeUpdateCoordinatorMessage(m *pilosa.UpdateCoordinatorMessage) *internal.UpdateCoordinatorMessage {
return &internal.UpdateCoordinatorMessage{
New: s.encodeNode(m.New),
}
}
func (s Serializer) encodeNodeStateMessage(m *pilosa.NodeStateMessage) *internal.NodeStateMessage {
return &internal.NodeStateMessage{
NodeID: m.NodeID,
@ -863,6 +867,10 @@ func (s Serializer) encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal
return &internal.RecalculateCaches{}
}
func (s Serializer) encodeLoadSchemaMessage(*pilosa.LoadSchemaMessage) *internal.LoadSchemaMessage {
return &internal.LoadSchemaMessage{}
}
func (s Serializer) encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest {
return &internal.TranslateKeysRequest{
Index: request.Index,
@ -949,10 +957,10 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *inter
func (s Serializer) decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) {
m.JobID = ri.JobID
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
s.decodeNode(ri.Node, m.Node)
m.Coordinator = &pilosa.Node{}
s.decodeNode(ri.Coordinator, m.Coordinator)
m.Primary = &topology.Node{}
s.decodeNode(ri.Primary, m.Primary)
m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources))
s.decodeResizeSources(ri.Sources, m.Sources)
m.TranslationSources = make([]*pilosa.TranslationResizeSource, len(ri.TranslationSources))
@ -971,7 +979,7 @@ func (s Serializer) decodeResizeSources(srcs []*internal.ResizeSource, m []*pilo
}
func (s Serializer) decodeResizeSource(rs *internal.ResizeSource, m *pilosa.ResizeSource) {
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
s.decodeNode(rs.Node, m.Node)
m.Index = rs.Index
m.Field = rs.Field
@ -987,7 +995,7 @@ func (s Serializer) decodeTranslationResizeSources(srcs []*internal.TranslationR
}
func (s Serializer) decodeTranslationResizeSource(rs *internal.TranslationResizeSource, m *pilosa.TranslationResizeSource) {
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
s.decodeNode(rs.Node, m.Node)
m.Index = rs.Index
m.PartitionID = int(rs.PartitionID)
@ -1040,7 +1048,7 @@ func (s Serializer) decodeFieldOptions(options *internal.FieldOptions, m *pilosa
s.decodeDecimal(options.Max, &m.Max)
m.Base = options.Base
m.Scale = options.Scale
m.BitDepth = uint(options.BitDepth)
m.BitDepth = uint64(options.BitDepth)
m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum)
m.Keys = options.Keys
m.ForeignIndex = options.ForeignIndex
@ -1051,9 +1059,9 @@ func (s Serializer) decodeDecimal(d *internal.Decimal, m *pql.Decimal) {
m.Scale = d.Scale
}
func (s Serializer) decodeNodes(a []*internal.Node, m []*pilosa.Node) {
func (s Serializer) decodeNodes(a []*internal.Node, m []*topology.Node) {
for i := range a {
m[i] = &pilosa.Node{}
m[i] = &topology.Node{}
s.decodeNode(a[i], m[i])
}
}
@ -1061,21 +1069,20 @@ func (s Serializer) decodeNodes(a []*internal.Node, m []*pilosa.Node) {
func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.ClusterStatus) {
m.State = cs.State
m.ClusterID = cs.ClusterID
m.Nodes = make([]*pilosa.Node, len(cs.Nodes))
m.Nodes = make([]*topology.Node, len(cs.Nodes))
s.decodeNodes(cs.Nodes, m.Nodes)
m.Schema = &pilosa.Schema{}
s.decodeSchema(cs.Schema, m.Schema)
}
func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) {
func (s Serializer) decodeNode(node *internal.Node, m *topology.Node) {
m.ID = node.ID
s.decodeURI(node.URI, &m.URI)
s.decodeURI(node.GRPCURI, &m.GRPCURI)
m.IsCoordinator = node.IsCoordinator
m.State = node.State
m.State = disco.NodeState(node.State)
}
func (s Serializer) decodeURI(i *internal.URI, m *pilosa.URI) {
func (s Serializer) decodeURI(i *internal.URI, m *pnet.URI) {
m.Scheme = i.Scheme
m.Host = i.Host
m.Port = uint16(i.Port)
@ -1090,8 +1097,8 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m
func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) {
m.Index = pb.Index
m.CreatedAt = pb.CreatedAt
m.Meta = &pilosa.IndexOptions{}
s.decodeIndexMeta(pb.Meta, m.Meta)
m.Meta = pilosa.IndexOptions{}
s.decodeIndexMeta(pb.Meta, &m.Meta)
}
func (s Serializer) decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) {
@ -1138,21 +1145,11 @@ func (s Serializer) decodeDeleteViewMessage(pb *internal.DeleteViewMessage, m *p
func (s Serializer) decodeResizeInstructionComplete(pb *internal.ResizeInstructionComplete, m *pilosa.ResizeInstructionComplete) {
m.JobID = pb.JobID
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
s.decodeNode(pb.Node, m.Node)
m.Error = pb.Error
}
func (s Serializer) decodeSetCoordinatorMessage(pb *internal.SetCoordinatorMessage, m *pilosa.SetCoordinatorMessage) {
m.New = &pilosa.Node{}
s.decodeNode(pb.New, m.New)
}
func (s Serializer) decodeUpdateCoordinatorMessage(pb *internal.UpdateCoordinatorMessage, m *pilosa.UpdateCoordinatorMessage) {
m.New = &pilosa.Node{}
s.decodeNode(pb.New, m.New)
}
func (s Serializer) decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pilosa.NodeStateMessage) {
m.NodeID = pb.NodeID
m.State = pb.State
@ -1160,12 +1157,12 @@ func (s Serializer) decodeNodeStateMessage(pb *internal.NodeStateMessage, m *pil
func (s Serializer) decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) {
m.Event = pilosa.NodeEventType(pb.Event)
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
s.decodeNode(pb.Node, m.Node)
}
func (s Serializer) decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
m.Node = &pilosa.Node{}
m.Node = &topology.Node{}
m.Indexes = s.decodeIndexStatuses(pb.Indexes)
m.Schema = &pilosa.Schema{}
s.decodeSchema(pb.Schema, m.Schema)
@ -1204,6 +1201,9 @@ func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldS
func (s Serializer) decodeRecalculateCaches(pb *internal.RecalculateCaches, m *pilosa.RecalculateCaches) {
}
func (s Serializer) decodeLoadSchemaMessage(pb *internal.LoadSchemaMessage, m *pilosa.LoadSchemaMessage) {
}
func (s Serializer) decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.QueryRequest) {
m.Query = pb.Query
m.Shards = pb.Shards
@ -1945,3 +1945,23 @@ func (s Serializer) encodeAttr(key string, value interface{}) *internal.Attr {
}
return pb
}
func (s Serializer) encodeResizeNodeMessage(m *pilosa.ResizeNodeMessage) *internal.ResizeNodeMessage {
return &internal.ResizeNodeMessage{
NodeID: m.NodeID,
Action: m.Action,
}
}
func (s Serializer) encodeResizeAbortMessage(*pilosa.ResizeAbortMessage) *internal.ResizeAbortMessage {
return &internal.ResizeAbortMessage{}
}
func decodeResizeNodeMessage(pb *internal.ResizeNodeMessage, m *pilosa.ResizeNodeMessage) {
m.NodeID = pb.NodeID
m.Action = pb.Action
}
func decodeResizeAbortMessage(pb *internal.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) {
}

57
etcd/cache.go Normal file
View file

@ -0,0 +1,57 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package etcd
import (
"sync"
"time"
"github.com/pilosa/pilosa/v2/topology"
)
// EtcdWithCache is a wrapper around the Etcd type which will return a
// cached value when the number of requests come in below a configured
// frequency. It also breaks the cache after a configured TTL.
type EtcdWithCache struct {
*Etcd
peersMu sync.Mutex // peer-list cache updates
nodes []*topology.Node // unmarshalled Node data
nodesTTL int // seconds
nodesLastRequest time.Time // last time requested
}
// NewEtcdWithCache returns a new instance of Cache.
func NewEtcdWithCache(opt Options, replicas int) *EtcdWithCache {
return &EtcdWithCache{
Etcd: NewEtcd(opt, replicas),
nodesTTL: 6,
}
}
// Nodes caches the result of the underlying implementation's node list.
func (c *EtcdWithCache) Nodes() []*topology.Node {
c.peersMu.Lock()
defer c.peersMu.Unlock()
now := time.Now()
if now.Sub(c.nodesLastRequest) > (time.Duration(c.nodesTTL) * time.Second) {
c.nodes = c.Etcd.Nodes()
c.nodesLastRequest = now
}
return c.nodes
}

1151
etcd/embed.go Normal file
View file

File diff suppressed because it is too large Load diff

4
event.go
View file

@ -14,6 +14,8 @@
package pilosa
import "github.com/pilosa/pilosa/v2/topology"
// NodeEventType are the types of node events.
type NodeEventType int
@ -27,5 +29,5 @@ const (
// NodeEvent is a single event related to node activity in the cluster.
type NodeEvent struct {
Event NodeEventType
Node *Node
Node *topology.Node
}

359
executor.go
View file

@ -26,11 +26,15 @@ import (
"time"
"unsafe"
"golang.org/x/sync/errgroup"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/pql"
pb "github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
)
@ -45,6 +49,8 @@ const (
columnLabel = "col"
rowLabel = "row"
errConnectionRefused = "connect: connection refused"
)
// executor recursively executes calls in a PQL query across all shards.
@ -52,7 +58,7 @@ type executor struct {
Holder *Holder
// Local hostname & cluster configuration.
Node *Node
Node *topology.Node
Cluster *cluster
// Client used for remote requests.
@ -134,6 +140,13 @@ func newExecutor(opts ...executorOption) *executor {
func (e *executor) Close() error {
e.workMu.Lock()
defer e.workMu.Unlock()
if e.shutdown {
// otherwise close(e.work) can result in
// panic: close of closed channel.
// We don't comprehend: why we are called 2x though(?)
// But pilosa/server TestClusteringNodesReplica2 did.
return nil
}
e.shutdown = true
_ = testhook.Closed(NewAuditor(), e, nil)
close(e.work)
@ -817,6 +830,9 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
case "Limit":
res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executeLimitCall %v", shardSlice(shards))
case "Percentile":
res, err := e.executePercentile(ctx, qcx, index, c, shards, opt)
return res, errors.Wrapf(err, "executePercentile %v", shardSlice(shards))
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
@ -1280,6 +1296,119 @@ func (e *executor) executeMax(ctx context.Context, qcx *Qcx, index string, c *pq
return other, nil
}
// executePercentile executes a Percentile() call.
func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (_ ValCount, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePercentile")
defer span.Finish()
// get nth
var nth float64
if nthArg, ok := c.Args["nth"].(pql.Decimal); ok {
nth = nthArg.Float64()
if nth < 0 || nth > 1.0 {
return ValCount{}, errors.Errorf("Percentile(): invalid nth value(%f), should be >= 0 and <= 1.0", nth)
}
} else {
return ValCount{}, errors.New("Percentile(): nth required")
}
// get field
if fieldArg := c.Args["field"]; fieldArg == "" {
return ValCount{}, errors.New("Percentile(): field required")
}
fieldName, _, _ := c.StringArg("field")
// filter call for min & max
var filterCall *pql.Call
// check if filter provided
if filterArg, ok := c.Args["filter"].(*pql.Call); ok && filterArg != nil {
filterCall = filterArg
}
// get min
q, _ := pql.ParseString(fmt.Sprintf(`Min(field="%s")`, fieldName))
minCall := q.Calls[0]
if filterCall != nil {
minCall.Children = append(minCall.Children, filterCall)
}
minVal, err := e.executeMin(ctx, qcx, index, minCall, shards, opt)
if err != nil {
return ValCount{}, errors.Wrap(err, "executing Min call for Percentile")
}
if nth == 0.0 {
return ValCount{Val: minVal.Val, Count: minVal.Count}, nil
}
// get max
q, _ = pql.ParseString(fmt.Sprintf(`Max(field="%s")`, fieldName))
maxCall := q.Calls[0]
if filterCall != nil {
maxCall.Children = append(maxCall.Children, filterCall)
}
maxVal, err := e.executeMax(ctx, qcx, index, maxCall, shards, opt)
if err != nil {
return ValCount{}, errors.Wrap(err, "executing Max call for Percentile")
}
// set up reusables
var countCall, rangeCall *pql.Call
if filterCall == nil {
countQuery, _ := pql.ParseString(fmt.Sprintf("Count(Row(%s < 0))", fieldName))
countCall = countQuery.Calls[0]
rangeCall = countCall.Children[0]
} else {
countQuery, _ := pql.ParseString(fmt.Sprintf(`Count(Intersect(Row(%s < 0)))`, fieldName))
countCall = countQuery.Calls[0]
intersectCall := countCall.Children[0]
intersectCall.Children = append(intersectCall.Children, filterCall)
rangeCall = intersectCall.Children[0]
}
k := (1 - nth) / nth
min, max := minVal.Val, maxVal.Val
// estimate nth val, eg median when nth=0.5
for min < max {
possibleNthVal := (max + min) / 2
// get left count
rangeCall.Args[fieldName] = &pql.Condition{
Op: pql.Token(pql.LT),
Value: possibleNthVal,
}
leftCountUint64, err := e.executeCount(ctx, qcx, index, countCall, shards, opt)
if err != nil {
return ValCount{}, errors.Wrap(err, "executing Count call L for Percentile")
}
leftCount := int64(leftCountUint64)
// get right count
rangeCall.Args[fieldName] = &pql.Condition{
Op: pql.Token(pql.GT),
Value: possibleNthVal,
}
rightCountUint64, err := e.executeCount(ctx, qcx, index, countCall, shards, opt)
if err != nil {
return ValCount{}, errors.Wrap(err, "executing Count call R for Percentile")
}
rightCount := int64(rightCountUint64)
// 'weight' the left count as per k
leftCountWeighted := int64(math.Round(k * float64(leftCount)))
// binary search
if leftCountWeighted > rightCount {
max = possibleNthVal - 1
} else if leftCountWeighted < rightCount {
min = possibleNthVal + 1
} else {
return ValCount{Val: possibleNthVal, Count: 1}, nil
}
}
return ValCount{Val: min, Count: 1}, nil
}
// executeMinRow executes a MinRow() call.
func (e *executor) executeMinRow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (_ interface{}, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinRow")
@ -3495,7 +3624,6 @@ func (e *executor) executeGroupByShard(ctx context.Context, qcx *Qcx, index stri
}
func (e *executor) executeRows(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) {
// Fetch field name from argument.
// Check "field" first for backwards compatibility.
// TODO: remove at Pilosa 2.0
@ -4080,7 +4208,7 @@ func (e *executor) executeExtractShard(ctx context.Context, qcx *Qcx, index stri
mergeBits(sign, 1<<63, data)
// Copy in the significand.
for i := uint(0); i < bsig.BitDepth; i++ {
for i := uint64(0); i < bsig.BitDepth; i++ {
bits, err := fragment.row(tx, bsiOffsetBit+uint64(i))
if err != nil {
return ExtractedIDMatrix{}, errors.Wrap(err, "loading BSI significand bit from fragment")
@ -4768,8 +4896,11 @@ func (e *executor) executeClearBitField(ctx context.Context, qcx *Qcx, index str
shard := colID / ShardWidth
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(e.Cluster.noder, e.Cluster.Hasher, e.Cluster.ReplicaN)
ret := false
for _, node := range e.Cluster.shardNodes(index, shard) {
for _, node := range snap.ShardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
@ -5058,7 +5189,7 @@ func (e *executor) executeSet(ctx context.Context, qcx *Qcx, index string, c *pq
// Set column on existence field.
if ef := idx.existenceField(); ef != nil {
// we create tx here, rather than just above, to avoid creating an extra empty shard.
tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Shard: shard})
tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Field: ef, Shard: shard})
if err != nil {
return false, err
}
@ -5126,7 +5257,10 @@ func (e *executor) executeSetBitField(ctx context.Context, qcx *Qcx, index strin
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.shardNodes(index, shard) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(e.Cluster.noder, e.Cluster.Hasher, e.Cluster.ReplicaN)
for _, node := range snap.ShardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
@ -5169,7 +5303,10 @@ func (e *executor) executeSetValueField(ctx context.Context, qcx *Qcx, index str
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.shardNodes(index, shard) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(e.Cluster.noder, e.Cluster.Hasher, e.Cluster.ReplicaN)
for _, node := range snap.ShardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
@ -5213,10 +5350,12 @@ func (e *executor) executeClearValueField(ctx context.Context, qcx *Qcx, index s
shard := colID / ShardWidth
ret := false
for _, node := range e.Cluster.shardNodes(index, shard) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(e.Cluster.noder, e.Cluster.Hasher, e.Cluster.ReplicaN)
for _, node := range snap.ShardNodes(index, shard) {
// Update locally if host matches.
if node.ID == e.Node.ID {
idx := e.Holder.Index(index)
tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Shard: shard})
if err != nil {
@ -5288,10 +5427,10 @@ func (e *executor) executeSetRowAttrs(ctx context.Context, qcx *Qcx, index strin
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID)
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
@ -5400,10 +5539,10 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, qcx *Qcx, index s
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID)
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil)
resp <- err
}(node)
@ -5452,10 +5591,10 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, qcx *Qcx, index st
}
// Execute on remote nodes in parallel.
nodes := Nodes(e.Cluster.nodes).FilterID(e.Node.ID)
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
@ -5472,7 +5611,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, qcx *Qcx, index st
}
// remoteExec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
func (e *executor) remoteExec(ctx context.Context, node *topology.Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
defer span.Finish()
@ -5494,13 +5633,22 @@ func (e *executor) remoteExec(ctx context.Context, node *Node, index string, q *
// shardsByNode returns a mapping of nodes to shards.
// Returns errShardUnavailable if a shard cannot be allocated to a node.
func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) {
m := make(map[*Node][]uint64)
func (e *executor) shardsByNode(nodes []*topology.Node, index string, shards []uint64) (map[*topology.Node][]uint64, error) {
m := make(map[*topology.Node][]uint64)
// Create a snapshot of the cluster to use for node/partition calculations.
// We use e.Cluster.Nodes() here instead of e.Cluster.noder because we need
// the node states in order to ensure that we don't include an unavailable
// node in the map of nodes to which we distribute the query.
snap := topology.NewClusterSnapshot(topology.NewLocalNoder(e.Cluster.Nodes()), e.Cluster.Hasher, e.Cluster.ReplicaN)
loop:
for _, shard := range shards {
for _, node := range e.Cluster.ShardNodes(index, shard) {
if Nodes(nodes).Contains(node) {
for _, node := range snap.ShardNodes(index, shard) {
// If the node being considered is in any state other than STARTED,
// then exclude it from the map. This way, one of that node's
// healthy replicas will be included instead.
if topology.Nodes(nodes).ContainsID(node.ID) && node.State == disco.NodeStateStarted {
m[node] = append(m[node], shard)
continue loop
}
@ -5514,7 +5662,11 @@ loop:
//
// If a mapping of shards to a node fails then the shards are resplit across
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) (_ interface{}, err error) {
//
// mapReduce has to ensure that it never returns before any work it spawned has
// terminated. It's not enough to cancel the jobs; we have to wait for them to be
// done, or we can unmap resources they're still using.
func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) (result interface{}, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapReduce")
defer span.Finish()
@ -5522,62 +5674,82 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
// Wrap context with a cancel to kill goroutines on exit.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// Create an errgroup so we can wait for all the goroutines to exit
eg, ctx := errgroup.WithContext(ctx)
// After we're done processing, we have to wait for any outstanding
// functions in the ErrGroup to complete. If we didn't have an error
// already at that point, we'll report any errors from the ErrGroup
// instead.
defer func() {
cancel()
errWait := eg.Wait()
if err == nil {
err = errWait
}
}()
// If this is the coordinating node then start with all nodes in the cluster.
//
// However, if this request is being sent from the coordinator then all
// However, if this request is being sent from the primary then all
// processing should be done locally so we start with just the local node.
var nodes []*Node
var nodes []*topology.Node
if !opt.Remote {
nodes = Nodes(e.Cluster.nodes).Clone()
nodes = topology.Nodes(e.Cluster.Nodes()).Clone()
} else {
nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)}
nodes = []*topology.Node{e.Cluster.nodeByID(e.Node.ID)}
}
// Start mapping across all primary owners.
if err := e.mapper(ctx, cancel, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil {
if err = e.mapper(ctx, eg, ch, nodes, index, shards, c, opt, e.Cluster.ReplicaN == 1, mapFn, reduceFn); err != nil {
return nil, errors.Wrap(err, "starting mapper")
}
// Iterate over all map responses and reduce.
var result interface{}
var shardN int
for {
expected := len(shards)
done := ctx.Done()
for expected > 0 {
select {
case <-ctx.Done():
return nil, errors.Wrap(ctx.Err(), "context done")
case <-done:
return nil, ctx.Err()
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
if resp.err != nil {
// We distinguish here between an error which indicates that the
// node is not available (and therefore we need to failover to a
// replica) and a valid error from a healthy node. In the case of
// the latter, there's no need to retry a replica, we should trust
// the error from the healthy node and return that immediately.
if resp.err != nil && strings.Contains(resp.err.Error(), errConnectionRefused) {
// Filter out unavailable nodes.
nodes = Nodes(nodes).Filter(resp.node)
nodes = topology.Nodes(nodes).FilterID(resp.node.ID)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, cancel, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
if err := e.mapper(ctx, eg, ch, nodes, index, resp.shards, c, opt, true, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, errors.Wrap(err, "calling mapper")
return nil, errors.Wrap(err, "mapping on secondary node")
}
continue
} else if resp.err != nil {
return nil, errors.Wrap(resp.err, "mapping on primary node")
}
// if we got a response that we aren't discarding
// because it's an error, subtract it from our count...
expected -= len(resp.shards)
// Reduce value.
result = reduceFn(ctx, result, resp.result)
if err, ok := result.(error); ok {
var ok bool
// note *not* shadowed.
if err, ok = result.(error); ok {
cancel()
return nil, err
}
// If all shards have been processed then return.
shardN += len(resp.shards)
if shardN >= len(shards) {
return result, nil
}
}
}
// note the deferred Wait above which might override this nil.
return result, nil
}
// makeEmbeddedDataForShards produces new rows containing the rowSegments
@ -5624,20 +5796,22 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
return newRows
}
func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
func (e *executor) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, nodes []*topology.Node, index string, shards []uint64, c *pql.Call, opt *execOptions, lastAttempt bool, mapFn mapFunc, reduceFn reduceFunc) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
defer span.Finish()
done := ctx.Done()
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
if err != nil {
return errors.Wrapf(err, "shards by node %v", shardSlice(shards))
}
done := ctx.Done()
// Execute each node in a separate goroutine.
for n, nodeShards := range m {
go func(n *Node, nodeShards []uint64) {
n := n
nodeShards := nodeShards
eg.Go(func() error {
resp := mapResponse{node: n, shards: nodeShards}
// Send local shards to mapper, otherwise remote exec.
@ -5657,17 +5831,29 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha
// Return response to the channel.
select {
case <-done:
// If someone just canceled the context
// arbitrarily, we could end up here with this
// being the first non-nil error handed to
// the ErrGroup, in which case, it's the best
// explanation we have for why everything's
// stopping.
return ctx.Err()
case ch <- resp:
// The cancel coming after the above send is intentional.
// We want to report the actual error that happened
// before we cause anything to return "context canceled".
if resp.err != nil {
cancel()
// If we return a non-nil error from this, the
// entire errGroup gets canceled. So we don't
// want to return a non-nil error if mapReduce
// might try to run another mapper against a
// different set of nodes. Note that this shouldn't
// matter; we just sent the error to mapReduce
// anyway, so it probably cancels the ErrGroup
// too.
if resp.err != nil && lastAttempt {
return resp.err
}
}
}(n, nodeShards)
return nil
})
}
return nil
}
@ -5680,12 +5866,15 @@ type job struct {
func worker(work chan job) {
for j := range work {
result, err := j.mapFn(j.ctx, j.shard)
select {
case <-j.ctx.Done():
case j.resultChan <- mapResponse{result: result, err: err}:
// Skip out early if the context is done, but still send
// an ack so mapperLocal can be sure we aren't about to
// work on something it sent us.
if err := j.ctx.Err(); err != nil {
j.resultChan <- mapResponse{result: nil, err: err}
continue
}
result, err := j.mapFn(j.ctx, j.shard)
j.resultChan <- mapResponse{result: result, err: err}
}
}
@ -5707,39 +5896,45 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
ch := make(chan mapResponse, len(shards))
expected := 0
for _, shard := range shards {
e.work <- job{
j := job{
shard: shard,
mapFn: mapFn,
ctx: ctx,
resultChan: ch,
}
}
// Reduce results
var maxShard int
var result interface{}
for {
select {
case <-done:
return nil, ctx.Err()
case resp := <-ch:
if resp.err != nil {
return nil, resp.err
}
result = reduceFn(ctx, result, resp.result)
if err, ok := result.(error); ok {
cancel()
return nil, err
}
maxShard++
}
// Exit once all shards are processed.
if maxShard == len(shards) {
return result, nil
break
case e.work <- j:
expected++
}
}
// we *absolutely must* get responses for everything we successfully
// transmitted to the work queue, or there could be ongoing access to
// the parent Qcx's stuff.
// Reduce results
var result interface{}
for expected > 0 {
resp := <-ch
expected--
if resp.err != nil && err == nil {
err = resp.err
}
if resp.err == nil && ctx.Err() == nil {
// Only useful to do a possibly-expensive
// reduce if we don't already know we don't
// need it.
result = reduceFn(ctx, result, resp.result)
if resultErr, ok := result.(error); ok {
cancel()
err = resultErr
}
}
}
return result, err
}
func (e *executor) preTranslate(ctx context.Context, index string, calls ...*pql.Call) (cols map[string]map[string]uint64, rows map[string]map[string]map[string]uint64, err error) {
@ -7028,7 +7223,7 @@ type mapFunc func(ctx context.Context, shard uint64) (_ interface{}, err error)
type reduceFunc func(ctx context.Context, prev, v interface{}) interface{}
type mapResponse struct {
node *Node
node *topology.Node
shards []uint64
result interface{}

273
executor_test.go
View file

@ -25,7 +25,7 @@ import (
"io/ioutil"
"math"
"math/rand"
"os"
_ "net/http/pprof"
"reflect"
"sort"
"strconv"
@ -38,10 +38,12 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/test"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pkg/errors"
@ -539,8 +541,8 @@ func TestExecutor_Execute_Count(t *testing.T) {
}
func roaringOnlyTest(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src == pilosa.RoaringTxn || (pilosa.DefaultTxsrc == pilosa.RoaringTxn && src == "") {
src := pilosa.CurrentBackend()
if src == pilosa.RoaringTxn || (storage.DefaultBackend == pilosa.RoaringTxn && src == "") {
// okay to run, we are under roaring only
} else {
t.Skip("skip for everything but roaring")
@ -651,6 +653,7 @@ func TestExecutor_Execute_Set(t *testing.T) {
})
t.Run("ErrInvalidColValueType", func(t *testing.T) {
hldr.SetBit("i", "f", 1, 0) // creates and Commits a Tx internally.
if err := idx.DeleteField("f"); err != nil {
t.Fatal(err)
@ -1344,7 +1347,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN() on integer field: "f"`) {
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: mapping on primary node: cannot compute TopN() on integer field: "f"`) {
t.Fatalf("unexpected error: %v", err)
}
})
@ -1363,7 +1366,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
Set(0, f=1)
`}); err != nil {
t.Fatal(err)
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN(), field has no cache: "f"`) {
} else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: mapping on primary node: cannot compute TopN(), field has no cache: "f"`) {
t.Fatalf("unexpected error: %v", err)
}
})
@ -2958,11 +2961,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr0 := c.GetHolder(0)
hldr1 := c.GetHolder(1)
_, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err := c.GetPrimary().API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -2995,7 +2998,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("remote with timestamp", func(t *testing.T) {
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y"))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y"))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3010,7 +3013,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("remote topn", func(t *testing.T) {
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3057,7 +3060,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("remote groupBy", func(t *testing.T) {
if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i",
Query: `GroupBy(Rows(f))`,
}); err != nil {
@ -3073,7 +3076,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("remote groupBy on ints", func(t *testing.T) {
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "fint", pilosa.OptFieldTypeInt(-1000, 1000))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "fint", pilosa.OptFieldTypeInt(-1000, 1000))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3115,7 +3118,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("groupBy on ints with offset regression", func(t *testing.T) {
_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000))
_, err = c.GetPrimary().API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3146,12 +3149,12 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("Row on ints with ASSIGN condition", func(t *testing.T) {
_, err := c.GetNode(0).API.CreateIndex(context.Background(), "intidx", pilosa.IndexOptions{})
_, err := c.GetPrimary().API.CreateIndex(context.Background(), "intidx", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "intidx", "gint", pilosa.OptFieldTypeInt(-1000, 1000))
_, err = c.GetPrimary().API.CreateField(context.Background(), "intidx", "gint", pilosa.OptFieldTypeInt(-1000, 1000))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3181,12 +3184,12 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("Row on decimals with ASSIGN condition", func(t *testing.T) {
_, err := c.GetNode(0).API.CreateIndex(context.Background(), "decidx", pilosa.IndexOptions{})
_, err := c.GetPrimary().API.CreateIndex(context.Background(), "decidx", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "decidx", "fdec", pilosa.OptFieldTypeDecimal(0))
_, err = c.GetPrimary().API.CreateField(context.Background(), "decidx", "fdec", pilosa.OptFieldTypeDecimal(0))
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -3215,19 +3218,19 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
})
t.Run("Row on foreign key with ASSIGN condition", func(t *testing.T) {
_, err := c.GetNode(0).API.CreateIndex(context.Background(), "parent", pilosa.IndexOptions{Keys: true})
_, err := c.GetPrimary().API.CreateIndex(context.Background(), "parent", pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "parent", "general", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
_, err = c.GetPrimary().API.CreateField(context.Background(), "parent", "general", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = c.GetNode(0).API.CreateIndex(context.Background(), "child", pilosa.IndexOptions{Keys: false})
_, err = c.GetPrimary().API.CreateIndex(context.Background(), "child", pilosa.IndexOptions{Keys: false})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = c.GetNode(0).API.CreateField(context.Background(), "child", "parentid",
_, err = c.GetPrimary().API.CreateField(context.Background(), "child", "parentid",
pilosa.OptFieldForeignIndex("parent"),
pilosa.OptFieldTypeInt(-9223372036854775808, 9223372036854775807),
)
@ -3265,7 +3268,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
c := test.MustNewCluster(t, 1)
defer c.Close()
c.GetNode(0).Config.MaxWritesPerRequest = 3
c.GetIdleNode(0).Config.MaxWritesPerRequest = 3
err := c.Start()
if err != nil {
t.Fatal(err)
@ -3527,8 +3530,9 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatal(err)
}
node0 := c.GetNode(0)
// Set bits.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
if _, err := node0.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20),
@ -3536,25 +3540,27 @@ func TestExecutor_Execute_Existence(t *testing.T) {
t.Fatal(err)
}
//index.Dump("after Set 3x")
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
if res, err := node0.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
}
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
if res, err := node0.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
t.Fatal(err)
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after Not: %+v", bits)
}
// Reopen cluster to ensure existence field is reloaded.
if err := c.GetNode(0).Reopen(); err != nil {
if err := node0.Reopen(); err != nil {
t.Fatal(err)
}
if err := node0.AwaitState(disco.ClusterStateNormal, 10*time.Second); err != nil {
t.Fatalf("restarting cluster: %v", err)
}
hldr2 := c.GetHolder(0)
index2 := hldr2.Index("i")
_ = index2
@ -4495,7 +4501,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
func benchmarkExistence(nn bool, b *testing.B) {
c := test.MustNewCluster(b, 1)
var err error
c.GetNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkExistence")
c.GetIdleNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkExistence")
if err != nil {
b.Fatalf("getting temp dir: %v", err)
}
@ -5931,16 +5937,18 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
`)
t.Run("test foreign index with keys", func(t *testing.T) {
// the execututor returns row IDs when the field has keys, so they should be included in the target.
// because the order is determined by the partitioned index key, they seem out of order.
// The execututor returns row IDs when the field has keys, but we
// don't include them because they are not necessary in the result
// comparison. Because of this, we use the CheckGroupByOnKey
// function here to check equality only on the key field.
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "child", RowID: 0, RowKey: "one"}}, Count: 3},
{Group: []pilosa.FieldRow{{Field: "child", RowID: 1, RowKey: "five"}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2},
{Group: []pilosa.FieldRow{{Field: "child", RowKey: "one"}}, Count: 3},
{Group: []pilosa.FieldRow{{Field: "child", RowKey: "three"}}, Count: 2},
{Group: []pilosa.FieldRow{{Field: "child", RowKey: "five"}}, Count: 1},
}
results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupBy(t, expected, results)
results := c.Query(t, "fic", `GroupBy(Rows(child), sort="count desc")`).Results[0].(*pilosa.GroupCounts).Groups()
test.CheckGroupByOnKey(t, expected, results)
})
}
@ -5954,7 +5962,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
func BenchmarkGroupBy(b *testing.B) {
c := test.MustNewCluster(b, 1)
var err error
c.GetNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkGroupBy-")
c.GetIdleNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkGroupBy-")
if err != nil {
b.Fatalf("getting temp dir: %v", err)
}
@ -6738,7 +6746,11 @@ func TestTimelessClearRegression(t *testing.T) {
}
func TestMissingKeyRegression(t *testing.T) {
c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions(pilosa.OptServerTxsrc("roaring"))})
c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions(
pilosa.OptServerStorageConfig(&storage.Config{
Backend: "roaring",
FsyncEnabled: true,
}))})
defer c.Close()
c.CreateField(t, "i", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f", pilosa.OptFieldKeys())
@ -6837,20 +6849,186 @@ func TestMissingKeyRegression(t *testing.T) {
// queries (HTTP, GRPC, Postgres), etc.).
func TestVariousQueries(t *testing.T) {
for _, clusterSize := range []int{1, 3, 4, 7} {
clusterSize := clusterSize
t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) {
t.Parallel()
c := test.MustRunCluster(t, clusterSize)
defer c.Close()
variousQueries(t, clusterSize)
variousQueriesOnTimeFields(t, clusterSize)
variousQueries(t, c)
variousQueriesOnTimeFields(t, c)
variousQueriesOnPercentiles(t, c)
})
}
}
// tests for abbreviating time values in queries
func variousQueriesOnTimeFields(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, clusterSize)
defer c.Close()
func variousQueriesOnPercentiles(t *testing.T, c *test.Cluster) {
// todo, make rand more random, 42 isnt the answer to everything
// however, to make tests reproducible, seed should be printed
// on failure?
r := rand.New(rand.NewSource(42))
// gen Numbers to test percentile query on, shuffle for extra spice
// size should always be greater than 0
type testValue struct {
colKey string
num int64
rowKey string
}
size := 100
testValues := make([]testValue, size)
rowKeys := [2]string{"foo", "bar"}
for i := 0; i < size; i++ {
num := int64(r.Uint32())
// flip coin to negate
if r.Uint64()%2 == 0 {
num = -num
}
testValues[i] = testValue{
colKey: fmt.Sprintf("user%d", i+1),
num: num,
rowKey: rowKeys[r.Uint64()%2], // flip a coin
}
}
// filter out nums that fulfil predicate
var nums []int64
for _, v := range testValues {
if v.rowKey == "foo" {
nums = append(nums, v.num)
}
}
// get min and max for calculating both expected median
// and bounds for bsi field
// get min & max
// helper function for calculating percentiles to
// cross-check with Pilosa's results
getExpectedPercentile := func(nums []int64, nth float64) int64 {
min, max := nums[0], nums[0]
for _, num := range nums {
if num < min {
min = num
}
if num > max {
max = num
}
}
if nth == 0.0 {
return min
}
k := (1 - nth) / nth
possibleNthVal := int64(0)
// bin search
for min < max {
possibleNthVal = (max + min) / 2
leftCount, rightCount := int64(0), int64(0)
for _, num := range nums {
if num < possibleNthVal {
leftCount++
} else if num > possibleNthVal {
rightCount++
}
}
leftCountWeighted := int64(math.Round(k * float64(leftCount)))
if leftCountWeighted > rightCount {
max = possibleNthVal - 1
} else if leftCountWeighted < rightCount {
min = possibleNthVal + 1
} else { // perfectly balanced, as all things should be
return possibleNthVal
}
}
return min
}
// generate numeric entries for index
intEntries := make([]test.IntKey, size)
for i := 0; i < size; i++ {
key := testValues[i].colKey
val := testValues[i].num
intEntries[i] = test.IntKey{Key: key, Val: val}
}
// generate string-set entries for index
var stringEntries [][2]string
for _, v := range testValues {
stringEntries = append(stringEntries,
[2]string{v.rowKey, v.colKey})
}
// get min max for bsi bounds
min, max := testValues[0].num, testValues[0].num
for _, v := range testValues {
if v.num < min {
min = v.num
}
if v.num > max {
max = v.num
}
}
// generic index
c.CreateField(t, "users2", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(min, max))
c.ImportIntKey(t, "users2", "net_worth", intEntries)
c.CreateField(t, "users2", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "val", pilosa.OptFieldKeys())
c.ImportKeyKey(t, "users2", "val", stringEntries)
splitSortBackToCSV := func(csvStr string) string {
ss := strings.Split(csvStr[:len(csvStr)-1], "\n")
sort.Strings(ss)
return strings.Join(ss, "\n") + "\n"
}
type testCase struct {
query string
// qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
csvVerifier string
}
// generate test cases per each nth argument
nths := []float64{0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}
var tests []testCase
for _, nth := range nths {
query := fmt.Sprintf(`Percentile(field="net_worth", filter=Row(val="foo"), nth=%f)`, nth)
expectedPercentile := getExpectedPercentile(nums, nth)
tests = append(tests, testCase{
query: query,
csvVerifier: fmt.Sprintf("%d,1\n", expectedPercentile),
})
}
for i, tst := range tests {
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
// resp := c.Query(t, "users2", tst.query)
tr := c.QueryGRPC(t, "users2", tst.query)
// if tst.qrVerifier != nil {
// tst.qrVerifier(t, resp)
// }
csvString, err := tableResponseToCSVString(tr)
if err != nil {
t.Fatal(err)
}
// verify everything after header
got := splitSortBackToCSV(csvString[strings.Index(csvString, "\n")+1:])
if got != tst.csvVerifier {
t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got)
}
// TODO: add HTTP and Postgres and ability to convert
// those results to CSV to run through CSV verifier
})
}
}
// tests for abbreviating time values in queries
func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) {
ts := func(t time.Time) int64 {
return t.Unix() * 1e+9
}
@ -6973,10 +7151,7 @@ func variousQueriesOnTimeFields(t *testing.T, clusterSize int) {
}
}
func variousQueries(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, clusterSize)
defer c.Close()
func variousQueries(t *testing.T, c *test.Cluster) {
// Create and populate "likenums" similar to "likes", but without keys on the field.
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
c.ImportIDKey(t, "users", "likenums", []test.KeyID{
@ -7113,7 +7288,7 @@ toronto,3
{ // 2019 All, this excludes userC (who likes pangolin & icecream) from the count.
// UserC visited Paris and Toronto in 2019
query: `GroupBy(
Rows(places_visited, from='2019-01-01T00:00', to='2019-12-31T23:59'),
Rows(places_visited, from='2019-01-01T00:00', to='2019-12-31T23:59'),
filter=Not(Intersect(Row(likes='pangolin'), Row(likes='icecream')))
)`,
csvVerifier: `nairobi,1
@ -7123,7 +7298,7 @@ toronto,2
},
{ // After excluding UserC, this gets the sum of the networth of everyone per cities travelled
query: `GroupBy(
Rows(places_visited, from='2019-01-01T00:00', to='2019-12-31T23:59'),
Rows(places_visited, from='2019-01-01T00:00', to='2019-12-31T23:59'),
filter=Not(Intersect(Row(likes='pangolin'), Row(likes='icecream'))),
aggregate=Sum(field=net_worth)
)`,

394
field.go
View file

@ -30,8 +30,7 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
@ -102,19 +101,12 @@ type Field struct {
broadcaster broadcaster
Stats stats.StatsClient
schemator disco.Schemator
serializer Serializer
// Field options.
options FieldOptions
// finalOptions is used with a final call to applyOptions.
// The initial call to applyOptions is made with options
// loaded from the meta file on disk (in the case when
// a field is being re-opened). If the field creator calls
// setOptions before calling Open(), then those options
// will be held in finalOptions, and applied instead of
// those from the meta file.
finalOptions *FieldOptions
bsiGroups []*bsiGroup
// Shards with data on any node in the cluster, according to this node.
@ -367,8 +359,10 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
schemator: disco.NopSchemator,
serializer: NopSerializer,
options: *applyDefaultOptions(&fo),
options: applyDefaultOptions(&fo),
remoteAvailableShards: roaring.NewBitmap(),
@ -507,6 +501,14 @@ func (f *Field) unprotectedSaveAvailableShards() error {
return nil
}
// SetRemoteAvailableShards replaces remoteAvailableShards with the provided
// value.
func (f *Field) SetRemoteAvailableShards(b *roaring.Bitmap) {
f.mu.Lock()
defer f.mu.Unlock()
f.remoteAvailableShards = b
}
// RemoveAvailableShard removes a shard from the bitmap cache.
//
// NOTE: This can be overridden on the next sync so all nodes should be updated.
@ -530,26 +532,6 @@ func (f *Field) Type() string {
return f.options.Type
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Field) SetCacheSize(v uint32) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ignore if no change occurred.
if v == 0 || f.options.CacheSize == v {
return nil
}
// Persist meta data to disk on change.
f.options.CacheSize = v
if err := f.saveMeta(); err != nil {
return errors.Wrap(err, "saving")
}
return nil
}
// CacheSize returns the ranked field cache size.
func (f *Field) CacheSize() uint32 {
f.mu.RLock()
@ -574,24 +556,12 @@ func (f *Field) Open() error {
return errors.Wrap(err, "creating field dir")
}
f.holder.Logger.Debugf("load meta file for index/field: %s/%s", f.index, f.name)
if err := f.loadMeta(); err != nil {
return errors.Wrap(err, "loading meta")
}
f.holder.Logger.Debugf("load available shards for index/field: %s/%s", f.index, f.name)
if err := f.loadAvailableShards(); err != nil {
return errors.Wrap(err, "loading available shards")
}
// If options were provided using setOptions(), then
// use those instead of the options from the meta file.
if f.finalOptions != nil {
f.options = *f.finalOptions
}
// Apply the field options loaded from meta (or set via setOptions()).
// Apply the field options loaded from etcd (or set via setOptions()).
f.holder.Logger.Debugf("apply options for index/field: %s/%s", f.index, f.name)
if err := f.applyOptions(f.options); err != nil {
return errors.Wrap(err, "applying options")
@ -619,6 +589,7 @@ func (f *Field) Open() error {
return errors.Wrap(err, "checking foreign index")
}
}
f.availableShardChan = make(chan []byte)
f.doneChan = make(chan struct{})
f.wg.Add(1)
@ -737,9 +708,30 @@ func (f *Field) ForeignIndex() string {
return f.options.ForeignIndex
}
func (f *Field) bitDepth() (uint64, error) {
var maxBitDepth uint64
view2shards := f.idx.fieldView2shard.getViewsForField(f.name)
for name, shardset := range view2shards {
view := f.view(name)
if view == nil {
continue
}
bd, err := view.bitDepth(shardset.shards())
if err != nil {
return 0, errors.Wrapf(err, "getting view(%s) bit depth", name)
}
if bd > maxBitDepth {
maxBitDepth = bd
}
}
return maxBitDepth, nil
}
// openViews opens and initializes the views inside the field.
func (f *Field) openViews() error {
view2shards := f.idx.fieldView2shard.getViewsForField(f.name)
if view2shards == nil {
// no data
@ -747,29 +739,11 @@ func (f *Field) openViews() error {
}
for name, shardset := range view2shards {
view := f.newView(f.viewPath(name), name)
if err := view.openWithShardSet(shardset); err != nil {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
if f.holder.txf.TxType() == RoaringTxn {
// Automatically upgrade BSI v1 fragments if they exist & reopen view.
if bsig := f.bsiGroup(f.name); bsig != nil {
if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil {
return errors.Wrap(err, "upgrade view bsi v2")
} else if ok {
if err := view.close(); err != nil {
return errors.Wrap(err, "closing upgraded view")
}
view = f.newView(f.viewPath(name), name)
if err := view.openWithShardSet(shardset); err != nil {
return fmt.Errorf("re-opening view: view=%s, err=%s", view.name, err)
}
}
}
}
view.rowAttrStore = f.rowAttrStore
f.holder.Logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name)
f.viewMap[view.name] = view
@ -777,98 +751,9 @@ func (f *Field) openViews() error {
return nil
}
// loadMeta reads meta data for the field, if any.
func (f *Field) loadMeta() error {
var pb internal.FieldOptions
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading meta")
} else {
if err := proto.Unmarshal(buf, &pb); err != nil {
return errors.Wrap(err, "unmarshaling")
}
}
// Since pb.Min and pb.Max were changed to pql.Decimal,
// and since they now have a different protobuf field
// number, an existing meta file may have values in the
// old min/max fields which need to be converted to
// pql.Decimal.
// TODO: we can remove the OldMin/OldMax once we're
// confident no one is still using the older version.
var min pql.Decimal
if pb.Min != nil {
min = pql.NewDecimal(pb.Min.Value, pb.Min.Scale)
} else {
min = pql.NewDecimal(pb.OldMin, pb.Scale)
}
var max pql.Decimal
if pb.Max != nil {
max = pql.NewDecimal(pb.Max.Value, pb.Max.Scale)
} else {
max = pql.NewDecimal(pb.OldMax, pb.Scale)
}
// Initialize "base" to "min" when upgrading from v1 BSI format.
if pb.BitDepth == 0 {
minInt64, maxInt64 := min.ToInt64(0), max.ToInt64(0)
pb.Base = bsiBase(minInt64, maxInt64)
pb.BitDepth = uint64(bitDepthInt64(maxInt64 - minInt64))
if pb.BitDepth == 0 {
pb.BitDepth = 1
}
}
// Copy metadata fields.
f.options.Type = pb.Type
f.options.CacheType = pb.CacheType
f.options.CacheSize = pb.CacheSize
f.options.Min = min
f.options.Max = max
f.options.Base = pb.Base
f.options.Scale = pb.Scale
f.options.BitDepth = uint(pb.BitDepth)
f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum)
f.options.Keys = pb.Keys
f.options.NoStandardView = pb.NoStandardView
f.options.ForeignIndex = pb.ForeignIndex
return nil
}
// saveMeta writes meta data for the field.
func (f *Field) saveMeta() error {
path := filepath.Join(f.path, ".meta")
// Create a temporary file to marshal to.
tempPath := f.path + tempExt
// Marshal metadata.
fo := f.options
buf, err := proto.Marshal(fo.encode())
if err != nil {
return errors.Wrap(err, "marshaling")
}
// Write to meta file.
if err := ioutil.WriteFile(tempPath, buf, 0666); err != nil {
return errors.Wrap(err, "writing meta")
}
// Move temp file to data file location.
if err := os.Rename(tempPath, path); err != nil {
return fmt.Errorf("rename temp: %s", err)
}
return nil
}
// setOptions saves options for final application during Open().
func (f *Field) setOptions(opts *FieldOptions) {
f.finalOptions = applyDefaultOptions(opts)
f.options = applyDefaultOptions(opts)
}
// applyOptions configures the field based on opt.
@ -918,10 +803,7 @@ func (f *Field) applyOptions(opt FieldOptions) error {
Scale: opt.Scale,
BitDepth: opt.BitDepth,
}
// Validate bsiGroup.
if err := bsig.validate(); err != nil {
return err
}
// Validate and create bsiGroup.
if err := f.createBSIGroup(bsig); err != nil {
return errors.Wrap(err, "creating bsigroup")
}
@ -935,11 +817,11 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.BitDepth = 0
f.options.Keys = opt.Keys
f.options.NoStandardView = opt.NoStandardView
// Set the time quantum.
if err := f.setTimeQuantum(opt.TimeQuantum); err != nil {
f.Close()
return errors.Wrap(err, "setting time quantum")
// Validate the time quantum.
if !opt.TimeQuantum.Valid() {
return ErrInvalidTimeQuantum
}
f.options.TimeQuantum = opt.TimeQuantum
f.options.ForeignIndex = opt.ForeignIndex
case FieldTypeBool:
f.options.Type = FieldTypeBool
@ -1032,17 +914,6 @@ func (f *Field) createBSIGroup(bsig *bsiGroup) error {
defer f.mu.Unlock()
// Append bsiGroup.
if err := f.addBSIGroup(bsig); err != nil {
return err
}
if err := f.saveMeta(); err != nil {
return errors.Wrap(err, "saving")
}
return nil
}
// addBSIGroup adds a single bsiGroup to bsiGroups.
func (f *Field) addBSIGroup(bsig *bsiGroup) error {
if err := bsig.validate(); err != nil {
return errors.Wrap(err, "validating bsigroup")
} else if f.hasBSIGroup(bsig.Name) {
@ -1067,27 +938,6 @@ func (f *Field) TimeQuantum() TimeQuantum {
return f.options.TimeQuantum
}
// setTimeQuantum sets the time quantum for the field.
func (f *Field) setTimeQuantum(q TimeQuantum) error {
f.mu.Lock()
defer f.mu.Unlock()
// Validate input.
if !q.Valid() {
return ErrInvalidTimeQuantum
}
// Update value on field.
f.options.TimeQuantum = q
// Persist meta data to disk.
if err := f.saveMeta(); err != nil {
return errors.Wrap(err, "saving meta")
}
return nil
}
// RowTime gets the row at the particular time with the granularity specified by
// the quantum.
func (f *Field) RowTime(tx Tx, rowID uint64, time time.Time, quantum string) (*Row, error) {
@ -1139,19 +989,22 @@ func (f *Field) recalculateCaches() {
// createViewIfNotExists returns the named view, creating it if necessary.
// Additionally, a CreateViewMessage is sent to the cluster.
func (f *Field) createViewIfNotExists(name string) (*view, error) {
view, created, err := f.createViewIfNotExistsBase(name)
cvm := &CreateViewMessage{
Index: f.index,
Field: f.name,
View: name,
}
// call this base method to isolate the mu.Lock and ensure we aren't holding
// the lock while calling SendSync below.
view, created, err := f.createViewIfNotExistsBase(cvm)
if err != nil {
return nil, err
}
if created {
// Broadcast view creation to the cluster.
err = f.broadcaster.SendSync(
&CreateViewMessage{
Index: f.index,
Field: f.name,
View: name,
})
err := f.holder.sendOrSpool(cvm)
if err != nil {
return nil, errors.Wrap(err, "sending CreateView message")
}
@ -1161,15 +1014,32 @@ func (f *Field) createViewIfNotExists(name string) (*view, error) {
}
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
// One purpose of isolating this method from createViewIfNotExists() is that we
// need to enforce the mu.Lock on everything in this method, but we can't be
// holding the lock when broadcasting the CreateViewMessage view
// broadcaster.SendSync(); calling that SendSync() while holding the lock can
// result in a deadlock waiting on the remote node to give up its lock obtained
// by performing the same action. The returned bool indicates whether the view
// was created or not.
func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if view := f.viewMap[name]; view != nil {
// If we already have this view, we can probably assume etcd already
// has it.
if view := f.viewMap[cvm.View]; view != nil {
return view, false, nil
}
view := f.newView(f.viewPath(name), name)
// Create the view in etcd as the system of record.
// Don't persist views related to the existence field.
if f.name != existenceFieldName {
if err := f.persistView(context.Background(), cvm); err != nil {
return nil, false, errors.Wrap(err, "persisting view")
}
}
view := f.newView(f.viewPath(cvm.View), cvm.View)
if err := view.openEmpty(); err != nil {
return nil, false, errors.Wrap(err, "opening view")
@ -1208,6 +1078,11 @@ func (f *Field) deleteView(name string) error {
delete(f.viewMap, name)
// Delete the view from etcd as the system of record.
if err := f.schemator.DeleteView(context.TODO(), f.index, f.name, name); err != nil {
return errors.Wrapf(err, "deleting view from etcd: %s/%s/%s", f.index, f.name, name)
}
return nil
}
@ -1411,22 +1286,16 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err
// Increase bit depth value if the unsigned value is greater.
if requiredBitDepth > bsig.BitDepth {
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
uvalue := uint64(baseValue)
if value < 0 {
uvalue = uint64(-baseValue)
}
bitDepth := bitDepth(uvalue)
bsig.BitDepth = bitDepth
f.options.BitDepth = bitDepth
return f.saveMeta()
}(); err != nil {
return false, errors.Wrap(err, "increasing bsi max")
uvalue := uint64(baseValue)
if value < 0 {
uvalue = uint64(-baseValue)
}
bitDepth := bitDepth(uvalue)
f.mu.Lock()
bsig.BitDepth = bitDepth
f.options.BitDepth = bitDepth
f.mu.Unlock()
}
// Fetch target view.
@ -1727,21 +1596,15 @@ func (f *Field) importValue(qcx *Qcx, columnIDs []uint64, values []int64, option
requiredDepth = v
}
// Increase bit depth if required.
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
bitDepth := bsig.BitDepth
if requiredDepth > bitDepth {
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
return f.saveMeta()
} else {
requiredDepth = bitDepth
}
return nil
}(); err != nil {
return errors.Wrap(err, "increasing bsi bit depth")
f.mu.Lock()
bitDepth := bsig.BitDepth
if requiredDepth > bitDepth {
bsig.BitDepth = requiredDepth
f.options.BitDepth = requiredDepth
} else {
requiredDepth = bitDepth
}
f.mu.Unlock()
// Import into each fragment.
for key, data := range dataByFragment {
@ -1839,9 +1702,9 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
return err
}
var bitDepth uint
var bitDepth uint64
if maxRowID+1 > bsiOffsetBit {
bitDepth = uint(maxRowID + 1 - bsiOffsetBit)
bitDepth = uint64(maxRowID + 1 - bsiOffsetBit)
}
bsig := f.bsiGroup(f.name)
@ -1883,7 +1746,7 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FieldOptions represents options to set when initializing a field.
type FieldOptions struct {
Base int64 `json:"base,omitempty"`
BitDepth uint `json:"bitDepth,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
Min pql.Decimal `json:"min,omitempty"`
Max pql.Decimal `json:"max,omitempty"`
Scale int64 `json:"scale,omitempty"`
@ -1922,38 +1785,16 @@ func newFieldOptions(opts ...FieldOption) (*FieldOptions, error) {
// applyDefaultOptions updates FieldOptions with the default
// values if o does not contain a valid type.
func applyDefaultOptions(o *FieldOptions) *FieldOptions {
func applyDefaultOptions(o *FieldOptions) FieldOptions {
if o == nil {
o = &FieldOptions{}
}
if o.Type == "" {
o.Type = DefaultFieldType
o.CacheType = DefaultCacheType
o.CacheSize = DefaultCacheSize
}
return o
}
// encode converts o into its internal representation.
func (o *FieldOptions) encode() *internal.FieldOptions {
return encodeFieldOptions(o)
}
func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions {
if o == nil {
return nil
}
return &internal.FieldOptions{
Type: o.Type,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
Base: o.Base,
Scale: o.Scale,
BitDepth: uint64(o.BitDepth),
Min: &internal.Decimal{Value: o.Min.Value, Scale: o.Min.Scale},
Max: &internal.Decimal{Value: o.Max.Value, Scale: o.Max.Scale},
TimeQuantum: string(o.TimeQuantum),
Keys: o.Keys,
NoStandardView: o.NoStandardView,
ForeignIndex: o.ForeignIndex,
}
return *o
}
// MarshalJSON marshals FieldOptions to JSON such that
@ -1977,7 +1818,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Type string `json:"type"`
Base int64 `json:"base"`
BitDepth uint `json:"bitDepth"`
BitDepth uint64 `json:"bitDepth"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Keys bool `json:"keys"`
@ -1996,7 +1837,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
Type string `json:"type"`
Base int64 `json:"base"`
Scale int64 `json:"scale"`
BitDepth uint `json:"bitDepth"`
BitDepth uint64 `json:"bitDepth"`
Min pql.Decimal `json:"min"`
Max pql.Decimal `json:"max"`
Keys bool `json:"keys"`
@ -2077,7 +1918,7 @@ type bsiGroup struct {
Max int64 `json:"max,omitempty"`
Base int64 `json:"base,omitempty"`
Scale int64 `json:"scale,omitempty"`
BitDepth uint `json:"bitDepth,omitempty"`
BitDepth uint64 `json:"bitDepth,omitempty"`
}
// baseValue adjusts the value to align with the range for Field for a certain
@ -2177,12 +2018,12 @@ func isValidCacheType(v string) bool {
}
// bitDepth returns the number of bits required to store a value.
func bitDepth(v uint64) uint {
return uint(bits.Len64(v))
func bitDepth(v uint64) uint64 {
return uint64(bits.Len64(v))
}
// bitDepthInt64 returns the required bit depth for abs(v).
func bitDepthInt64(v int64) uint {
func bitDepthInt64(v int64) uint64 {
if v < 0 {
return bitDepth(uint64(-v))
}
@ -2193,3 +2034,16 @@ func bitDepthInt64(v int64) uint {
func FormatQualifiedFieldName(index, field string) string {
return fmt.Sprintf("%s\x00%s\x00", index, field)
}
// persistView stores the view information in etcd.
func (f *Field) persistView(ctx context.Context, cvm *CreateViewMessage) error {
if cvm.Index == "" {
return ErrIndexRequired
} else if cvm.Field == "" {
return ErrFieldRequired
} else if cvm.View == "" {
return ErrViewRequired
}
return f.schemator.CreateView(ctx, cvm.Index, cvm.Field, cvm.View)
}

63
field_internal_test.go
View file

@ -15,6 +15,7 @@
package pilosa
import (
"context"
"fmt"
"math"
"os"
@ -207,7 +208,10 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
if err != nil {
t.Fatal(err)
}
h := NewHolder(path, nil)
cfg := DefaultHolderConfig()
cfg.StorageConfig.Backend = CurrentBackendOrDefault()
h := NewHolder(path, cfg)
panicOn(h.Open())
idx, err := h.CreateIndex("i", IndexOptions{})
@ -247,7 +251,11 @@ func (f *TestField) Reopen() error {
f.parent = nil
return err
}
if err := f.parent.Open(); err != nil {
schema, err := f.parent.Schemator.Schema(context.Background())
if err != nil {
return err
}
if err := f.parent.OpenWithSchema(schema[f.parent.name]); err != nil {
f.parent = nil
return err
}
@ -297,13 +305,11 @@ func TestField_CreateViewIfNotExists(t *testing.T) {
}
func TestField_SetTimeQuantum(t *testing.T) {
f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH")))
defer f.Close()
// Set & retrieve time quantum.
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
// Retrieve time quantum.
if q := f.TimeQuantum(); q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
@ -316,17 +322,13 @@ func TestField_SetTimeQuantum(t *testing.T) {
}
func TestField_RowTime(t *testing.T) {
f := OpenField(t, OptFieldTypeTime(TimeQuantum("")))
f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH")))
defer f.Close()
// Obtain transaction.
tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
defer tx.Rollback()
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
}
f.MustSetBit(tx, 1, 1, time.Date(2010, time.January, 5, 12, 0, 0, 0, time.UTC))
f.MustSetBit(tx, 1, 2, time.Date(2011, time.January, 5, 12, 0, 0, 0, time.UTC))
f.MustSetBit(tx, 1, 3, time.Date(2010, time.February, 5, 12, 0, 0, 0, time.UTC))
@ -552,7 +554,7 @@ func TestField_ApplyOptions(t *testing.T) {
} {
fld := &Field{}
fld.options = *applyDefaultOptions(&FieldOptions{})
fld.options = applyDefaultOptions(&FieldOptions{})
if err := fld.applyOptions(tt.opts); err != nil {
t.Fatal(err)
@ -922,3 +924,38 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
// the test: can we re-open a BSI fragment under Tx store
_ = f.Reopen()
}
// Ensure that an integer field has the same BitDepth after reopening.
func TestField_SaveMeta(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-10, 1000))
defer f.Close()
colID := uint64(1)
val := int64(88)
expBitDepth := uint64(7)
// Obtain transaction.
tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0})
defer tx.Rollback()
if changed, err := f.SetValue(tx, colID, val); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected SetValue to return changed = true")
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
}
if f.options.BitDepth != expBitDepth {
t.Fatalf("expected BitDepth after set to be: %d, got: %d", expBitDepth, f.options.BitDepth)
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
}
if f.options.BitDepth != expBitDepth {
t.Fatalf("expected BitDepth after reopen to be: %d, got: %d", expBitDepth, f.options.BitDepth)
}
}

155
fragment.go
View file

@ -42,11 +42,13 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/logger"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
)
@ -236,6 +238,24 @@ func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragm
// cachePath returns the path to the fragment's cache data.
func (f *fragment) cachePath() string { return f.path() + cacheExt }
func (f *fragment) bitDepth() (uint64, error) {
tx, err := f.holder.BeginTx(false, f.idx, f.shard)
if err != nil {
return 0, errors.Wrapf(err, "beginning new tx(false, %s, %d)", f.index(), f.shard)
}
defer tx.Rollback()
maxRowID, _, err := f.maxRow(tx, nil)
if err != nil {
return 0, errors.Wrapf(err, "getting fragment max row id")
}
if maxRowID+1 > bsiOffsetBit {
return maxRowID + 1 - bsiOffsetBit, nil
}
return 0, nil
}
type FragmentInfo struct {
BitmapInfo roaring.BitmapInfo
BlockChecksums []FragmentBlock `json:"BlockChecksums,omitempty"`
@ -963,7 +983,7 @@ func (f *fragment) bit(tx Tx, rowID, columnID uint64) (bool, error) {
}
// value uses a column of bits to read a multi-bit value.
func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, exists bool, err error) {
func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint64) (value int64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -975,7 +995,7 @@ func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, ex
}
// Compute other bits into a value.
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
if v, err := f.bit(tx, uint64(bsiOffsetBit+i), columnID); err != nil {
return 0, false, errors.Wrapf(err, "getting value bit %d", i)
} else if v {
@ -994,16 +1014,16 @@ func (f *fragment) value(tx Tx, columnID uint64, bitDepth uint) (value int64, ex
}
// clearValue uses a column of bits to clear a multi-bit value.
func (f *fragment) clearValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (f *fragment) clearValue(tx Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
return f.setValueBase(tx, columnID, bitDepth, value, true)
}
// setValue uses a column of bits to set a multi-bit value.
func (f *fragment) setValue(tx Tx, columnID uint64, bitDepth uint, value int64) (changed bool, err error) {
func (f *fragment) setValue(tx Tx, columnID uint64, bitDepth uint64, value int64) (changed bool, err error) {
return f.setValueBase(tx, columnID, bitDepth, value, false)
}
func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) {
func (f *fragment) positionsForValue(columnID uint64, bitDepth uint64, value int64, clear bool, toSet, toClear []uint64) ([]uint64, []uint64, error) {
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
@ -1028,7 +1048,7 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
toSet = append(toSet, bit)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
bit, err := f.pos(uint64(bsiOffsetBit+i), columnID)
if err != nil {
return toSet, toClear, errors.Wrap(err, "getting pos")
@ -1044,7 +1064,7 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
}
// TODO get rid of this and use positionsForValue to generate a single write op, and set that with importPositions.
func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint64, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -1071,7 +1091,7 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value
uvalue = uint64(-value)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
if uvalue&(1<<i) != 0 {
if c, err := f.unprotectedSetBit(tx, uint64(bsiOffsetBit+i), columnID); err != nil {
return err
@ -1123,14 +1143,14 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value
}
// importSetValue is a more efficient SetValue just for imports.
func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint, value int64, clear bool) (changed int, err error) { // nolint: unparam
func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint64, value int64, clear bool) (changed int, err error) { // nolint: unparam
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
uvalue = uint64(-value)
}
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
bit, err := f.pos(uint64(bsiOffsetBit+i), columnID)
if err != nil {
return changed, errors.Wrap(err, "getting pos")
@ -1192,7 +1212,7 @@ func (f *fragment) importSetValue(txb *TxBitmap, columnID uint64, bitDepth uint,
// sum returns the sum of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint64, err error) {
func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count uint64, err error) {
// Compute count based on the existence row.
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1223,7 +1243,7 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint
//
// Execute once for positive numbers and once for negative. Subtract the
// negative sum from the positive sum.
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return sum, count, err
@ -1241,7 +1261,7 @@ func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint) (sum int64, count uint
// min returns the min of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) min(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) {
func (f *fragment) min(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) {
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
return min, count, err
@ -1270,7 +1290,7 @@ func (f *fragment) min(tx Tx, filter *Row, bitDepth uint) (min int64, count uint
}
// minUnsigned the lowest value without considering the sign bit. Filter is required.
func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint) (min int64, count uint64, err error) {
func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint64) (min int64, count uint64, err error) {
for i := int(bitDepth - 1); i >= 0; i-- {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
@ -1292,7 +1312,7 @@ func (f *fragment) minUnsigned(tx Tx, filter *Row, bitDepth uint) (min int64, co
// max returns the max of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *fragment) max(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) {
func (f *fragment) max(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) {
consider, err := f.row(tx, bsiExistsBit)
if err != nil {
return max, count, err
@ -1321,7 +1341,7 @@ func (f *fragment) max(tx Tx, filter *Row, bitDepth uint) (max int64, count uint
}
// maxUnsigned the highest value without considering the sign bit. Filter is required.
func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint) (max int64, count uint64, err error) {
func (f *fragment) maxUnsigned(tx Tx, filter *Row, bitDepth uint64) (max int64, count uint64, err error) {
for i := int(bitDepth - 1); i >= 0; i-- {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
@ -1422,7 +1442,7 @@ func (f *fragment) maxRowID(tx Tx) (_ uint64, err error) {
}
// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeOp(tx Tx, op pql.Token, bitDepth uint64, predicate int64) (*Row, error) {
switch op {
case pql.EQ:
return f.rangeEQ(tx, bitDepth, predicate)
@ -1448,7 +1468,7 @@ func absInt64(v int64) uint64 {
}
}
func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) {
// Start with set of columns with values set.
b, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1456,7 +1476,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
}
upredicate := absInt64(predicate)
if uint(bits.Len64(upredicate)) > bitDepth {
if uint64(bits.Len64(upredicate)) > bitDepth {
// Predicate is out of range.
return NewRow(), nil
}
@ -1490,7 +1510,7 @@ func (f *fragment) rangeEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
return b, nil
}
func (f *fragment) rangeNEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error) {
func (f *fragment) rangeNEQ(tx Tx, bitDepth uint64, predicate int64) (*Row, error) {
// Start with set of columns with values set.
b, err := f.row(tx, bsiExistsBit)
if err != nil {
@ -1509,7 +1529,7 @@ func (f *fragment) rangeNEQ(tx Tx, bitDepth uint, predicate int64) (*Row, error)
return b, nil
}
func (f *fragment) rangeLT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeLT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) {
if predicate == 1 && !allowEquality {
predicate, allowEquality = 0, true
}
@ -1555,9 +1575,9 @@ func (f *fragment) rangeLT(tx Tx, bitDepth uint, predicate int64, allowEquality
}
// rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit.
func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) {
switch {
case uint(bits.Len64(predicate)) > bitDepth:
case uint64(bits.Len64(predicate)) > bitDepth:
fallthrough
case predicate == (1<<bitDepth)-1 && allowEquality:
// This query matches all possible values.
@ -1565,7 +1585,7 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate
case predicate == (1<<bitDepth)-1 && !allowEquality:
// This query matches everything that is not (1<<bitDepth)-1.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return nil, err
@ -1600,7 +1620,7 @@ func (f *fragment) rangeLTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate
return matched, nil
}
func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeGT(tx Tx, bitDepth uint64, predicate int64, allowEquality bool) (*Row, error) {
if predicate == -1 && !allowEquality {
predicate, allowEquality = 0, true
}
@ -1642,7 +1662,7 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
}
}
func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *fragment) rangeGTUnsigned(tx Tx, filter *Row, bitDepth uint64, predicate uint64, allowEquality bool) (*Row, error) {
prep:
switch {
case predicate == 0 && allowEquality:
@ -1651,7 +1671,7 @@ prep:
case predicate == 0 && !allowEquality:
// This query matches everything that is not 0.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
for i := uint64(0); i < bitDepth; i++ {
row, err := f.row(tx, uint64(bsiOffsetBit+i))
if err != nil {
return nil, err
@ -1659,7 +1679,7 @@ prep:
matches = matches.Union(filter.Intersect(row))
}
return matches, nil
case !allowEquality && uint(bits.Len64(predicate)) > bitDepth:
case !allowEquality && uint64(bits.Len64(predicate)) > bitDepth:
// The predicate is bigger than the BSI width, so nothing can be bigger.
return NewRow(), nil
case allowEquality:
@ -1698,7 +1718,7 @@ func (f *fragment) notNull(tx Tx) (*Row, error) {
}
// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *fragment) rangeBetween(tx Tx, bitDepth uint, predicateMin, predicateMax int64) (*Row, error) {
func (f *fragment) rangeBetween(tx Tx, bitDepth uint64, predicateMin, predicateMax int64) (*Row, error) {
b, err := f.row(tx, bsiExistsBit)
if err != nil {
return nil, err
@ -1747,7 +1767,7 @@ func (f *fragment) rangeBetween(tx Tx, bitDepth uint, predicateMin, predicateMax
}
// rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit.
func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint64, predicateMin, predicateMax uint64) (*Row, error) {
switch {
case predicateMax > (1<<bitDepth)-1:
// The upper bound cannot be violated.
@ -1779,11 +1799,11 @@ func (f *fragment) rangeBetweenUnsigned(tx Tx, filter *Row, bitDepth uint, predi
predicateMax &^= equalMask
var err error
remaining, err = f.rangeGTUnsigned(tx, remaining, uint(diffLen), predicateMin, true)
remaining, err = f.rangeGTUnsigned(tx, remaining, uint64(diffLen), predicateMin, true)
if err != nil {
return nil, err
}
remaining, err = f.rangeLTUnsigned(tx, remaining, uint(diffLen), predicateMax, true)
remaining, err = f.rangeLTUnsigned(tx, remaining, uint64(diffLen), predicateMax, true)
if err != nil {
return nil, err
}
@ -2626,7 +2646,7 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64) error {
return errors.Wrap(f.importPositions(tx, toSet, toClear, rowSet), "importing positions")
}
func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error {
// TODO figure out how to avoid re-allocating these each time. Probably
// possible to store them on the fragment with a capacity based on
// MaxOpN. For now, we know that the total number of bits to be
@ -2661,7 +2681,7 @@ func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int
return err
}
rowSet := make(map[uint64]struct{}, bitDepth+1)
for i := uint(0); i < bitDepth+1; i++ {
for i := uint64(0); i < bitDepth+1; i++ {
rowSet[uint64(i)] = struct{}{}
}
err := f.importPositions(tx, toSet, toClear, rowSet)
@ -2678,7 +2698,7 @@ func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int
}
// importValue bulk imports a set of range-encoded values.
func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDepth uint64, clear bool) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -3277,52 +3297,6 @@ func (f *fragment) blockToRoaringData(block int) ([]byte, error) {
})
}
// upgradeRoaringBSIv2 upgrades a fragment that contains old BSI formatting
// to a new BSI format (v2). The new format moves the "exists" bit to the
// beginning & adds a negative sign bit.
func upgradeRoaringBSIv2(f *fragment, bitDepth uint) (string, error) {
// If flag set, already upgraded. Exit.
if f.storage.Flags&roaringFlagBSIv2 == 1 {
return "", nil
}
other := roaring.NewBitmap()
other.Flags = roaringFlagBSIv2
func() {
f.mu.Lock()
defer f.mu.Unlock()
_ = f.storage.ForEach(func(i uint64) error {
rowID, columnID := i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)
if rowID == uint64(bitDepth) {
_, _ = other.Add(pos(bsiExistsBit, columnID)) // move exists bit to beginning
} else {
_, _ = other.Add(pos(rowID+bsiOffsetBit, columnID)) // move other bits up
}
return nil
})
}()
// Create temporary file next to existing file.
newPath := f.path() + ".tmp"
file, err := os.OpenFile(newPath, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
return "", err
}
defer file.Close()
// Write & flush to temporary file.
if _, err := other.WriteTo(file); err != nil {
return "", err
} else if err := file.Sync(); err != nil {
return "", err
} else if err := file.Close(); err != nil {
return "", err
}
return newPath, nil
}
type rowIterator interface {
// TODO(kuba) linter suggests to use io.Seeker
// Seek(offset int64, whence int) (int64, error)
@ -3629,7 +3603,7 @@ func (h *blockHasher) WriteValue(v uint64) {
type fragmentSyncer struct {
Fragment *fragment
Node *Node
Node *topology.Node
Cluster *cluster
// FieldType helps determine which method of syncing to use.
@ -3654,8 +3628,11 @@ func (s *fragmentSyncer) syncFragment() error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "FragmentSyncer.syncFragment")
defer span.Finish()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
// Determine replica set.
nodes := s.Cluster.shardNodes(s.Fragment.index(), s.Fragment.shard)
nodes := snap.ShardNodes(s.Fragment.index(), s.Fragment.shard)
if len(nodes) == 1 {
return nil
}
@ -3771,9 +3748,12 @@ func (s *fragmentSyncer) syncBlockFromPrimary(id int) error {
f := s.Fragment
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
// Determine replica set. Return early if this is not
// the primary node.
nodes := s.Cluster.shardNodes(f.index(), f.shard)
nodes := snap.ShardNodes(f.index(), f.shard)
if s.Node.ID != nodes[0].ID {
f.holder.Logger.Debugf("non-primary replica expecting sync from primary: %s, index=%s, field=%s, shard=%d", nodes[0].ID, f.index(), f.field(), f.shard)
return nil
@ -3820,10 +3800,13 @@ func (s *fragmentSyncer) syncBlock(id int) error {
f := s.Fragment
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
// Read pairs from each remote block.
var uris []*URI
var uris []*pnet.URI
var pairSets []pairSet
for _, node := range s.Cluster.shardNodes(f.index(), f.shard) {
for _, node := range snap.ShardNodes(f.index(), f.shard) {
if s.Node.ID == node.ID {
continue
}

38
fragment_internal_test.go
View file

@ -38,6 +38,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -455,7 +456,7 @@ func TestFragment_SetValue(t *testing.T) {
})
t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool {
if err := quick.Check(func(bitDepth uint64, bitN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
bitN = (bitN % 99) + 1
@ -987,7 +988,7 @@ func TestFragment_Range(t *testing.T) {
// benchmarkSetValues is a helper function to explore, very roughly, the cost
// of setting values.
func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
for i := 0; i < b.N; i++ {
// We're not checking the error because this is a benchmark.
@ -999,7 +1000,7 @@ func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc f
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_SetValue(b *testing.B) {
depths := []uint{4, 8, 16}
depths := []uint64{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f, idx, tx := mustOpenFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0, "none")
@ -1020,7 +1021,7 @@ func BenchmarkFragment_SetValue(b *testing.B) {
// benchmarkImportValues is a helper function to explore, very roughly, the cost
// of setting values using the special setter used for imports.
func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint64, f *fragment, cfunc func(uint64) uint64) {
column := uint64(0)
b.StopTimer()
columns := make([]uint64, b.N)
@ -1039,7 +1040,7 @@ func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfun
// Benchmark performance of setValue for BSI ranges.
func BenchmarkFragment_ImportValue(b *testing.B) {
depths := []uint{4, 8, 16}
depths := []uint64{4, 8, 16}
for _, bitDepth := range depths {
name := fmt.Sprintf("Depth%d", bitDepth)
f, idx, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0)
@ -1719,8 +1720,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
func roaringOnlyTest(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src == RoaringTxn || (DefaultTxsrc == RoaringTxn && src == "") {
src := CurrentBackend()
if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") {
// okay to run, we are under roaring only
} else {
t.Skip("skip for everything but roaring")
@ -1728,8 +1729,8 @@ func roaringOnlyTest(t *testing.T) {
}
func roaringOnlyBenchmark(b *testing.B) {
src := os.Getenv("PILOSA_TXSRC")
if src == RoaringTxn || (DefaultTxsrc == RoaringTxn && src == "") {
src := CurrentBackend()
if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") {
// okay to run, we are under roaring only
} else {
b.Skip("skip for everything but roaring")
@ -3573,7 +3574,7 @@ func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64)
func newTestHolder(tb testing.TB) *Holder {
path, _ := testhook.TempDirInDir(tb, *TempDir, "holder-dir")
h := NewHolder(path, nil)
h := NewHolder(path, mustHolderConfig())
panicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
@ -3584,8 +3585,14 @@ func newTestHolder(tb testing.TB) *Holder {
// fragTestMustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Index {
cim := &CreateIndexMessage{
Index: index,
CreatedAt: 0,
Meta: opt,
}
holder.mu.Lock()
idx, err := holder.createIndex(index, opt)
idx, err := holder.createIndex(cim, false)
holder.mu.Unlock()
panicOn(err)
@ -4398,7 +4405,7 @@ func TestFragmentPositionsForValue(t *testing.T) {
tests := []struct {
columnID uint64
bitDepth uint
bitDepth uint64
value int64
clear bool
toSet []uint64
@ -5253,7 +5260,7 @@ func TestImportMultipleValues(t *testing.T) {
vals []int64
checkCols []uint64
checkVals []int64
depth uint
depth uint64
}{
{
cols: []uint64{0, 0},
@ -5305,7 +5312,7 @@ func TestImportValueRowCache(t *testing.T) {
cols []uint64
vals []int64
checkCols []uint64
depth uint
depth uint64
}
tests := []struct {
tc1 testCase
@ -5529,8 +5536,7 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
}
func notBlueGreenTest(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if strings.Contains(src, "_") {
if strings.Contains(CurrentBackend(), "_") {
t.Skip("skip under blue green")
}
}

9
go.mod
View file

@ -1,6 +1,6 @@
module github.com/pilosa/pilosa/v2
replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210115113447-5d28bda617d2
require (
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
@ -11,17 +11,17 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
github.com/dustin/go-humanize v1.0.0
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.4.2
github.com/google/go-cmp v0.5.2
github.com/google/uuid v1.1.4 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/improbable-eng/grpc-web v0.13.0
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.8.0
@ -47,18 +47,19 @@ require (
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/zeebo/blake3 v0.0.4
go.etcd.io/bbolt v1.3.5
go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20201214095126-aec9a390925b // indirect
golang.org/x/text v0.3.3 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
modernc.org/mathutil v1.0.0
modernc.org/strutil v1.0.0
sigs.k8s.io/yaml v1.2.0 // indirect
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
)

64
go.sum
View file

@ -45,25 +45,39 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I=
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@ -90,6 +104,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -116,10 +132,14 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0=
github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@ -129,11 +149,19 @@ github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6Ylu
github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@ -162,13 +190,18 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10yRKrDHFHOc=
github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@ -177,6 +210,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@ -192,6 +226,8 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
@ -206,23 +242,28 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/molecula/etcd v0.0.0-20210115113447-5d28bda617d2 h1:pkzCVLSrFQGVQv3raVGJw6aJCJdIZC/z59tUsSU1Zws=
github.com/molecula/etcd v0.0.0-20210115113447-5d28bda617d2/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -269,11 +310,13 @@ github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21
github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
@ -282,10 +325,12 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@ -303,6 +348,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@ -310,6 +357,8 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQG
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
@ -323,9 +372,12 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -373,6 +425,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -413,7 +466,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -465,6 +520,7 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@ -479,6 +535,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@ -503,5 +560,8 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03
modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible h1:GWnLrAdetgJM0Co5bwwczO49iFZBSInpyGAT77BP9Y0=
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0=

639
gossip/gossip.go
View file

@ -1,639 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gossip
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/toml"
"github.com/pkg/errors"
)
// Ensure GossipMemberSet implements interfaces.
var _ memberlist.Delegate = &memberSet{}
// memberSet represents a gossip implementation of MemberSet using memberlist.
type memberSet struct {
mu sync.RWMutex
memberlist *memberlist.Memberlist
broadcasts *memberlist.TransmitLimitedQueue
papi *pilosa.API
config *config
Logger logger.Logger
// stdLogger is only used when passed into memberlist library things that take a std library logger rather than an interface.
stdLogger *log.Logger
// logOutput is similar to stdLogger in that it's passed to memberlist things which can't take a pilosa Logger.
logOutput io.Writer
transport *Transport
eventReceiver *eventReceiver
}
// Open implements the MemberSet interface to start network activity.
func (g *memberSet) Open() (err error) {
g.mu.Lock()
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
if err != nil {
return errors.Wrap(err, "creating memberlist")
}
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
g.mu.RLock()
defer g.mu.RUnlock()
return g.memberlist.NumMembers()
},
RetransmitMult: 3,
}
var uris = make([]*pilosa.URI, len(g.config.gossipSeeds))
for i, addr := range g.config.gossipSeeds {
uris[i], err = pilosa.NewURIFromAddress(addr)
if err != nil {
return fmt.Errorf("new uri from address: %s", err)
}
}
var nodes = make([]*pilosa.Node, len(uris))
for i, uri := range uris {
nodes[i] = &pilosa.Node{URI: *uri}
}
g.mu.RLock()
err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings())
g.mu.RUnlock()
if err != nil {
return errors.Wrap(err, "joinWithRetry")
}
return nil
}
// Close attempts to gracefully leave the cluster, and finally calls shutdown
// after (at most) a timeout period.
func (g *memberSet) Close() error {
g.eventReceiver.Close()
leaveErr := g.memberlist.Leave(5 * time.Second)
shutdownErr := g.memberlist.Shutdown()
if leaveErr != nil || shutdownErr != nil {
return fmt.Errorf("leaving: '%v', shutting down: '%v'", leaveErr, shutdownErr)
}
return nil
}
// joinWithRetry wraps the standard memberlist Join function in a retry.
func (g *memberSet) joinWithRetry(hosts []string) error {
err := retry(60, 2*time.Second, func() error {
_, err := g.memberlist.Join(hosts)
return err
})
return err
}
// retry periodically retries function fn a specified number of attempts.
func retry(attempts int, sleep time.Duration, fn func() error) (err error) { // nolint: unparam
for i := 0; ; i++ {
err = fn()
if err == nil {
return
}
if i >= (attempts - 1) {
break
}
time.Sleep(sleep)
log.Println("retrying after error:", err)
}
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
////////////////////////////////////////////////////////////////
type config struct {
gossipSeeds []string
memberlistConfig *memberlist.Config
}
// memberSetOption describes a functional option for GossipMemberSet.
type memberSetOption func(*memberSet) error
// WithTransport is a functional option for providing a transport to NewMemberSet.
func WithTransport(transport *Transport) memberSetOption {
return func(g *memberSet) error {
g.transport = transport
return nil
}
}
// WithLogger is a functional option for providing a Go logger to NewMemberSet.
// If the memberSet's transport is nil, this logger will be used when creating
// one. If WithLogOutput is not used, this logger will be passed to memberlist
// for it to use internally. This logger is not used for logging by code in this
// (gossip) package - for that, use the WithPilosaLogger option.
func WithLogger(logger *log.Logger) memberSetOption {
return func(g *memberSet) error {
g.stdLogger = logger
return nil
}
}
// WithLogOutput allows one to pass a Writer which will in turn be passed to
// memberlist for use in logging.
func WithLogOutput(o io.Writer) memberSetOption {
return func(g *memberSet) error {
g.logOutput = o
return nil
}
}
// WithPilosaLogger allows one to configure a memberSet with a logger of their
// choice which satisfies the pilosa logger interface.
func WithPilosaLogger(l logger.Logger) memberSetOption {
return func(g *memberSet) error {
g.Logger = l
return nil
}
}
// NewMemberSet returns a new instance of GossipMemberSet based on options. The
// logging options which can be passed to NewMemberSet are complicated for
// historical reasons - please pass WithPilosaLogger, and either WithLogOutput
// or WithLogger. If you pass WithLogOutput, be sure to also pass in a Transport
// using WithTransport.
func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) {
host := api.Node().URI.Host
g := &memberSet{
papi: api,
Logger: logger.NopLogger,
}
// options
for _, opt := range options {
if err := opt(g); err != nil {
return nil, errors.Wrap(err, "executing option")
}
}
ger := newEventReceiver(g.Logger, api)
g.eventReceiver = ger
if g.transport == nil {
port, err := strconv.Atoi(cfg.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
}
if g.stdLogger == nil {
if g.logOutput != nil {
g.stdLogger = logger.NewStandardLogger(g.logOutput).Logger()
} else {
g.stdLogger = log.New(os.Stderr, "", log.LstdFlags)
}
}
// Set up the transport.
transport, err := NewTransport(host, port, g.stdLogger)
if err != nil {
return nil, fmt.Errorf("new tranport: %s", err)
}
g.transport = transport
}
port := g.transport.net.GetAutoBindPort()
var gossipKey []byte
var err error
if cfg.Key != "" {
gossipKey, err = ioutil.ReadFile(cfg.Key)
if err != nil {
return nil, fmt.Errorf("reading gossip key: %s", err)
}
}
////////////////////
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.Transport = g.transport.net
conf.Name = api.Node().ID
conf.BindAddr = api.Node().URI.Host
conf.BindPort = port
// AdvertisePort
if cfg.AdvertisePort != "" {
if p, err := strconv.Atoi(cfg.Port); err != nil {
return nil, fmt.Errorf("convert advertise port: %s", err)
} else {
conf.AdvertisePort = p
}
} else {
conf.AdvertisePort = port
}
// AdvertiseHost
if cfg.AdvertiseHost != "" {
conf.AdvertiseAddr = cfg.AdvertiseHost
} else {
conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
}
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
conf.PushPullInterval = time.Duration(cfg.PushPullInterval)
conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout)
conf.ProbeInterval = time.Duration(cfg.ProbeInterval)
conf.GossipNodes = cfg.Nodes
conf.GossipInterval = time.Duration(cfg.Interval)
conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime)
//
conf.Delegate = g
conf.SecretKey = gossipKey
conf.Events = ger
if g.logOutput != nil {
conf.LogOutput = g.logOutput
} else {
conf.Logger = g.stdLogger
}
g.config = &config{
memberlistConfig: conf,
gossipSeeds: cfg.Seeds,
}
return g, nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *memberSet) NodeMeta(limit int) []byte {
buf, err := g.papi.Serializer.Marshal(g.papi.Node())
if err != nil {
g.Logger.Printf("marshal message error: %s", err)
return []byte{}
}
return buf
}
// NotifyMsg implementation of the memberlist.Delegate interface
// called when a user-data message is received.
func (g *memberSet) NotifyMsg(b []byte) {
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(b))
if err != nil {
g.Logger.Printf("cluster message error: %s", err)
}
}
// GetBroadcasts implementation of the memberlist.Delegate interface
// called when user data messages can be broadcast.
func (g *memberSet) GetBroadcasts(overhead, limit int) [][]byte {
return g.broadcasts.GetBroadcasts(overhead, limit)
}
// LocalState implementation of the memberlist.Delegate interface
// sends this Node's state data.
func (g *memberSet) LocalState(join bool) []byte {
m := &pilosa.NodeStatus{
Node: g.papi.Node(),
Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
}
for _, idx := range m.Schema.Indexes {
is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
for _, f := range idx.Fields {
availableShards := roaring.NewBitmap()
if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards(false)
}
fs := &pilosa.FieldStatus{
Name: f.Name,
CreatedAt: f.CreatedAt,
AvailableShards: availableShards,
}
is.Fields = append(is.Fields, fs)
}
m.Indexes = append(m.Indexes, is)
}
// Marshal nodestate data to bytes.
buf, err := pilosa.MarshalInternalMessage(m, g.papi.Serializer)
if err != nil {
g.Logger.Printf("error marshalling nodestate data, err=%s", err)
return []byte{}
}
return buf
}
// MergeRemoteState implementation of the memberlist.Delegate interface
// receive and process the remote side's LocalState.
func (g *memberSet) MergeRemoteState(buf []byte, join bool) {
err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf))
if err != nil {
g.Logger.Printf("merge state error: %s", err)
}
}
// eventReceiver is used to enable an application to receive
// events about joins and leaves over a channel.
//
// Care must be taken that events are processed in a timely manner from
// the channel, since this delegate will block until an event can be sent.
type eventReceiver struct {
ch chan memberlist.NodeEvent
closed chan struct{}
papi *pilosa.API
logger logger.Logger
}
// newEventReceiver returns a new instance of GossipEventReceiver.
func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver {
ger := &eventReceiver{
ch: make(chan memberlist.NodeEvent, 1),
closed: make(chan struct{}),
logger: logger,
papi: papi,
}
go ger.listen()
return ger
}
func (g *eventReceiver) NotifyJoin(n *memberlist.Node) {
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
select {
case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}:
case <-g.closed:
}
}
func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
select {
case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}:
case <-g.closed:
}
}
func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
// copy node to avoid data race
n2 := *n
n2.Meta = make([]byte, len(n.Meta))
copy(n2.Meta, n.Meta)
select {
case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}:
case <-g.closed:
}
}
func (g *eventReceiver) Close() {
close(g.closed)
}
func (g *eventReceiver) listen() {
var nodeEventType pilosa.NodeEventType
for {
var e memberlist.NodeEvent
select {
case <-g.closed:
return
case e = <-g.ch:
}
switch e.Event {
case memberlist.NodeJoin:
nodeEventType = pilosa.NodeJoin
case memberlist.NodeLeave:
nodeEventType = pilosa.NodeLeave
case memberlist.NodeUpdate:
nodeEventType = pilosa.NodeUpdate
default:
continue
}
// Get the node from the event.Node meta data.
var n pilosa.Node
if err := g.papi.Serializer.Unmarshal(e.Node.Meta, &n); err != nil {
panic("failed to unmarshal event node meta into node")
}
ne := &pilosa.NodeEvent{
Event: nodeEventType,
Node: &n,
}
buf, err := pilosa.MarshalInternalMessage(ne, g.papi.Serializer)
if err != nil {
panic(err)
}
if err := g.papi.ClusterMessage(context.Background(), bytes.NewBuffer(buf)); err != nil {
g.logger.Printf("receive event error: %s", err)
}
}
}
// Transport is a gossip transport for binding to a port.
type Transport struct {
//memberlist.Transport
net *memberlist.NetTransport
URI *pilosa.URI
}
// NewTransport returns a NetTransport based on the given host and port.
// It will dynamically bind to a port if port is 0.
// This is useful for test cases where specifying a port is not reasonable.
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.Logger = logger
net, err := newTransport(conf)
if err != nil {
return nil, fmt.Errorf("new transport: %s", err)
}
uri, err := pilosa.NewURIFromHostPort(host, uint16(net.GetAutoBindPort()))
if err != nil {
return nil, fmt.Errorf("new uri from host port: %s", err)
}
return &Transport{
net: net,
URI: uri,
}, nil
}
// newTransport returns a NetTransport based on the memberlist configuration.
// It will dynamically bind to a port if conf.BindPort is 0.
func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
nc := &memberlist.NetTransportConfig{
BindAddrs: []string{conf.BindAddr},
BindPort: conf.BindPort,
Logger: conf.Logger,
}
// See comment below for details about the retry in here.
makeNetRetry := func(limit int) (*memberlist.NetTransport, error) {
var err error
for try := 0; try < limit; try++ {
var nt *memberlist.NetTransport
if nt, err = memberlist.NewNetTransport(nc); err == nil {
return nt, nil
}
if strings.Contains(err.Error(), "address already in use") {
conf.Logger.Printf("[DEBUG] Got bind error: %v", err)
continue
}
}
return nil, fmt.Errorf("failed to obtain an address: %v", err)
}
// The dynamic bind port operation is inherently racy because
// even though we are using the kernel to find a port for us, we
// are attempting to bind multiple protocols (and potentially
// multiple addresses) with the same port number. We build in a
// few retries here since this often gets transient errors in
// busy unit tests.
limit := 1
if conf.BindPort == 0 {
limit = 10
}
nt, err := makeNetRetry(limit)
if err != nil {
return nil, errors.Wrap(err, "could not set up network transport")
}
return nt, nil
}
// Config holds toml-friendly memberlist configuration.
type Config struct {
// Port indicates the port to which pilosa should bind for internal state sharing.
Port string `toml:"port"`
// AdvertiseHost is the hostname or IP other nodes should use to connect to
// this host. If left blank, the value for Host will be used. This is useful
// in some proxy and NAT scenarios.
AdvertiseHost string `toml:"advertise-host"`
// AdvertisePort is the port other nodes will use to connect to this one.
// Behaves like AdvertiseHost.
AdvertisePort string `toml:"advertise-port"`
Seeds []string `toml:"seeds"`
Key string `toml:"key"`
// StreamTimeout is the timeout for establishing a stream connection with
// a remote node for a full state sync, and for stream read and write
// operations. Maps to memberlist TCPTimeout.
StreamTimeout toml.Duration `toml:"stream-timeout"`
// SuspicionMult is the multiplier for determining the time an
// inaccessible node is considered suspect before declaring it dead.
// The actual timeout is calculated using the formula:
//
// SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
//
// This allows the timeout to scale properly with expected propagation
// delay with a larger cluster size. The higher the multiplier, the longer
// an inaccessible node is considered part of the cluster before declaring
// it dead, giving that suspect node more time to refute if it is indeed
// still alive.
SuspicionMult int `toml:"suspicion-mult"`
// PushPullInterval is the interval between complete state syncs.
// Complete state syncs are done with a single node over TCP and are
// quite expensive relative to standard gossiped messages. Setting this
// to zero will disable state push/pull syncs completely.
//
// Setting this interval lower (more frequent) will increase convergence
// speeds across larger clusters at the expense of increased bandwidth
// usage.
PushPullInterval toml.Duration `toml:"push-pull-interval"`
// ProbeInterval and ProbeTimeout are used to configure probing behavior
// for memberlist.
//
// ProbeInterval is the interval between random node probes. Setting
// this lower (more frequent) will cause the memberlist cluster to detect
// failed nodes more quickly at the expense of increased bandwidth usage.
//
// ProbeTimeout is the timeout to wait for an ack from a probed node
// before assuming it is unhealthy. This should be set to 99-percentile
// of RTT (round-trip time) on your network.
ProbeInterval toml.Duration `toml:"probe-interval"`
ProbeTimeout toml.Duration `toml:"probe-timeout"`
// Interval and Nodes are used to configure the gossip
// behavior of memberlist.
//
// Interval is the interval between sending messages that need
// to be gossiped that haven't been able to piggyback on probing messages.
// If this is set to zero, non-piggyback gossip is disabled. By lowering
// this value (more frequent) gossip messages are propagated across
// the cluster more quickly at the expense of increased bandwidth.
//
// Nodes is the number of random nodes to send gossip messages to
// per Interval. Increasing this number causes the gossip messages
// to propagate across the cluster more quickly at the expense of
// increased bandwidth.
//
// ToTheDeadTime is the interval after which a node has died that
// we will still try to gossip to it. This gives it a chance to refute.
Interval toml.Duration `toml:"interval"`
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}
// hostToIP converts host to an IP4 address based on net.LookupIP().
func hostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}

23
handler.go
View file

@ -189,10 +189,10 @@ func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreate
if valueSetCount > 1 {
return errors.Errorf("must pass ints, floats, or strings but not multiple")
}
if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 {
if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
if (ivr.IndexCreatedAt != 0 && ivr.IndexCreatedAt != indexCreatedAt) ||
(ivr.FieldCreatedAt != 0 && ivr.FieldCreatedAt != fieldCreatedAt) {
return ErrPreconditionFailed
}
return nil
}
@ -226,11 +226,11 @@ type ImportRequest struct {
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
if (ir.IndexCreatedAt != 0 && ir.IndexCreatedAt != indexCreatedAt) ||
(ir.FieldCreatedAt != 0 && ir.FieldCreatedAt != fieldCreatedAt) {
return ErrPreconditionFailed
}
return nil
}
@ -254,10 +254,9 @@ type ImportRoaringRequest struct {
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 {
if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
if (irr.IndexCreatedAt != 0 && irr.IndexCreatedAt != indexCreatedAt) ||
(irr.FieldCreatedAt != 0 && irr.FieldCreatedAt != fieldCreatedAt) {
return ErrPreconditionFailed
}
return nil
}

592
holder.go
View file

@ -30,12 +30,14 @@ import (
"syscall"
"time"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/rbf"
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
@ -51,6 +53,9 @@ const (
// existenceFieldName is the name of the internal field used to store existence values.
existenceFieldName = "_exists"
// DefaultDiscoDir is the default data directory used by the disco implementation.
DefaultDiscoDir = ".disco"
)
func init() {
@ -76,6 +81,8 @@ type Holder struct {
opened lockedChan
broadcaster broadcaster
schemator disco.Schemator
serializer Serializer
NewAttrStore func(string) AttrStore
@ -114,7 +121,13 @@ type Holder struct {
// Queue of fields (having a foreign index) which have
// opened before their foreign index has opened.
foreignIndexFields []*Field
foreignIndexFields []*Field
foreignIndexFieldsMu sync.Mutex
// Queue of messages to broadcast in bulk when the cluster comes up.
// This is wrong, but. . . yeah.
startMsgs []Message
startMsgsMu sync.Mutex
// opening is set to true while Holder is opening.
// It's used to determine if foreign index application
@ -145,9 +158,9 @@ type HolderOpts struct {
// about fragments when opening them.
Inspect bool
// Txsrc controls the tx/storage engine we instatiate. Set by
// server.go OptServerTxsrc
Txsrc string
// StorageBackend controls the tx/storage engine we instatiate. Set by
// server.go OptServerStorageConfig
StorageBackend string
// RowcacheOn, if true, turns on the row cache for all storage backends.
RowcacheOn bool
@ -206,30 +219,34 @@ type HolderConfig struct {
OpenTransactionStore OpenTransactionStoreFunc
OpenIDAllocator OpenIDAllocatorFunc
TranslationSyncer TranslationSyncer
Serializer Serializer
Schemator disco.Schemator
CacheFlushInterval time.Duration
StatsClient stats.StatsClient
NewAttrStore func(string) AttrStore
Logger logger.Logger
Txsrc string
RowcacheOn bool
StorageConfig *storage.Config
RBFConfig *rbfcfg.Config
AntiEntropyInterval time.Duration
}
func DefaultHolderConfig() *HolderConfig {
return &HolderConfig{
PartitionN: DefaultPartitionN,
PartitionN: topology.DefaultPartitionN,
OpenTranslateStore: OpenInMemTranslateStore,
OpenTranslateReader: nil,
OpenTransactionStore: OpenInMemTransactionStore,
OpenIDAllocator: func(string) (*idAllocator, error) { return &idAllocator{}, nil },
TranslationSyncer: NopTranslationSyncer,
Serializer: GobSerializer,
Schemator: disco.InMemSchemator,
CacheFlushInterval: defaultCacheFlushInterval,
StatsClient: stats.NopStatsClient,
NewAttrStore: newNopAttrStore,
Logger: logger.NopLogger,
Txsrc: DefaultTxsrc,
StorageConfig: storage.NewDefaultConfig(),
RBFConfig: rbfcfg.NewDefaultConfig(),
}
}
@ -238,14 +255,11 @@ func DefaultHolderConfig() *HolderConfig {
func NewHolder(path string, cfg *HolderConfig) *Holder {
if cfg == nil {
cfg = DefaultHolderConfig()
// still want the PILOSA_TXSRC to override, for tests use.
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc != "" {
_ = MustTxsrcToTxtype(txsrc)
// INVAR: have valid txsrc.
cfg.Txsrc = txsrc
}
} else if cfg.RBFConfig == nil {
}
if cfg.StorageConfig == nil {
cfg.StorageConfig = storage.NewDefaultConfig()
}
if cfg.RBFConfig == nil {
cfg.RBFConfig = rbfcfg.NewDefaultConfig()
}
@ -266,8 +280,10 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
OpenTransactionStore: cfg.OpenTransactionStore,
OpenIDAllocator: cfg.OpenIDAllocator,
translationSyncer: cfg.TranslationSyncer,
serializer: cfg.Serializer,
schemator: cfg.Schemator,
Logger: cfg.Logger,
Opts: HolderOpts{Txsrc: cfg.Txsrc, RowcacheOn: cfg.RowcacheOn},
Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend, RowcacheOn: cfg.RowcacheOn},
SnapshotQueue: defaultSnapshotQueue,
@ -278,9 +294,9 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
indexes: make(map[string]*Index),
}
rbf.SetRowcacheOn(cfg.RowcacheOn)
storage.SetRowCacheOn(cfg.RowcacheOn)
txf, err := NewTxFactory(cfg.Txsrc, path, h)
txf, err := NewTxFactory(cfg.StorageConfig.Backend, path, h)
panicOn(err)
h.txf = txf
h.txf.blueGreenOffIfRunningBlueGreen()
@ -570,12 +586,11 @@ func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo,
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.opening = true
defer func() { h.opening = false }()
if h.txf == nil {
txf, err := NewTxFactory(h.cfg.Txsrc, h.path, h)
txf, err := NewTxFactory(h.cfg.StorageConfig.Backend, h.path, h)
if err != nil {
return errors.Wrap(err, "Holder.Open NewTxFactory()")
}
@ -594,13 +609,6 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "creating directory")
}
// Verify that we are not trying to open with v1 translation data.
if ok, err := h.hasV1TranslateKeysFile(); err != nil {
return errors.Wrap(err, "verify v1 translation file")
} else if !ok {
return ErrCannotOpenV1TranslateFile
}
tstore, err := h.OpenTransactionStore(h.path)
if err != nil {
return errors.Wrap(err, "opening transaction store")
@ -614,6 +622,12 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "opening ID allocator")
}
// Load schema from etcd.
schema, err := h.schemator.Schema(context.Background())
if err != nil {
return errors.Wrap(err, "getting schema")
}
// Open path to read all index directories.
f, err := os.Open(h.path)
if err != nil {
@ -636,6 +650,19 @@ func (h *Holder) Open() error {
continue
}
// Only continue with indexes which are present in schema.
idx, ok := schema[fi.Name()]
if !ok {
continue
}
// decode the CreateIndexMessage from the schema data in order to
// get its metadata, such as CreateAt.
cim, err := decodeCreateIndexMessage(h.serializer, idx.Data)
if err != nil {
return errors.Wrap(err, "decoding create index message")
}
h.Logger.Printf("opening index: %s", filepath.Base(fi.Name()))
index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
@ -646,12 +673,16 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "opening index")
}
if h.isCoordinator() {
index.createdAt = timestamp()
err = index.OpenWithTimestamp()
} else {
err = index.Open()
}
// Since we don't have createAt stored on disk within the data
// directory, we need to populate it from the etcd schema data.
// TODO: we may no longer need the createdAt value stored in memory on
// the index struct; it may only be needed in the schema return value
// from the API, which already comes from etcd. In that case, this logic
// could be removed, and the createdAt on the index struct could be
// removed.
index.createdAt = cim.CreatedAt
err = index.OpenWithSchema(idx)
if err != nil {
_ = h.txf.Close()
if err == ErrName {
@ -693,6 +724,27 @@ func (h *Holder) Open() error {
}
func (h *Holder) sendOrSpool(msg Message) error {
if h.maybeSpool(msg) {
return nil
}
return h.broadcaster.SendSync(msg)
}
func (h *Holder) maybeSpool(msg Message) bool {
h.startMsgsMu.Lock()
defer h.startMsgsMu.Unlock()
if h.startMsgs == nil {
// Startup is done.
return false
}
h.startMsgs = append(h.startMsgs, msg)
return true
}
// Activate runs the background tasks relevant to keeping a holder in a stable
// state, such as scanning it for needed snapshots, or flushing caches. This
// is separate from opening because, while a server would nearly always want
@ -713,6 +765,8 @@ func (h *Holder) Activate() {
func (h *Holder) checkForeignIndex(f *Field) error {
if h.opening {
if fi := h.Index(f.options.ForeignIndex); fi == nil {
h.foreignIndexFieldsMu.Lock()
defer h.foreignIndexFieldsMu.Unlock()
h.foreignIndexFields = append(h.foreignIndexFields, f)
return nil
}
@ -822,17 +876,12 @@ func (h *Holder) HasData() (bool, error) {
continue
}
return true, nil
}
return false, nil
}
// Skip DisCo data directory.
if fi.Name() == DefaultDiscoDir {
continue
}
// hasV1TranslateKeysFile returns true if a v1 translation data file exists on disk.
func (h *Holder) hasV1TranslateKeysFile() (bool, error) {
if _, err := os.Stat(filepath.Join(h.path, ".keys")); os.IsNotExist(err) {
return true, nil
} else if err != nil {
return false, err
}
return false, nil
}
@ -847,32 +896,52 @@ func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap {
}
// Schema returns schema information for all indexes, fields, and views.
// If includeHiddenAndViews=true, include fields beginning with "_",
// as well as view details.
func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo {
func (h *Holder) Schema() ([]*IndexInfo, error) {
return h.schema(context.TODO(), true)
}
// limitedSchema returns schema information for all indexes and fields.
func (h *Holder) limitedSchema() ([]*IndexInfo, error) {
return h.schema(context.TODO(), false)
}
func (h *Holder) schema(ctx context.Context, includeViews bool) ([]*IndexInfo, error) {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{
Name: index.Name(),
CreatedAt: index.CreatedAt(),
Options: index.Options(),
ShardWidth: ShardWidth,
Fields: []*FieldInfo{},
schema, err := h.schemator.Schema(ctx)
if err != nil {
return nil, errors.Wrapf(err, "getting schema via schemator")
}
for _, index := range schema {
cim, err := decodeCreateIndexMessage(h.serializer, index.Data)
if err != nil {
return nil, errors.Wrap(err, "decoding CreateIndexMessage")
}
for _, field := range index.Fields() {
if !includeHiddenAndViews && strings.HasPrefix(field.name, "_") {
di := &IndexInfo{
Name: cim.Index,
CreatedAt: cim.CreatedAt,
Options: cim.Meta,
ShardWidth: ShardWidth,
Fields: make([]*FieldInfo, 0, len(index.Fields)),
}
for fieldName, field := range index.Fields {
if fieldName == existenceFieldName {
continue
}
fi := &FieldInfo{
Name: field.Name(),
CreatedAt: field.CreatedAt(),
Options: field.Options(),
cfm, err := decodeCreateFieldMessage(h.serializer, field.Data)
if err != nil {
return nil, errors.Wrap(err, "decoding CreateFieldMessage")
}
if includeHiddenAndViews {
fi.Views = []*ViewInfo{}
for _, view := range field.views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
fi := &FieldInfo{
Name: cfm.Field,
CreatedAt: cfm.CreatedAt,
Options: *cfm.Meta,
}
if includeViews {
for viewName := range field.Views {
fi.Views = append(fi.Views, &ViewInfo{Name: viewName})
}
sort.Sort(viewInfoSlice(fi.Views))
}
@ -882,34 +951,26 @@ func (h *Holder) Schema(includeHiddenAndViews bool) []*IndexInfo {
a = append(a, di)
}
sort.Sort(indexInfoSlice(a))
return a
return a, nil
}
// applySchema applies an internal Schema to Holder.
func (h *Holder) applySchema(schema *Schema) error {
// Create indexes that don't exist.
// Create indexes.
// We use h.CreateIndex() instead of h.CreateIndexIfNotExists() because we
// want to limit the use of this method for now to only new indexes.
for _, i := range schema.Indexes {
idx, err := h.CreateIndexIfNotExists(i.Name, i.Options)
idx, err := h.CreateIndex(i.Name, i.Options)
if err != nil {
return errors.Wrap(err, "creating index")
}
if i.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = i.CreatedAt
idx.mu.Unlock()
}
// Create fields that don't exist.
for _, f := range i.Fields {
fld, err := idx.createFieldIfNotExists(f.Name, &f.Options)
fld, err := idx.CreateFieldIfNotExistsWithOptions(f.Name, &f.Options)
if err != nil {
return errors.Wrap(err, "creating field")
}
if f.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = f.CreatedAt
fld.mu.Unlock()
}
// Create views that don't exist.
for _, v := range f.Views {
@ -920,33 +981,13 @@ func (h *Holder) applySchema(schema *Schema) error {
}
}
}
return nil
}
func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
for _, ii := range indexes {
idx := h.Index(ii.Name)
if idx == nil {
continue
}
if ii.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = ii.CreatedAt
idx.mu.Unlock()
}
for _, fi := range ii.Fields {
fld := idx.Field(fi.Name)
if fld == nil {
continue
}
if fi.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = fi.CreatedAt
fld.mu.Unlock()
}
}
// Send the load schema message to all nodes.
if err := h.sendOrSpool(&LoadSchemaMessage{}); err != nil {
return errors.Wrap(err, "sending LoadSchemaMessage")
}
return nil
}
// IndexPath returns the path where a given index is stored.
@ -997,7 +1038,93 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
if h.Index(name) != nil {
return nil, newConflictError(ErrIndexExists)
}
return h.createIndex(name, opt)
cim := &CreateIndexMessage{
Index: name,
CreatedAt: timestamp(),
Meta: opt,
}
// Create the index in etcd as the system of record.
if err := h.persistIndex(context.Background(), cim); err != nil {
return nil, errors.Wrap(err, "persisting index")
}
return h.createIndex(cim, false)
}
// LoadSchemaMessage is an internal message used to inform a node to load the
// latest schema from etcd.
type LoadSchemaMessage struct{}
// LoadSchema creates all indexes based on the information stored in schemator.
// It does not return an error if an index already exists. The thinking is that
// this method will load all indexes that don't already exist. We likely want to
// revisit this; for example, we might want to confirm that the createdAt
// timestamps on each of the indexes matches the value in etcd.
func (h *Holder) LoadSchema() error {
h.mu.Lock()
defer h.mu.Unlock()
return h.loadSchema()
}
// LoadIndex creates an index based on the information stored in schemator.
// An error is returned if the index already exists.
func (h *Holder) LoadIndex(name string) (*Index, error) {
h.mu.Lock()
defer h.mu.Unlock()
// Ensure index doesn't already exist.
if h.Index(name) != nil {
return nil, newConflictError(ErrIndexExists)
}
return h.loadIndex(name)
}
// LoadField creates a field based on the information stored in schemator.
// An error is returned if the field already exists.
func (h *Holder) LoadField(index, field string) (*Field, error) {
// Ensure field doesn't already exist.
if h.Field(index, field) != nil {
return nil, newConflictError(ErrFieldExists)
}
h.mu.Lock()
defer h.mu.Unlock()
return h.loadField(index, field)
}
// LoadView creates a view based on the information stored in schemator. Unlike
// index and field, it is not considered an error if the view already exists.
func (h *Holder) LoadView(index, field, view string) (*view, error) {
// If the view already exists, just return with it here.
if v := h.view(index, field, view); v != nil {
return v, nil
}
return h.loadView(index, field, view)
}
// CreateIndexAndBroadcast creates an index locally, then broadcasts the
// creation to other nodes so they can create locally as well. An error is
// returned if the index already exists.
func (h *Holder) CreateIndexAndBroadcast(cim *CreateIndexMessage) (*Index, error) {
h.mu.Lock()
defer h.mu.Unlock()
// Ensure index doesn't already exist.
if h.Index(cim.Index) != nil {
return nil, newConflictError(ErrIndexExists)
}
// Create the index in etcd as the system of record.
if err := h.persistIndex(context.Background(), cim); err != nil {
return nil, errors.Wrap(err, "persisting index")
}
return h.createIndex(cim, true)
}
// CreateIndexIfNotExists returns an index by name.
@ -1006,37 +1133,74 @@ func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index,
h.mu.Lock()
defer h.mu.Unlock()
// Return index if it exists.
cim := &CreateIndexMessage{
Index: name,
CreatedAt: timestamp(),
Meta: opt,
}
// Create the index in etcd as the system of record.
err := h.persistIndex(context.Background(), cim)
if err != nil && errors.Cause(err) != disco.ErrIndexExists {
return nil, errors.Wrap(err, "persisting index")
}
if index := h.Index(name); index != nil {
return index, nil
}
return h.createIndex(name, opt)
// It may happen that index is not in memory, but it's already in etcd,
// then we need to create it locally.
return h.createIndex(cim, false)
}
func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
if name == "" {
// persistIndex stores the index information in etcd.
func (h *Holder) persistIndex(ctx context.Context, cim *CreateIndexMessage) error {
if cim.Index == "" {
return ErrIndexRequired
}
if err := validateName(cim.Index); err != nil {
return errors.Wrap(err, "validating name")
}
if b, err := h.serializer.Marshal(cim); err != nil {
return errors.Wrap(err, "marshaling")
} else if err := h.schemator.CreateIndex(ctx, cim.Index, b); err != nil {
return errors.Wrapf(err, "writing index to disco: %s", cim.Index)
}
return nil
}
func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, error) {
if cim.Index == "" {
return nil, errors.New("index name required")
}
// Otherwise create a new index.
index, err := h.newIndex(h.IndexPath(name), name)
index, err := h.newIndex(h.IndexPath(cim.Index), cim.Index)
if err != nil {
return nil, errors.Wrap(err, "creating")
}
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
index.keys = cim.Meta.Keys
index.trackExistence = cim.Meta.TrackExistence
index.createdAt = cim.CreatedAt
if err = index.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
}
if err = index.saveMeta(); err != nil {
return nil, errors.Wrap(err, "meta")
}
// Update options.
h.addIndex(index)
if broadcast {
// Send the create index message to all nodes.
if err := h.broadcaster.SendSync(cim); err != nil {
return nil, errors.Wrap(err, "sending CreateIndex message")
}
}
// Since this is a new index, we need to kick off
// its translation sync.
if err := h.translationSyncer.Reset(); err != nil {
@ -1046,6 +1210,93 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
return index, nil
}
func (h *Holder) loadSchema() error {
schema, err := h.schemator.Schema(context.TODO())
if err != nil {
return errors.Wrap(err, "getting schema")
}
// TODO: This is kind of inefficient because we're ignoring the index.Data
// and field.Data values, which contains the index and field information,
// and only using the map key to call loadIndex() and loadField(). These
// make another call to schemator to get the same index and field
// information that we already have in the map. It probably makes sense to
// either copy the parts of the loadIndex and loadField methods here (like
// decodeCreateIndexMessage) or split loadIndex and loadField into smaller
// methods that we could reuse here.
for indexName, index := range schema {
_, err := h.loadIndex(indexName)
if err != nil {
return errors.Wrap(err, "loading index")
}
for fieldName, field := range index.Fields {
_, err := h.loadField(indexName, fieldName)
if err != nil {
return errors.Wrap(err, "loading field")
}
for viewName := range field.Views {
_, err := h.loadView(indexName, fieldName, viewName)
if err != nil {
return errors.Wrap(err, "loading view")
}
}
}
}
return nil
}
func (h *Holder) loadIndex(indexName string) (*Index, error) {
b, err := h.schemator.Index(context.TODO(), indexName)
if err != nil {
return nil, errors.Wrapf(err, "getting index: %s", indexName)
}
cim, err := decodeCreateIndexMessage(h.serializer, b)
if err != nil {
return nil, errors.Wrap(err, "decoding CreateIndexMessage")
}
return h.createIndex(cim, false)
}
func (h *Holder) loadField(indexName, fieldName string) (*Field, error) {
b, err := h.schemator.Field(context.TODO(), indexName, fieldName)
if err != nil {
return nil, errors.Wrapf(err, "getting field: %s/%s", indexName, fieldName)
}
// Get index.
idx := h.Index(indexName)
if idx == nil {
return nil, errors.Errorf("local index not found: %s", indexName)
}
cfm, err := decodeCreateFieldMessage(h.serializer, b)
if err != nil {
return nil, errors.Wrap(err, "decoding CreateFieldMessage")
}
return idx.createFieldIfNotExists(cfm)
}
func (h *Holder) loadView(indexName, fieldName, viewName string) (*view, error) {
b, err := h.schemator.View(context.Background(), indexName, fieldName, viewName)
if err != nil {
return nil, errors.Wrapf(err, "getting view: %s/%s/%s", indexName, fieldName, viewName)
} else if !b {
return nil, errors.Wrapf(err, "tried to load a nonexistent view: %s/%s/%s", indexName, fieldName, viewName)
}
// Get field.
fld := h.Field(indexName, fieldName)
if fld == nil {
return nil, errors.Errorf("local field not found: %s/%s", indexName, fieldName)
}
return fld.createViewIfNotExists(viewName)
}
func (h *Holder) newIndex(path, name string) (*Index, error) {
index, err := NewIndex(h, path, name)
if err != nil {
@ -1053,6 +1304,8 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
}
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.broadcaster
index.serializer = h.serializer
index.Schemator = h.schemator
index.newAttrStore = h.NewAttrStore
index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ".data"))
index.OpenTranslateStore = h.OpenTranslateStore
@ -1089,6 +1342,11 @@ func (h *Holder) DeleteIndex(name string) error {
// Remove reference.
h.deleteIndex(name)
// Delete the index from etcd as the system of record.
if err := h.schemator.DeleteIndex(context.TODO(), name); err != nil {
return errors.Wrapf(err, "deleting index from etcd: %s", name)
}
// I'm not sure if calling Reset() here is necessary
// since closing the index stops its translation
// sync processes.
@ -1176,13 +1434,6 @@ func (h *Holder) recalculateCaches() {
}
}
func (h *Holder) isCoordinator() bool {
if s, ok := h.broadcaster.(*Server); ok {
return s.isCoordinator
}
return false
}
// setFileLimit attempts to set the open file limit to the FileLimit constant defined above.
func (h *Holder) setFileLimit() {
oldLimit := &syscall.Rlimit{}
@ -1284,7 +1535,7 @@ type holderSyncer struct {
Holder *Holder
Node *Node
Node *topology.Node
Cluster *cluster
// Translation sync handling.
@ -1317,8 +1568,17 @@ func (s *holderSyncer) SyncHolder() error {
s.mu.Lock() // only allow one instance of SyncHolder to be running at a time
defer s.mu.Unlock()
ti := time.Now()
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
schema, err := s.Holder.Schema()
if err != nil {
return errors.Wrap(err, "getting schema")
}
// Iterate over schema in sorted order.
for _, di := range s.Holder.Schema(true) {
for _, di := range schema {
// Verify syncer has not closed.
if s.IsClosing() {
return nil
@ -1351,7 +1611,7 @@ func (s *holderSyncer) SyncHolder() error {
itr.Seek(0)
for shard, eof := itr.Next(); !eof; shard, eof = itr.Next() {
// Ignore shards that this host doesn't own.
if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) {
if !snap.OwnsShard(s.Node.ID, di.Name, shard) {
continue
}
@ -1396,7 +1656,7 @@ func (s *holderSyncer) syncIndex(index string) error {
s.Stats.CountWithCustomTags(MetricColumnAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
for _, node := range topology.Nodes(s.Cluster.noder.Nodes()).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.ColumnAttrDiff(ctx, &node.URI, index, blks)
@ -1443,7 +1703,7 @@ func (s *holderSyncer) syncField(index, name string) error {
s.Stats.CountWithCustomTags(MetricRowAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag, fieldTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.nodes).FilterID(s.Node.ID) {
for _, node := range topology.Nodes(s.Cluster.noder.Nodes()).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.RowAttrDiff(ctx, &node.URI, index, name, blks)
@ -1513,16 +1773,19 @@ func (s *holderSyncer) resetTranslationSync() error {
return errors.Wrap(err, "stop translation sync")
}
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
// Set read-only flag for all translation stores.
s.setTranslateReadOnlyFlags()
s.setTranslateReadOnlyFlags(snap)
// Connect to each node that has a primary for which we are a replica.
if err := s.initializeIndexTranslateReplication(); err != nil {
if err := s.initializeIndexTranslateReplication(snap); err != nil {
return errors.Wrap(err, "initialize index translate replication")
}
// Connect to coordinator to stream field data.
if err := s.initializeFieldTranslateReplication(); err != nil {
// Connect to primary to stream field data.
if err := s.initializeFieldTranslateReplication(snap); err != nil {
return errors.Wrap(err, "initialize field translate replication")
}
return nil
@ -1592,10 +1855,10 @@ func (s *holderSyncer) stopTranslationSync() error {
// setTranslateReadOnlyFlags updates all translation stores to enable or disable
// writing new translation keys. Index stores are writable if the node owns the
// partition. Field stores are writable if the node is the coordinator.
func (s *holderSyncer) setTranslateReadOnlyFlags() {
// partition. Field stores are writable if the node is the primary.
func (s *holderSyncer) setTranslateReadOnlyFlags(snap *topology.ClusterSnapshot) {
s.Cluster.mu.RLock()
isCoordinator := s.Cluster.unprotectedIsCoordinator()
isPrimaryFieldTranslator := snap.IsPrimaryFieldTranslationNode(s.Cluster.Node.ID)
for _, index := range s.Holder.Indexes() {
// There is a race condition here:
@ -1616,8 +1879,8 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
//
// Update: there was another path down to Index.Close(), so
// we shrink to lock to be inside index.TranslateStore() now.
for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
primary := s.Cluster.unprotectedPrimaryPartitionNode(partitionID)
for partitionID := 0; partitionID < snap.PartitionN; partitionID++ {
primary := snap.PrimaryPartitionNode(partitionID)
isPrimary := primary != nil && s.Node.ID == primary.ID
if ts := index.TranslateStore(partitionID); ts != nil {
@ -1626,7 +1889,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
}
for _, field := range index.Fields() {
field.TranslateStore().SetReadOnly(!isCoordinator)
field.TranslateStore().SetReadOnly(!isPrimaryFieldTranslator)
}
}
s.Cluster.mu.RUnlock()
@ -1634,8 +1897,8 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
// initializeIndexTranslateReplication connects to each node that is the
// primary for a partition that we are a replica of.
func (s *holderSyncer) initializeIndexTranslateReplication() error {
for _, node := range s.Cluster.Nodes() {
func (s *holderSyncer) initializeIndexTranslateReplication(snap *topology.ClusterSnapshot) error {
for _, node := range snap.Nodes {
// Skip local node.
if node.ID == s.Node.ID {
continue
@ -1647,10 +1910,10 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error {
if !index.Keys() {
continue
}
for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
partitionNodes := s.Cluster.partitionNodes(partitionID)
isPrimary := partitionNodes[0].ID == node.ID // remote is primary?
isReplica := Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica?
for partitionID := 0; partitionID < snap.PartitionN; partitionID++ {
partitionNodes := snap.PartitionNodes(partitionID)
isPrimary := partitionNodes[0].ID == node.ID // remote is primary?
isReplica := topology.Nodes(partitionNodes[1:]).ContainsID(s.Node.ID) // local is replica?
if !isPrimary || !isReplica {
continue
}
@ -1686,10 +1949,10 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error {
return nil
}
// initializeFieldTranslateReplication connects the coordinator to stream field data.
func (s *holderSyncer) initializeFieldTranslateReplication() error {
// Skip if coordinator.
if s.Cluster.isCoordinator() {
// initializeFieldTranslateReplication connects the primary to stream field data.
func (s *holderSyncer) initializeFieldTranslateReplication(snap *topology.ClusterSnapshot) error {
// Skip if primary.
if snap.IsPrimaryFieldTranslationNode(s.Cluster.Node.ID) {
return nil
}
@ -1711,9 +1974,9 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error {
return nil
}
// Connect to coordinator and begin streaming.
coordinator := s.Cluster.coordinatorNode()
rd, err := s.Holder.OpenTranslateReader(context.Background(), coordinator.URI.String(), m)
// Connect to primary and begin streaming.
primary := snap.PrimaryFieldTranslationNode()
rd, err := s.Holder.OpenTranslateReader(context.Background(), primary.URI.String(), m)
if err != nil {
return err
}
@ -1728,6 +1991,9 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error {
}
func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(s.Cluster.noder, s.Cluster.Hasher, s.Cluster.ReplicaN)
for {
var entry TranslateEntry
if err := rd.ReadEntry(&entry); err != nil {
@ -1743,7 +2009,7 @@ func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) {
}
// Apply replication to store.
store := idx.TranslateStore(s.Cluster.Topology.KeyPartition(entry.Index, entry.Key))
store := idx.TranslateStore(snap.KeyToKeyPartition(entry.Index, entry.Key))
if err := store.ForceSet(entry.ID, entry.Key); err != nil {
s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key)
return
@ -1777,7 +2043,7 @@ func (s *holderSyncer) readFieldTranslateReader(rd TranslateEntryReader) {
// holderCleaner removes fragments and data files that are no longer used.
type holderCleaner struct {
Node *Node
Node *topology.Node
Holder *Holder
Cluster *cluster
@ -1786,6 +2052,11 @@ type holderCleaner struct {
Closing <-chan struct{}
}
// TODO: this is here to satisfy the linter since holderCleaner was removed from
// the gossip implementation of removeNode. But presumably we will use it once
// we have ported over the etcd implementation.
var _ holderCleaner
// IsClosing returns true if the cleaner has been marked to close.
func (c *holderCleaner) IsClosing() bool {
select {
@ -1799,6 +2070,9 @@ func (c *holderCleaner) IsClosing() bool {
// CleanHolder compares the holder with the cluster state and removes
// any unnecessary fragments and files.
func (c *holderCleaner) CleanHolder() error {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.Cluster.noder, c.Cluster.Hasher, c.Cluster.ReplicaN)
for _, index := range c.Holder.Indexes() {
// Verify cleaner has not closed.
if c.IsClosing() {
@ -1806,7 +2080,7 @@ func (c *holderCleaner) CleanHolder() error {
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedShards := c.Cluster.containsShards(index.Name(), index.AvailableShards(includeRemote), c.Node)
containedShards := snap.ContainsShards(index.Name(), index.AvailableShards(includeRemote), c.Node)
// Get the fragments registered in memory.
for _, field := range index.Fields() {
@ -2030,3 +2304,19 @@ func (h *Holder) HasRoaringData() (has bool, err error) {
}
return
}
func decodeCreateIndexMessage(ser Serializer, b []byte) (*CreateIndexMessage, error) {
var cim CreateIndexMessage
if err := ser.Unmarshal(b, &cim); err != nil {
return nil, errors.Wrap(err, "unmarshaling")
}
return &cim, nil
}
func decodeCreateFieldMessage(ser Serializer, b []byte) (*CreateFieldMessage, error) {
var cfm CreateFieldMessage
if err := ser.Unmarshal(b, &cfm); err != nil {
return nil, errors.Wrap(err, "unmarshaling")
}
return &cfm, nil
}

28
holder_internal_test.go
View file

@ -20,6 +20,7 @@ import (
"os"
"testing"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/testhook"
)
@ -76,12 +77,16 @@ func (t *testHolderOperator) ProcessFragment(*fragment) error {
return nil
}
func makeHolder(tb testing.TB) (*Holder, string, error) {
func makeHolder(tb testing.TB, backend string) (*Holder, string, error) {
path, err := testhook.TempDir(tb, "pilosa-")
if err != nil {
return nil, "", err
}
h := NewHolder(path, nil)
cfg := mustHolderConfig()
if backend != "" {
cfg.StorageConfig.Backend = backend
}
h := NewHolder(path, cfg)
return h, path, h.Open()
}
@ -170,7 +175,7 @@ func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
}
func TestHolderOperatorProcess(t *testing.T) {
h, path, err := makeHolder(t)
h, path, err := makeHolder(t, "")
if err != nil {
t.Fatalf("creating holder: %v", err)
}
@ -200,7 +205,7 @@ func TestHolderOperatorProcess(t *testing.T) {
}
func TestHolderOperatorCancel(t *testing.T) {
h, path, err := makeHolder(t)
h, path, err := makeHolder(t, "")
if err != nil {
t.Fatalf("creating holder: %v", err)
}
@ -247,3 +252,18 @@ func TestHolderOperatorCancel(t *testing.T) {
t.Fatalf("holder processor did not cancel. expected something other than %#v", expected)
}
}
// mustHolderConfig is meant to help minimize the number of places in the code
// where we're reading the PILOSA_STORAGE_BACKEND environment variable for
// testing purposes. Ideally we would handle this differently, but this is a
// first attempt at improving things. Note: the actual os.Getenv() call was
// moved to the CurrentBackend() function.
func mustHolderConfig() *HolderConfig {
cfg := DefaultHolderConfig()
if backend := CurrentBackend(); backend != "" {
_ = MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
cfg.Schemator = disco.InMemSchemator
return cfg
}

88
holder_test.go
View file

@ -15,7 +15,6 @@
package pilosa_test
import (
"bytes"
"context"
"math"
"os"
@ -32,31 +31,8 @@ import (
)
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexName", func(t *testing.T) {
h := test.MustOpenHolder(t)
bufLogger := test.NewBufferLogger()
h.Holder.Logger = bufLogger
defer h.Close()
if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatal(err)
}
if bufbytes, err := bufLogger.ReadAll(); err != nil {
t.Fatal(err)
} else if !bytes.Contains(bufbytes, []byte("ERROR opening index: !")) {
t.Fatalf("expected log error:\n%s", bufbytes)
}
})
t.Run("ErrIndexPermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -75,10 +51,11 @@ func TestHolder_Open(t *testing.T) {
}()
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("unexpected error: %s", err)
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
@ -96,6 +73,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFieldPermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -119,6 +97,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
@ -142,6 +121,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
h := test.MustOpenHolder(t)
defer h.Close()
@ -165,6 +145,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
roaringOnlyTest(t)
if os.Geteuid() == 0 {
@ -202,6 +183,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
t.Skip("we don't open the holder directly from disk anymore; we use the etcd schema")
roaringOnlyTest(t)
h := test.MustOpenHolder(t)
@ -432,11 +414,12 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
c := test.MustNewCluster(t, 2)
c.GetNode(0).Config.Cluster.ReplicaN = 2
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 2
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
@ -543,10 +526,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// the row boundaries of the block.
func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
c := test.MustNewCluster(t, 3)
c.GetNode(0).Config.Cluster.ReplicaN = 3
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 3
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 3
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 3
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(2).Config.Cluster.ReplicaN = 3
c.GetIdleNode(2).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
@ -598,10 +583,12 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
// Ensure holder correctly handles clears during block sync.
func TestHolderSyncer_Clears(t *testing.T) {
c := test.MustNewCluster(t, 3)
c.GetNode(0).Config.Cluster.ReplicaN = 3
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 3
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 3
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 3
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(2).Config.Cluster.ReplicaN = 3
c.GetIdleNode(2).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
@ -647,10 +634,10 @@ func TestHolderSyncer_Clears(t *testing.T) {
// Ensure holder can sync time quantum views with a remote holder.
func TestHolderSyncer_TimeQuantum(t *testing.T) {
c := test.MustNewCluster(t, 2)
c.GetNode(0).Config.Cluster.ReplicaN = 2
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 2
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
@ -700,10 +687,10 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
func TestHolderSyncer_IntField(t *testing.T) {
t.Run("BasicSync", func(t *testing.T) {
c := test.MustNewCluster(t, 2)
c.GetNode(0).Config.Cluster.ReplicaN = 2
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 2
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
@ -711,7 +698,6 @@ func TestHolderSyncer_IntField(t *testing.T) {
defer c.Close()
var idx0 *pilosa.Index
_ = idx0
idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_ = idx0
if err != nil {
@ -758,10 +744,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
t.Run("MultiShard", func(t *testing.T) {
c := test.MustNewCluster(t, 2)
c.GetNode(0).Config.Cluster.ReplicaN = 2
c.GetNode(0).Config.AntiEntropy.Interval = 0
c.GetNode(1).Config.Cluster.ReplicaN = 2
c.GetNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(0).Config.Cluster.ReplicaN = 2
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 2
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)

200
http/client.go
View file

@ -30,13 +30,15 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/encoding/proto"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
)
// InternalClient represents a client to the Pilosa cluster.
type InternalClient struct {
defaultURI *pilosa.URI
defaultURI *pnet.URI
serializer pilosa.Serializer
// The client to use for HTTP communication.
@ -49,7 +51,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
return nil, pilosa.ErrHostRequired
}
uri, err := pilosa.NewURIFromAddress(host)
uri, err := pnet.NewURIFromAddress(host)
if err != nil {
return nil, errors.Wrap(err, "getting URI")
}
@ -58,7 +60,7 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
return client, nil
}
func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient {
func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client) *InternalClient {
return &InternalClient{
defaultURI: defaultURI,
serializer: proto.Serializer{},
@ -102,6 +104,39 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64
return rsp.Standard, nil
}
// SchemaNode returns all index and field schema information from the specified
// node.
func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*pilosa.IndexInfo, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema")
defer span.Finish()
// TODO: /?views parameter will be ignored, till we implement schemator!
// Execute request against the host.
u := uri.Path(fmt.Sprintf("/schema?views=%v", views))
// Build request.
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, err
}
defer resp.Body.Close()
var rsp getSchemaResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
return rsp.Indexes, nil
}
// Schema returns all index and field schema information.
func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Schema")
@ -133,7 +168,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error
return rsp.Indexes, nil
}
func (c *InternalClient) PostSchema(ctx context.Context, uri *pilosa.URI, s *pilosa.Schema, remote bool) error {
func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilosa.Schema, remote bool) error {
u := uri.Path(fmt.Sprintf("/schema?remote=%v", remote))
buf, err := json.Marshal(s)
if err != nil {
@ -165,15 +200,15 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndex")
defer span.Finish()
// Get the coordinator node. Schema changes must go through
// coordinator to avoid weird race conditions.
// Get the primary node. Schema changes must go through
// primary to avoid weird race conditions.
nodes, err := c.Nodes(ctx)
if err != nil {
return fmt.Errorf("getting nodes: %s", err)
}
coord := getCoordinatorNode(nodes)
coord := getPrimaryNode(nodes)
if coord == nil {
return fmt.Errorf("could not find the coordinator node")
return fmt.Errorf("could not find the primary node")
}
// Encode query request.
@ -207,7 +242,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
}
// FragmentNodes returns a list of nodes that own a shard.
func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) {
func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentNodes")
defer span.Finish()
@ -231,7 +266,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
}
defer resp.Body.Close()
var a []*pilosa.Node
var a []*topology.Node
if err := json.NewDecoder(resp.Body).Decode(&a); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
@ -239,7 +274,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard
}
// Nodes returns a list of all nodes.
func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) {
func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Nodes")
defer span.Finish()
@ -262,7 +297,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*pilosa.Node, error) {
}
defer resp.Body.Close()
var a []*pilosa.Node
var a []*topology.Node
if err := json.NewDecoder(resp.Body).Decode(&a); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
@ -277,7 +312,7 @@ func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *
}
// QueryNode executes query against the index, sending the request to the node specified.
func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "QueryNode")
defer span.Finish()
@ -368,9 +403,9 @@ func (c *InternalClient) Import(ctx context.Context, index, field string, shard
return nil
}
func getCoordinatorNode(nodes []*pilosa.Node) *pilosa.Node {
func getPrimaryNode(nodes []*topology.Node) *topology.Node {
for _, node := range nodes {
if node.IsCoordinator {
if node.IsPrimary {
return node
}
}
@ -402,22 +437,22 @@ func (c *InternalClient) ImportK(ctx context.Context, index, field string, bits
return fmt.Errorf("Error Creating Payload: %s", err)
}
// Get the coordinator node; all bits are sent to the
// primary translate store (i.e. coordinator).
// Get the primary node; all bits are sent to the
// primary translate store (i.e. primary).
// TODO... is that right^^?
// RESPONSE: It looks like in ctl/import.go, we could change the
// logic in ImportCommand.importBits() to only use ImportK
// when useRowKeys = true. It's no longer necessary to
// send column key translations to the coordinator (although
// send column key translations to the primary (although
// it should still work). As far as I know, the only thing
// that uses ImportK is the pilosa import sub-command.
nodes, err := c.Nodes(ctx)
if err != nil {
return fmt.Errorf("getting nodes: %s", err)
}
coord := getCoordinatorNode(nodes)
coord := getPrimaryNode(nodes)
if coord == nil {
return fmt.Errorf("could not find the coordinator node")
return fmt.Errorf("could not find the primary node")
}
// Import to node.
@ -482,7 +517,7 @@ func (c *InternalClient) marshalImportPayload(index, field string, shard uint64,
}
// importNode sends a pre-marshaled import request to a node.
func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error {
func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, index, field string, buf []byte, opts *pilosa.ImportOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.importNode")
defer span.Finish()
@ -621,15 +656,15 @@ func (c *InternalClient) ImportValueK(ctx context.Context, index, field string,
}
}
// Get the coordinator node; all bits are sent to the
// primary translate store (i.e. coordinator).
// Get the primary node; all bits are sent to the
// primary translate store.
nodes, err := c.Nodes(ctx)
if err != nil {
return fmt.Errorf("getting nodes: %s", err)
}
coord := getCoordinatorNode(nodes)
coord := getPrimaryNode(nodes)
if coord == nil {
return fmt.Errorf("could not find the coordinator node")
return fmt.Errorf("could not find the primary node")
}
// Import to node.
@ -664,7 +699,7 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui
// ImportRoaring does fast import of raw bits in roaring format (pilosa or
// official format, see API.ImportRoaring).
func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error {
func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
defer span.Finish()
@ -718,7 +753,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind
}
// ImportColumnAttrs does bulk import of column attrs
func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pilosa.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
defer span.Finish()
@ -802,7 +837,7 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sha
}
// exportNode copies a CSV export from a node to w.
func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error {
func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node, index, field string, shard uint64, w io.Writer) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.exportNodeCSV")
defer span.Finish()
@ -840,11 +875,11 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i
// RetrieveShardFromURI returns a ReadCloser which contains the data of the
// specified shard from the specified node. Caller *must* close the returned
// ReadCloser or risk leaking goroutines/tcp connections.
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) {
func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI")
defer span.Finish()
node := &pilosa.Node{
node := &topology.Node{
URI: uri,
}
@ -882,7 +917,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e
return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{})
}
// CreateField creates a new field on the server.
// CreateFieldWithOptions creates a new field on the server.
func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions")
defer span.Finish()
@ -900,20 +935,26 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
// should probably happen in the field anyway??
fieldOpt := fieldOptions{
Type: opt.Type,
Keys: &opt.Keys,
}
if fieldOpt.Type == pilosa.FieldTypeSet {
switch fieldOpt.Type {
case pilosa.FieldTypeSet, pilosa.FieldTypeMutex:
fieldOpt.CacheType = &opt.CacheType
fieldOpt.CacheSize = &opt.CacheSize
} else if fieldOpt.Type == pilosa.FieldTypeInt {
fieldOpt.Keys = &opt.Keys
case pilosa.FieldTypeInt:
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
} else if fieldOpt.Type == pilosa.FieldTypeTime {
case pilosa.FieldTypeTime:
fieldOpt.TimeQuantum = &opt.TimeQuantum
} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
case pilosa.FieldTypeBool:
// pass
case pilosa.FieldTypeDecimal:
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
fieldOpt.Scale = &opt.Scale
default:
fieldOpt.Type = pilosa.DefaultFieldType
fieldOpt.Keys = &opt.Keys
}
// TODO: remove buf completely? (depends on whether importer needs to create specific field types)
@ -925,15 +966,15 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
return errors.Wrap(err, "marshaling")
}
// Get the coordinator node. Schema changes must go through
// coordinator to avoid weird race conditions.
// Get the primary node. Schema changes must go through
// primary to avoid weird race conditions.
nodes, err := c.Nodes(ctx)
if err != nil {
return fmt.Errorf("getting nodes: %s", err)
}
coord := getCoordinatorNode(nodes)
coord := getPrimaryNode(nodes)
if coord == nil {
return fmt.Errorf("could not find the coordinator node")
return fmt.Errorf("could not find the primary node")
}
// Create URL & HTTP request.
@ -961,7 +1002,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) {
func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]pilosa.FragmentBlock, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FragmentBlocks")
defer span.Finish()
@ -1005,7 +1046,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
}
// BlockData returns row/column id pairs for a block.
func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.BlockData")
defer span.Finish()
@ -1054,7 +1095,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
}
// ColumnAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ColumnAttrDiff")
defer span.Finish()
@ -1094,7 +1135,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in
}
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RowAttrDiff")
defer span.Finish()
@ -1137,7 +1178,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
}
// SendMessage posts a message synchronously.
func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg []byte) error {
func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.SendMessage")
defer span.Finish()
@ -1161,9 +1202,9 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [
return errors.Wrap(err, "draining SendMessage response body")
}
// TranslateKeysNode function is mainly called to translate keys from coordinator node.
// If coordinator node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound.
func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
// TranslateKeysNode function is mainly called to translate keys from primary node.
// If primary node returns 404 error the function wraps it with pilosa.ErrTranslatingKeyNotFound.
func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "TranslateKeysNode")
defer span.Finish()
@ -1218,7 +1259,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pilosa.URI,
}
// TranslateIDsNode sends an id translation request to a specific node.
func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI, index, field string, ids []uint64) ([]string, error) {
func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "TranslateIDsNode")
defer span.Finish()
@ -1269,7 +1310,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pilosa.URI,
}
// GetNodeUsage retrieves the size-on-disk information for the specified node.
func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pilosa.URI) (map[string]pilosa.NodeUsage, error) {
func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]pilosa.NodeUsage, error) {
u := uri.Path("/ui/usage?remote=true")
req, err := http.NewRequest("GET", u, nil)
if err != nil {
@ -1300,7 +1341,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pilosa.URI) (map
}
// GetPastQueries retrieves the query history log for the specified node.
func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pilosa.URI) ([]pilosa.PastQueryStatus, error) {
func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]pilosa.PastQueryStatus, error) {
u := uri.Path("/query-history?remote=true")
req, err := http.NewRequest("GET", u, nil)
if err != nil {
@ -1330,7 +1371,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pilosa.URI) ([
return queries, nil
}
func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pilosa.URI, index string, keys ...string) (transMap map[string]uint64, err error) {
func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (transMap map[string]uint64, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FindIndexKeysNode")
defer span.Finish()
@ -1379,7 +1420,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pilosa.URI,
return transMap, nil
}
func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pilosa.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) {
func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FindFieldKeysNode")
defer span.Finish()
@ -1427,7 +1468,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pilosa.URI,
return transMap, nil
}
func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pilosa.URI, index string, keys ...string) (transMap map[string]uint64, err error) {
func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (transMap map[string]uint64, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateIndexKeysNode")
defer span.Finish()
@ -1476,7 +1517,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pilosa.UR
return transMap, nil
}
func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pilosa.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) {
func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (transMap map[string]uint64, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldKeysNode")
defer span.Finish()
@ -1567,7 +1608,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou
// We're using the defaultURI here because this is only used by
// tests, and we want to test requests against all hosts. A robust
// client implementation would ensure that these requests go to
// the coordinator.
// the primary.
u := uriPathToURL(c.defaultURI, "/transaction/"+id)
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
@ -1639,7 +1680,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa
// We're using the defaultURI here because this is only used by
// tests, and we want to test requests against all hosts. A robust
// client implementation would ensure that these requests go to
// the coordinator.
// the primary.
u := uriPathToURL(c.defaultURI, "/transaction/"+id)
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
@ -1922,7 +1963,7 @@ func pos(rowID, columnID uint64) uint64 {
return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth)
}
func uriPathToURL(uri *pilosa.URI, path string) url.URL {
func uriPathToURL(uri *pnet.URI, path string) url.URL {
return url.URL{
Scheme: uri.Scheme,
Host: uri.HostPort(),
@ -1930,7 +1971,7 @@ func uriPathToURL(uri *pilosa.URI, path string) url.URL {
}
}
func nodePathToURL(node *pilosa.Node, path string) url.URL {
func nodePathToURL(node *topology.Node, path string) url.URL {
return url.URL{
Scheme: node.URI.Scheme,
Host: node.URI.HostPort(),
@ -1941,11 +1982,11 @@ func nodePathToURL(node *pilosa.Node, path string) url.URL {
// RetrieveTranslatePartitionFromURI returns a ReadCloser which contains the data of the
// specified translate partition from the specified node. Caller *must* close the returned
// ReadCloser or risk leaking goroutines/tcp connections.
func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pilosa.URI) (io.ReadCloser, error) {
func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveTranslatePartitionFromURI")
defer span.Finish()
node := &pilosa.Node{
node := &topology.Node{
URI: uri,
}
@ -1974,7 +2015,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context,
return resp.Body, nil
}
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys")
defer span.Finish()
@ -2006,7 +2047,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, i
return nil
}
func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pilosa.URI, index, field string, remote bool, rddbdata io.Reader) error {
func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys")
defer span.Finish()
@ -2037,3 +2078,36 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pilosa.URI, i
defer resp.Body.Close()
return nil
}
// Status function is just a public function for this particular implementation of InternalClient.
// It's not require by pilosa.InternalClient interface.
// The function returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...)
func (c *InternalClient) Status(ctx context.Context) (string, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.Status")
defer span.Finish()
// Execute request against the host.
u := c.defaultURI.Path("/status")
// Build request.
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return "", errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return "", err
}
defer resp.Body.Close()
var rsp getStatusResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return "", fmt.Errorf("json decode: %s", err)
}
return rsp.State, nil
}

95
http/client_test.go
View file

@ -33,6 +33,7 @@ import (
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pkg/errors"
)
@ -48,15 +49,16 @@ func TestClient_MultiNode(t *testing.T) {
)
defer c.Close()
hldr := []test.Holder{}
for _, command := range c.Nodes {
hldr = append(hldr, test.Holder{Holder: command.Server.Holder()})
}
hldr0 := c.GetHolder(0)
hldr1 := c.GetHolder(1)
hldr2 := c.GetHolder(2)
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN.
// Create a dispersed set of bitmaps across 3 nodes such that each
// individual node and shard width increment would reveal a different TopN.
shardNums := []uint64{1, 2, 6}
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())`
// This was generated with:
// `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
@ -95,26 +97,26 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
hldr[0].MustSetBits("i", "f", 100, baseBit0+10)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2)
hldr0.MustSetBits("i", "f", 100, baseBit0+10)
hldr0.MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr0.MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr0.MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr0.MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr0.MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2)
hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustSetBits("i", "f", 1, baseBit1+4)
hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr1.MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr1.MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr1.MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr1.MustSetBits("i", "f", 1, baseBit1+4)
hldr1.MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustSetBits("i", "f", 21, baseBit2+10)
hldr[2].MustSetBits("i", "f", 100, baseBit2+10)
hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
hldr2.MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr2.MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr2.MustSetBits("i", "f", 21, baseBit2+10)
hldr2.MustSetBits("i", "f", 100, baseBit2+10)
hldr2.MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr2.MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
hldr2.MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
@ -297,7 +299,7 @@ func TestClient_Export(t *testing.T) {
bw := bufio.NewWriter(buf)
// Send export request for every partition.
for i := 0; i < pilosa.DefaultPartitionN; i++ {
for i := 0; i < topology.DefaultPartitionN; i++ {
if err := c.ExportCSV(context.Background(), "keyed", "unkeyedf", uint64(i), bw); err != nil {
t.Fatal(err)
}
@ -338,7 +340,7 @@ func TestClient_Export(t *testing.T) {
bw := bufio.NewWriter(buf)
// Send export request.
for i := 0; i < pilosa.DefaultPartitionN; i++ {
for i := 0; i < topology.DefaultPartitionN; i++ {
if err := c.ExportCSV(context.Background(), "keyed", "keyedf", uint64(i), bw); err != nil {
t.Fatal(err)
}
@ -468,17 +470,15 @@ func TestClient_ImportColumnAttrs(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_ImportRoaring(t *testing.T) {
cluster := test.MustNewCluster(t, 2)
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 2
}
err := cluster.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
cluster := test.MustRunCluster(t, 2,
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerReplicaN(2))},
[]server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerReplicaN(2))},
)
defer cluster.Close()
_, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
@ -764,7 +764,7 @@ func TestClient_ImportKeys(t *testing.T) {
}
})
// Import to node1 (ensure import is routed to coordinator for translation).
// Import to node1 (ensure import is routed to primary for translation).
t.Run("Import node1", func(t *testing.T) {
if err := c1.ImportK(context.Background(), "keyed", "keyedf1", []pilosa.Bit{
{RowKey: "green", ColumnKey: "eve"},
@ -1226,8 +1226,11 @@ func TestClientTransactions(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
client0 := MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
client1 := MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
coord := c.GetPrimary()
other := c.GetNonPrimary()
client0 := MustNewClient(coord.URL(), http.GetHTTPClient(nil))
client1 := MustNewClient(other.URL(), http.GetHTTPClient(nil))
// can create, list, get, and finish a transaction
var expDeadline time.Time
@ -1354,10 +1357,10 @@ func TestClientTransactions(t *testing.T) {
trns)
}
// non-coordinator
// non-primary
if trns, err := client1.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
!strings.Contains(err.Error(), pilosa.ErrNodeNotCoordinator.Error()) {
t.Fatalf("unexpected error starting on non-coordinator: %v", err)
!strings.Contains(err.Error(), pilosa.ErrNodeNotPrimary.Error()) {
t.Fatalf("unexpected error starting on non-primary: %v", err)
} else {
test.CompareTransactions(t,
nil,
@ -1423,17 +1426,17 @@ func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pi
}
}
// verify that serverInfo has TxSrc
func TestClient_ServerInfoHasTxSrc(t *testing.T) {
// verify that serverInfo has Backend
func TestClient_ServerInfoHasBackend(t *testing.T) {
//srcs := []string{"roaring", "rbf", "lmdb"}
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
si := cmd.API.Info()
if si.TxSrc == "" {
panic("should have gotten a TxSrc back")
if si.StorageBackend == "" {
panic("should have gotten a StorageBackend back")
}
pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid
pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid
}
func TestClient_ImportRoaringExists(t *testing.T) {
cluster := test.MustNewCluster(t, 1)

141
http/handler.go
View file

@ -44,6 +44,7 @@ import (
"github.com/pilosa/pilosa/v2/encoding/proto"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -66,6 +67,8 @@ type Handler struct {
api *pilosa.API
ln net.Listener
// url is used to hold the advertise bind address for printing a log during startup.
url string
closeTimeout time.Duration
@ -134,9 +137,13 @@ func OptHandlerLogger(logger logger.Logger) handlerOption {
}
}
func OptHandlerListener(ln net.Listener) handlerOption {
// OptHandlerListener set the listener that will be used by the HTTP server.
// Url must be the advertised URL. It will be used to show a log to the user
// about where the Web UI is. This option is mandatory.
func OptHandlerListener(ln net.Listener, url string) handlerOption {
return func(h *Handler) error {
h.ln = ln
h.url = url
return nil
}
}
@ -217,7 +224,6 @@ func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired()
h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired()
h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired()
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard")
h.validators["GetIndexes"] = queryValidationSpecRequired()
h.validators["GetIndex"] = queryValidationSpecRequired()
@ -233,7 +239,7 @@ func (h *Handler) populateValidators() {
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns", "profile")
h.validators["GetInfo"] = queryValidationSpecRequired()
h.validators["RecalculateCaches"] = queryValidationSpecRequired()
h.validators["GetSchema"] = queryValidationSpecRequired()
h.validators["GetSchema"] = queryValidationSpecRequired().Optional("views")
h.validators["PostSchema"] = queryValidationSpecRequired().Optional("remote")
h.validators["GetStatus"] = queryValidationSpecRequired()
h.validators["GetVersion"] = queryValidationSpecRequired()
@ -366,7 +372,6 @@ func newRouter(handler *Handler) http.Handler {
router := mux.NewRouter()
router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.Handle("/metrics", promhttp.Handler())
@ -448,7 +453,7 @@ func newRouter(handler *Handler) http.Handler {
// Endpoints to support lattice UI embedded via statik.
// The messiness here reflects the fact that assets live in a nontrivial
// directory structure that is controlled externally.
latticeHandler := NewStatikHandler(handler)
latticeHandler := newStatikHandler(handler)
router.PathPrefix("/static").Handler(latticeHandler)
router.Path("/").Handler(latticeHandler)
router.Path("/favicon.png").Handler(latticeHandler)
@ -499,11 +504,11 @@ type statikHandler struct {
statikFS http.FileSystem
}
// NewStatikHandler returns a new instance of statikHandler
func NewStatikHandler(h *Handler) statikHandler {
// newStatikHandler returns a new instance of statikHandler
func newStatikHandler(h *Handler) statikHandler {
fs, err := h.fileSystem.New()
if err == nil {
h.logger.Printf("enabled Web UI (%s) at %s", h.api.LatticeVersion(), h.api.Node().URI)
h.logger.Printf("enabled Web UI (%s) at %s", h.api.LatticeVersion(), h.url)
}
return statikHandler{
@ -666,8 +671,15 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
return
}
q := r.URL.Query()
withViews := q.Get("views") == "true"
w.Header().Set("Content-Type", "application/json")
schema := h.api.Schema(r.Context())
schema, err := h.api.Schema(r.Context(), withViews)
if err != nil {
h.logger.Printf("getting schema error: %s", err)
}
if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
h.logger.Printf("write schema response error: %s", err)
}
@ -752,8 +764,15 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
state, err := h.api.State()
if err != nil {
http.Error(w, "getting cluster state error: "+err.Error(), http.StatusInternalServerError)
return
}
status := getStatusResponse{
State: h.api.State(),
State: string(state),
Nodes: h.api.Hosts(r.Context()),
LocalID: h.api.Node().ID,
ClusterName: h.api.ClusterName(),
@ -812,10 +831,10 @@ type getSchemaResponse struct {
}
type getStatusResponse struct {
State string `json:"state"`
Nodes []*pilosa.Node `json:"nodes"`
LocalID string `json:"localID"`
ClusterName string `json:"clusterName"`
State string `json:"state"`
Nodes []*topology.Node `json:"nodes"`
LocalID string `json:"localID"`
ClusterName string `json:"clusterName"`
}
func hash(s string) string {
@ -838,15 +857,14 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
req, ok := qreq.(*pilosa.QueryRequest)
if DoPerQueryProfiling {
txsrc := os.Getenv("PILOSA_TXSRC")
backend := pilosa.CurrentBackend()
reqHash := hash(req.Query)
qlen := len(req.Query)
if qlen > 100 {
qlen = 100
}
name := "_query." + reqHash + "." + txsrc + "." + time.Now().Format("20060102150405") + "." + req.Query[:qlen]
name := "_query." + reqHash + "." + backend + "." + time.Now().Format("20060102150405") + "." + req.Query[:qlen]
f, err := os.Create(name)
if err != nil {
panic(err)
@ -857,13 +875,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
defer pprof.StopCPUProfile()
} // end DoPerQueryProfiling
/*
er = trace.Start(f)
if er != nil {
panic(er)
}
defer trace.Stop()
*/
var err error
err, _ = qerr.(error)
@ -994,8 +1005,16 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
q := r.URL.Query()
withViews := q.Get("views") == "true"
indexName := mux.Vars(r)["index"]
for _, idx := range h.api.Schema(r.Context()) {
schema, err := h.api.Schema(r.Context(), withViews)
if err != nil {
h.logger.Printf("getting schema error: %s", err)
}
for _, idx := range schema {
if idx.Name == indexName {
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(idx); err != nil {
@ -1490,7 +1509,7 @@ func (h *Handler) handleGetTransactionList(w http.ResponseWriter, r *http.Reques
trnsMap, err := h.api.Transactions(r.Context())
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrNodeNotCoordinator:
case pilosa.ErrNodeNotPrimary:
http.Error(w, err.Error(), http.StatusBadRequest)
default:
http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError)
@ -1525,7 +1544,7 @@ func (h *Handler) handleGetTransactions(w http.ResponseWriter, r *http.Request)
trnsMap, err := h.api.Transactions(r.Context())
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrNodeNotCoordinator:
case pilosa.ErrNodeNotPrimary:
http.Error(w, err.Error(), http.StatusBadRequest)
default:
http.Error(w, "problem getting transactions: "+err.Error(), http.StatusInternalServerError)
@ -1547,7 +1566,7 @@ type TransactionResponse struct {
func (h *Handler) doTransactionResponse(w http.ResponseWriter, err error, trns *pilosa.Transaction) {
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrNodeNotCoordinator, pilosa.ErrTransactionExists:
case pilosa.ErrNodeNotPrimary, pilosa.ErrTransactionExists:
w.WriteHeader(http.StatusBadRequest)
case pilosa.ErrTransactionExclusive:
w.WriteHeader(http.StatusConflict)
@ -1789,17 +1808,28 @@ func (h *Handler) handleGetMetricsJSON(w http.ResponseWriter, r *http.Request) {
transport := http.DefaultTransport.(*http.Transport).Clone()
for _, node := range h.api.Hosts(r.Context()) {
metricsURI := node.URI.String() + "/metrics"
// The buffer size of 60 is performance controlling, but we
// haven't studied what the optimal setting is. It was
// earlier set to this value to capture all output from
// prom2json at once. The output got larger recently, so
// now we handle unlimited size output using a goroutine.
mfChan := make(chan *dto.MetricFamily, 60)
err := prom2json.FetchMetricFamilies(metricsURI, mfChan, transport)
if err != nil {
http.Error(w, "fetching metrics: "+err.Error(), http.StatusInternalServerError)
return
}
errChan := make(chan error)
go func() {
err := prom2json.FetchMetricFamilies(metricsURI, mfChan, transport)
errChan <- err
}()
nodeMetrics := []*prom2json.Family{}
for mf := range mfChan {
nodeMetrics = append(nodeMetrics, prom2json.NewFamily(mf))
}
err := <-errChan
if err != nil {
http.Error(w, "fetching metrics: "+err.Error(), http.StatusInternalServerError)
return
}
metrics[node.ID] = nodeMetrics
}
@ -2035,47 +2065,6 @@ func parseUint64Slice(s string) ([]uint64, error) {
return a, nil
}
func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
// Decode request.
var req setCoordinatorRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest)
return
}
oldNode, newNode, err := h.api.SetCoordinator(r.Context(), req.ID)
if err != nil {
if errors.Cause(err) == pilosa.ErrNodeIDNotExists {
http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound)
} else {
http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(setCoordinatorResponse{
Old: oldNode,
New: newNode,
}); err != nil {
h.logger.Printf("response encoding error: %s", err)
}
}
type setCoordinatorRequest struct {
ID string `json:"id"`
}
type setCoordinatorResponse struct {
Old *pilosa.Node `json:"old"`
New *pilosa.Node `json:"new"`
}
// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request.
func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -2114,7 +2103,7 @@ type removeNodeRequest struct {
}
type removeNodeResponse struct {
Remove *pilosa.Node `json:"remove"`
Remove *topology.Node `json:"remove"`
}
// handlePostClusterResizeAbort handles POST /cluster/resize/abort request.
@ -2127,7 +2116,7 @@ func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Re
var msg string
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrNodeNotCoordinator:
case pilosa.ErrNodeNotPrimary:
http.Error(w, err.Error(), http.StatusBadRequest)
return
case pilosa.ErrResizeNotRunning:

6
http/handler_test.go
View file

@ -33,11 +33,13 @@ func TestHandlerOptions(t *testing.T) {
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
ln, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
t.Fatalf("creating listener: %v", err)
}
_, err = http.NewHandler(http.OptHandlerListener(ln))
_, err = http.NewHandler(http.OptHandlerListener(ln, ln.Addr().String()))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}

725
index.go
View file

@ -17,8 +17,6 @@ package pilosa
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
@ -26,14 +24,11 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/internal"
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pkg/errors"
"github.com/zeebo/blake3"
"golang.org/x/sync/errgroup"
)
@ -59,6 +54,8 @@ type Index struct {
columnAttrs AttrStore
broadcaster broadcaster
Schemator disco.Schemator
serializer Serializer
Stats stats.StatsClient
// Passed to field for foreign-index lookup.
@ -102,6 +99,9 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
holder: holder,
trackExistence: true,
Schemator: disco.InMemSchemator,
serializer: NopSerializer,
translateStores: make(map[int]TranslateStore),
translationSyncer: NopTranslationSyncer,
@ -174,25 +174,41 @@ func (i *Index) options() IndexOptions {
// Open opens and initializes the index.
func (i *Index) Open() error {
return i.open(false)
return i.open(nil)
}
// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (i *Index) OpenWithTimestamp() error { return i.open(true) }
// OpenWithSchema opens the index and uses the provided schema to verify that
// the index's fields are expected.
func (i *Index) OpenWithSchema(idx *disco.Index) error {
if idx == nil {
return ErrInvalidSchema
}
func (i *Index) open(withTimestamp bool) (err error) {
// decode the CreateIndexMessage from the schema data in order to
// get its metadata.
cim, err := decodeCreateIndexMessage(i.serializer, idx.Data)
if err != nil {
return errors.Wrap(err, "decoding create index message")
}
i.createdAt = cim.CreatedAt
i.trackExistence = cim.Meta.TrackExistence
i.keys = cim.Meta.Keys
return i.open(idx)
}
// open opens the index with an optional schema (disco.Index). If a schema is
// provided, it will apply the metadata from the schema to the index, and then
// open all fields found in the schema. If a schema is not provided, the
// metadata for the index is not changed from its existing value, and fields are
// not validated against the schema as they are opened.
func (i *Index) open(idx *disco.Index) (err error) {
// Ensure the path exists.
i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
return errors.Wrap(err, "creating directory")
}
// Read meta file.
i.holder.Logger.Debugf("load meta file for index: %s", i.name)
if err := i.loadMeta(); err != nil {
return errors.Wrap(err, "loading meta file")
}
// we don't want to open *all* the views for each shard, since
// most are empty when we are doing time quantums. It slows
// down startup dramatically. So we ask for the meta data
@ -203,11 +219,27 @@ func (i *Index) open(withTimestamp bool) (err error) {
}
i.fieldView2shard = fieldView2shard
// Add index to a map in holder. Used by openFields.
i.holder.addIndex(i)
i.holder.Logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(withTimestamp); err != nil {
if err := i.openFields(idx); err != nil {
return errors.Wrap(err, "opening fields")
}
// Set bit depths.
// This is called in Index.open() (as opposed to Field.Open()) because the
// Field.bitDepth() method uses a transaction which relies on the index and
// its entry for the field in the Index.field map. If we try to set a
// field's BitDepth in Field.Open(), which itself might be inside the
// Index.openField() loop, then the field has not yet been added to the
// Index.field map. I think it would be better if Field.bitDepth didn't rely
// on its index at all, but perhaps with transactions that not possible. I
// don't know.
if err := i.setFieldBitDepths(); err != nil {
return errors.Wrap(err, "setting field bitDepths")
}
if i.trackExistence {
if err := i.openExistenceField(); err != nil {
return errors.Wrap(err, "opening existence field")
@ -253,7 +285,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields(withTimestamp bool) error {
func (i *Index) openFields(idx *disco.Index) error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -282,6 +314,27 @@ fileLoop:
continue
}
var cfm *CreateFieldMessage = &CreateFieldMessage{}
var err error
// Only continue with fields which are present in the provided,
// non-nil index schema. The reason we have to check for idx != nil
// here is because there are tests which call index.Open without
// having a disco.Index available.
if idx != nil {
fld, ok := idx.Fields[fi.Name()]
if !ok {
continue
}
// Decode the CreateFieldMessage from the schema data in order to
// get its metadata.
cfm, err = decodeCreateFieldMessage(i.holder.serializer, fld.Data)
if err != nil {
return errors.Wrap(err, "decoding create field message")
}
}
indexQueue <- struct{}{}
eg.Go(func() error {
defer func() {
@ -289,32 +342,11 @@ fileLoop:
}()
i.holder.Logger.Debugf("open field: %s", fi.Name())
mu.Lock()
// goroutine safe
i.holder.addIndex(i)
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
fld.createdAt = timestamp()
}
mu.Unlock()
_, err := i.openField(&mu, cfm, fi.Name())
if err != nil {
return errors.Wrapf(ErrName, "'%s'", fi.Name())
return errors.Wrap(err, "opening field")
}
// Pass holder through to the field for use in looking
// up a foreign index.
fld.holder = i.holder
// open the views we have data for.
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
i.holder.Logger.Debugf("add field to index.fields: %s", fi.Name())
i.mu.Lock()
i.fields[fld.Name()] = fld
i.mu.Unlock()
return nil
})
}
@ -331,9 +363,60 @@ fileLoop:
return err
}
// openField opens the field directory, initializes the field, and adds it to
// the in-memory map of fields maintained by Index.
func (i *Index) openField(mu *sync.Mutex, cfm *CreateFieldMessage, file string) (*Field, error) {
mu.Lock()
fld, err := i.newField(i.fieldPath(filepath.Base(file)), filepath.Base(file))
mu.Unlock()
if err != nil {
return nil, errors.Wrapf(ErrName, "'%s'", file)
}
// Pass holder through to the field for use in looking
// up a foreign index.
fld.holder = i.holder
fld.createdAt = cfm.CreatedAt
fld.options = applyDefaultOptions(cfm.Meta)
// open the views we have data for.
if err := fld.Open(); err != nil {
return nil, fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
i.holder.Logger.Debugf("add field to index.fields: %s", file)
i.mu.Lock()
i.fields[fld.Name()] = fld
i.mu.Unlock()
return fld, nil
}
// openExistenceField gets or creates the existence field and associates it to the index.
func (i *Index) openExistenceField() error {
f, err := i.createFieldIfNotExists(existenceFieldName, &FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
cfm := &CreateFieldMessage{
Index: i.name,
Field: existenceFieldName,
CreatedAt: 0,
Meta: &FieldOptions{CacheType: CacheTypeNone, CacheSize: 0},
}
// First try opening the existence field from disk. If it doesn't already
// exist on disk, then we fall through to the code path which creates it.
var mu sync.Mutex
fld, err := i.openField(&mu, cfm, existenceFieldName)
if err == nil {
i.existenceFld = fld
return nil
} else if errors.Cause(err) != ErrName {
return errors.Wrap(err, "opening existence file")
}
// If we have gotten here, it means that we couldn't successfully open the
// existence field from disk, so we need to create it.
f, err := i.createFieldIfNotExists(cfm)
if err != nil {
return errors.Wrap(err, "creating existence field")
}
@ -341,56 +424,28 @@ func (i *Index) openExistenceField() error {
return nil
}
// loadMeta reads meta data for the index, if any.
func (i *Index) loadMeta() error {
// TrackExistence is by default true
pb := &internal.IndexMeta{TrackExistence: true}
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "reading")
} else {
if err := proto.Unmarshal(buf, pb); err != nil {
return errors.Wrap(err, "unmarshalling")
// setFieldBitDepths sets the BitDepth for all int and decimal fields in the index.
func (i *Index) setFieldBitDepths() error {
for name, f := range i.fields {
switch f.Type() {
case FieldTypeInt, FieldTypeDecimal:
// pass
default:
continue
}
bd, err := f.bitDepth()
if err != nil {
return errors.Wrapf(err, "getting bit depth for field: %s", name)
}
f.mu.Lock()
f.options.BitDepth = bd
f.mu.Unlock()
}
// Copy metadata fields.
if pb == nil {
i.trackExistence = true
} else {
i.trackExistence = pb.TrackExistence
}
i.keys = pb.GetKeys()
return nil
}
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{
Keys: i.keys,
TrackExistence: i.trackExistence,
})
if err != nil {
return errors.Wrap(err, "marshalling")
}
// Write to meta file.
if err := ioutil.WriteFile(filepath.Join(i.path, ".meta"), buf, 0666); err != nil {
return errors.Wrap(err, "writing")
}
return nil
}
// Close closes the index and its fields.
func (i *Index) Close() error {
i.mu.Lock()
defer i.mu.Unlock()
defer func() {
@ -462,7 +517,9 @@ func (i *Index) Field(name string) *Field {
return i.field(name)
}
func (i *Index) field(name string) *Field { return i.fields[name] }
func (i *Index) field(name string) *Field {
return i.fields[name]
}
// Fields returns a list of all fields in the index.
func (i *Index) Fields() []*Field {
@ -514,7 +571,44 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
return nil, errors.Wrap(err, "applying option")
}
return i.createField(name, fo)
cfm := &CreateFieldMessage{
Index: i.name,
Field: name,
CreatedAt: timestamp(),
Meta: fo,
}
// Create the field in etcd as the system of record.
if err := i.persistField(context.Background(), cfm); err != nil {
return nil, errors.Wrap(err, "persisting field")
}
return i.createField(cfm, false)
}
// CreateFieldAndBroadcast creates a field locally, then broadcasts the
// creation to other nodes so they can create locally as well. An error is
// returned if the field already exists.
func (i *Index) CreateFieldAndBroadcast(cfm *CreateFieldMessage) (*Field, error) {
err := validateName(cfm.Field)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
i.mu.Lock()
defer i.mu.Unlock()
// Ensure field doesn't already exist.
if i.fields[cfm.Field] != nil {
return nil, newConflictError(ErrFieldExists)
}
// Create the field in etcd as the system of record.
if err := i.persistField(context.Background(), cfm); err != nil {
return nil, errors.Wrap(err, "persisting field")
}
return i.createField(cfm, true)
}
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
@ -538,10 +632,36 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field
return nil, errors.Wrap(err, "applying option")
}
return i.createField(name, fo)
cfm := &CreateFieldMessage{
Index: i.name,
Field: name,
CreatedAt: timestamp(),
Meta: fo,
}
// Create the field in etcd as the system of record.
if err := i.persistField(context.Background(), cfm); err != nil {
// There is a case where the index is not in memory, but it is in
// persistent storage. In that case, this will return an "index exists"
// error, which in that case should return the index. TODO: We may need
// to allow for that in the future.
return nil, errors.Wrap(err, "persisting field")
}
return i.createField(cfm, false)
}
func (i *Index) createFieldIfNotExists(name string, opt *FieldOptions) (*Field, error) {
// CreateFieldIfNotExistsWithOptions is a method which I created because I
// needed the functionality of CreateFieldIfNotExists, but instead of taking
// function options, taking a *FieldOptions struct. TODO: This should
// definintely be refactored so we don't have these virtually equivalent
// methods, but I'm puttin this here for now just to see if it works.
func (i *Index) CreateFieldIfNotExistsWithOptions(name string, opt *FieldOptions) (*Field, error) {
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
i.mu.Lock()
defer i.mu.Unlock()
@ -550,21 +670,83 @@ func (i *Index) createFieldIfNotExists(name string, opt *FieldOptions) (*Field,
return f, nil
}
return i.createField(name, opt)
cfm := &CreateFieldMessage{
Index: i.name,
Field: name,
CreatedAt: timestamp(),
Meta: opt,
}
// Create the field in etcd as the system of record.
if err := i.persistField(context.Background(), cfm); err != nil {
// There is a case where the index is not in memory, but it is in
// persistent storage. In that case, this will return an "index exists"
// error, which in that case should return the index. TODO: We may need
// to allow for that in the future.
return nil, errors.Wrap(err, "persisting field")
}
return i.createField(cfm, false)
}
func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
if name == "" {
// persistField stores the field information in etcd.
func (i *Index) persistField(ctx context.Context, cfm *CreateFieldMessage) error {
if cfm.Index == "" {
return ErrIndexRequired
} else if cfm.Field == "" {
return ErrFieldRequired
}
if err := validateName(cfm.Field); err != nil {
return errors.Wrap(err, "validating name")
}
if b, err := i.serializer.Marshal(cfm); err != nil {
return errors.Wrap(err, "marshaling")
} else if err := i.Schemator.CreateField(ctx, cfm.Index, cfm.Field, b); err != nil {
return errors.Wrapf(err, "writing field to disco: %s/%s", cfm.Index, cfm.Field)
}
return nil
}
// createFieldIfNotExists creates the field if it does not already exist in the
// in-memory index structure. This is not related to whether or not the field
// exists in etcd.
func (i *Index) createFieldIfNotExists(cfm *CreateFieldMessage) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find field in cache first.
if f := i.fields[cfm.Field]; f != nil {
return f, nil
}
return i.createField(cfm, false)
}
// createField, in addition to creating a new Field, calls Field.Open which
// potentially aquires a lock on Index. So until/unless we refactor the
// Index.createField() function call path, we cannot call Index.createField
// while holding an Index lock.
func (i *Index) createField(cfm *CreateFieldMessage, broadcast bool) (*Field, error) {
opt := cfm.Meta
if opt == nil {
opt = &FieldOptions{}
}
// TODO: can we do a general FieldOption validation here instead of just cache type?
if cfm.Field == "" {
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Initialize field.
f, err := i.newField(i.fieldPath(name), name)
f, err := i.newField(i.fieldPath(cfm.Field), cfm.Field)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
f.createdAt = cfm.CreatedAt
// Pass holder through to the field for use in looking
// up a foreign index.
@ -577,17 +759,19 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
return nil, errors.Wrap(err, "opening")
}
if err := f.saveMeta(); err != nil {
f.Close()
return nil, errors.Wrap(err, "saving meta")
}
// Add to index's field lookup.
i.fields[name] = f
i.fields[cfm.Field] = f
// enable Txf to find the index in field_test.go TestField_SetValue
f.idx = i
if broadcast {
// Send the create field message to all nodes.
if err := i.holder.sendOrSpool(cfm); err != nil {
return nil, errors.Wrap(err, "sending CreateField message")
}
}
// Kick off the field's translation sync process.
if err := i.translationSyncer.Reset(); err != nil {
return nil, errors.Wrap(err, "resetting translation syncer")
@ -604,6 +788,8 @@ func (i *Index) newField(path, name string) (*Field, error) {
f.idx = i
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.schemator = i.Schemator
f.serializer = i.serializer
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
f.OpenTranslateStore = i.OpenTranslateStore
return f, nil
@ -614,6 +800,11 @@ func (i *Index) DeleteField(name string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Disallow deleting the existence field.
if name == existenceFieldName {
return newNotFoundError(ErrFieldNotFound, existenceFieldName)
}
// Confirm field exists.
f := i.field(name)
if f == nil {
@ -629,21 +820,14 @@ func (i *Index) DeleteField(name string) error {
return errors.Wrap(err, "Txf.DeleteFieldFromStore")
}
// If the field being deleted is the existence field,
// turn off existence tracking on the index.
if name == existenceFieldName {
i.trackExistence = false
i.existenceFld = nil
// Update meta data on disk.
if err := i.saveMeta(); err != nil {
return errors.Wrap(err, "saving existence meta data")
}
}
// Remove reference.
delete(i.fields, name)
// Delete the field from etcd as the system of record.
if err := i.Schemator.DeleteField(context.TODO(), i.name, name); err != nil {
return errors.Wrapf(err, "deleting field from etcd: %s/%s", i.name, name)
}
return i.translationSyncer.Reset()
}
@ -706,323 +890,12 @@ func FormatQualifiedIndexName(index string) string {
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Mostly for debugging.
func (idx *Index) Dump(label string) {
func (i *Index) Dump(label string) {
fileline := FileLine(2)
fmt.Printf("\n%v Dump: %v\n\n", fileline, label)
idx.holder.txf.dbPerShard.DumpAll()
i.holder.txf.dbPerShard.DumpAll()
}
func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
// If we go to a database per shard then index will need this, or
// something like it, to read database files/directories
// and figure out what all the shards are so that a view
// can open its fragments.
file, err := os.Open(filepath.Join(viewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
idx.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", idx.name, field, view, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
type AllTranslatorSummary struct {
Sums []*TranslatorSummary
RepairNeeded bool
}
func (ats *AllTranslatorSummary) Checksum() string {
ats.Sort()
hasher := blake3.New()
for _, sum := range ats.Sums {
_, _ = hasher.Write([]byte(sum.Checksum))
}
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("blake3-%x", buf)
}
func NewAllTranslatorSummary() *AllTranslatorSummary {
return &AllTranslatorSummary{}
}
func (ats *AllTranslatorSummary) Append(b *AllTranslatorSummary) {
ats.Sums = append(ats.Sums, b.Sums...)
ats.RepairNeeded = ats.RepairNeeded || b.RepairNeeded
}
func (ats *AllTranslatorSummary) Sort() {
// return sorted by index then PartitionID then Field
sort.Slice(ats.Sums, func(i, j int) bool {
a := ats.Sums[i]
b := ats.Sums[j]
if a.Index < b.Index {
return true
}
if a.Index > b.Index {
return false
}
// INVAR: a.Index == b.Index
if a.PartitionID < b.PartitionID {
return true
}
if a.PartitionID > b.PartitionID {
return false
}
if a.Field < b.Field {
return true
}
if a.Field > b.Field {
return false
}
return a.NodeID < b.NodeID
})
}
// sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil
func (idx *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string, parallelReaders int) (ats *AllTranslatorSummary, err error) {
idx.mu.RLock()
defer idx.mu.RUnlock()
ats = &AllTranslatorSummary{}
var atsMu sync.Mutex
if verbose {
fmt.Printf("\n# index: %v\n# =================\n", idx.name)
}
pjob := newParallelJobs(parallelReaders)
floop:
for _, fld := range idx.fields {
fld := fld
fun := func(worker int) error {
//vv("ComputeTranslatorSummary() on fld '%v'", fld.name)
sum, err := fld.translateStore.ComputeTranslatorSummaryRows()
if err != nil {
return err
}
sum.Field = fld.name
sum.Index = idx.Name()
sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, idx.Name())))
sum.IsColKey = false
if verbose {
fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
}
atsMu.Lock()
ats.Sums = append(ats.Sums, sum)
atsMu.Unlock()
return nil
}
if !pjob.run(fun) {
break floop
}
} // end floop
if verbose {
fmt.Printf("# ====================\n")
}
tloop:
for partitionID, store := range idx.translateStores {
partitionID := partitionID
store := store
fun2 := func(worker int) error {
//vv("ComputeTranslatorSummary() running on store.Path = '%v'", store.GetStorePath())
if checkKeys {
prim := topo.PrimaryNodeIndex(partitionID)
primID := topo.nodeIDs[prim]
// note: we fix irrespective of nodeID == primID now, so that we
// get a fine grain report of what maps were off.
if verbose {
// This is pilosa-fsck output, not regular log.
fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID)
}
changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs)
if err != nil {
return errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()")
}
if changed {
atsMu.Lock()
ats.RepairNeeded = true
atsMu.Unlock()
}
}
// key repair has to be above, because we compute the checksum below.
sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo)
if err != nil {
return err
}
if sum == nil {
// probably one of the Noop stores from the tests.
return nil
}
sum.IsColKey = true
sum.PartitionID = partitionID
sum.Index = idx.Name()
sum.StorePath = store.GetStorePath()
sum.NodeID = nodeID
sum.IsPrimary = topo.IsPrimary(nodeID, partitionID)
replicas := topo.GetNonPrimaryReplicas(partitionID)
for _, replica := range replicas {
if nodeID == replica {
sum.IsReplica = true
break
}
}
sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, idx.Name())))
if verbose {
// This is not regular index logging. This is output of the pilosa-fsck tool.
// So it must be printing straight to stdout.
fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex)
}
atsMu.Lock()
ats.Sums = append(ats.Sums, sum)
atsMu.Unlock()
return nil
}
if !pjob.run(fun2) {
break tloop
}
} // end tloop
err = pjob.waitForFinish()
return ats, err
}
// returned by WriteFragmentChecksums
type IndexFragmentSummary struct {
Dir string
NodeID string
Index string
IndexPath string
Frg []*FragSum
RelPath2fsum map[string]*FragSum
}
func (ifs *IndexFragmentSummary) String() (s string) {
s = fmt.Sprintf(`&pilosa.IndexFragmentSummary{
Dir: '%v'
NodeID: '%v'
Index: '%v'
IndexPath: '%v'
`, ifs.Dir, ifs.NodeID, ifs.Index, ifs.IndexPath)
for _, frg := range ifs.Frg {
s += frg.String() + "\n"
}
s += "}\n"
return
}
// used in IndexFragmentSummary
type FragSum struct {
AbsPath string
RelPath string
// critically, NodeID is how pilosa-fsck figures out if this
// fragment should be deleted if it is on a node it should not be.
NodeID string
Index string
Field string
View string
Shard uint64
Hotbits int
Checksum string
Primary int
ScanDone bool // pilosa-fsck will set this once done to avoid repairing multiple times.
}
func (fsum *FragSum) String() (s string) {
return fmt.Sprintf("%#v", fsum)
}
// if verbose, then print to w.
func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool, topo *Topology, verbose bool) (sum *IndexFragmentSummary) {
sum = &IndexFragmentSummary{
Index: idx.name,
IndexPath: idx.path,
RelPath2fsum: make(map[string]*FragSum),
}
paths, err := listFilesUnderDir(idx.path, false, "", true)
panicOn(err)
index := idx.name
n := 0
for _, relpath := range paths {
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
if err != nil {
continue // ignore .meta paths
}
abspath := idx.path + sep + relpath
primary := topo.GetPrimaryForShardReplication(index, shard)
checksum, hotbits := RoaringFragmentChecksum(abspath, index, field, view, shard)
if verbose {
fmt.Fprintf(w, "# frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v primary:%03v\n", checksum, field, view, shard, hotbits, primary)
}
fsum := &FragSum{
AbsPath: abspath,
RelPath: relpath,
Index: index,
Field: field,
View: view,
Shard: shard,
Hotbits: hotbits,
Checksum: checksum,
Primary: primary,
}
sum.Frg = append(sum.Frg, fsum)
_, already := sum.RelPath2fsum[relpath]
if already {
panic(fmt.Sprintf("relpath '%v' was already present!?!", relpath))
}
sum.RelPath2fsum[relpath] = fsum
n++
}
if n == 0 {
if verbose {
fmt.Fprintf(w, "empty index '%v'", idx.path)
}
}
return
}
func (idx *Index) Txf() *TxFactory {
return idx.holder.txf
func (i *Index) Txf() *TxFactory {
return i.holder.txf
}

37
index_internal_test.go
View file

@ -52,40 +52,3 @@ func (i *Index) reopen() error {
}
return nil
}
// Ensure that deleting the existence field is handled properly.
func TestIndex_Existence_Delete(t *testing.T) {
// Create Index (with existence tracking).
index := mustOpenIndex(t, IndexOptions{TrackExistence: true})
defer index.Close()
// Ensure existence field has been created.
ef := index.Field(existenceFieldName)
if ef == nil {
t.Fatalf("expected field to have been created: %s", existenceFieldName)
} else if !index.trackExistence {
t.Fatalf("expected index.trackExistence to be true")
} else if index.existenceFld == nil {
t.Fatalf("expected index.existenceField to be non-nil")
}
// Delete existence field.
if err := index.DeleteField(existenceFieldName); err != nil {
t.Fatal(err)
}
// Re-open index.
if err := index.reopen(); err != nil {
t.Fatal(err)
}
// Ensure existence field no longer exists.
ef = index.Field(existenceFieldName)
if ef != nil {
t.Fatalf("expected field to have been deleted: %s", existenceFieldName)
} else if index.trackExistence {
t.Fatalf("expected index.trackExistence to be false")
} else if index.existenceFld != nil {
t.Fatalf("expected index.existenceField to be nil")
}
}

77
internal/clustertests/cluster_test.go
View file

@ -22,6 +22,7 @@ import (
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/disco"
picli "github.com/pilosa/pilosa/v2/http"
)
@ -29,17 +30,25 @@ func TestClusterStuff(t *testing.T) {
if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
t.Skip()
}
cli, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
if err != nil {
t.Fatalf("getting client: %v", err)
}
cli2, err := picli.NewInternalClient("pilosa2:10101", picli.GetHTTPClient(nil))
if err != nil {
t.Fatalf("getting client: %v", err)
}
cli3, err := picli.NewInternalClient("pilosa3:10101", picli.GetHTTPClient(nil))
if err != nil {
t.Fatalf("getting client: %v", err)
}
t.Run("long pause", func(t *testing.T) {
err := cli.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{})
err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
err = cli.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100})
err = cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100})
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -50,19 +59,22 @@ func TestClusterStuff(t *testing.T) {
data[i%10].ColumnID = uint64((i/10)*pilosa.ShardWidth + i%10)
shard := uint64(i / 10)
if i%10 == 9 {
err = cli.Import(context.Background(), "testidx", "testf", shard, data)
err = cli1.Import(context.Background(), "testidx", "testf", shard, data)
if err != nil {
t.Fatalf("importing: %v", err)
}
}
}
r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying: %v", err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count after import is %d", r.Results[0].(uint64))
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64))
}
}
pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
@ -78,18 +90,45 @@ func TestClusterStuff(t *testing.T) {
t.Fatalf("waiting on pumba pause cmd: %v", err)
}
// TODO change the sleep to wait for status to return to NORMAL - need support in internal client for getting status
t.Log("done with pause, waiting for stability")
time.Sleep(time.Second * 20)
waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second)
t.Log("done waiting for stability")
r, err = cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying: %v", err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count after import is %d", r.Results[0].(uint64))
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64))
}
}
})
}
func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration) {
t.Helper()
for i := 0; i < n; i++ {
s, err := stator(context.TODO())
if err != nil {
t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String())
} else {
t.Logf("Status (try %d/%d): %s (retrying in %s)", i, n, s, sleep.String())
}
if s == status {
return
}
time.Sleep(sleep)
}
s, err := stator(context.TODO())
if err != nil {
t.Fatalf("querying status: %v", err)
}
if status != s {
waited := time.Duration(n) * sleep
t.Fatalf("waited %s for status: %s, got: %s", waited.String(), status, s)
}
}

22
internal/clustertests/docker-compose.yml
View file

@ -8,8 +8,12 @@ services:
ports:
- "33455:10101"
environment:
- PILOSA_CLUSTER_COORDINATOR=true
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
- PILOSA_NAME=pilosa1
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
networks:
- pilosanet
command:
@ -22,7 +26,12 @@ services:
ports:
- "33456:10101"
environment:
- PILOSA_GOSSIP_SEEDS=pilosa1:14000
- PILOSA_NAME=pilosa2
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
networks:
- pilosanet
command:
@ -35,7 +44,12 @@ services:
ports:
- "33457:10101"
environment:
- PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000
- PILOSA_NAME=pilosa3
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
networks:
- pilosanet
command:

1292
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

25
internal/private.proto
View file

@ -112,7 +112,7 @@ message URI {
message Node {
string ID = 1;
URI URI = 2;
bool IsCoordinator = 3;
bool IsPrimary = 3;
string State = 4;
URI GRPCURI = 5;
}
@ -174,7 +174,7 @@ message DeleteViewMessage {
message ResizeInstruction {
int64 JobID = 1;
Node Node = 2;
Node Coordinator = 3;
Node Primary = 3;
repeated ResizeSource Sources = 4;
repeated TranslationResizeSource TranslationSources = 8;
NodeStatus NodeStatus = 7;
@ -201,14 +201,6 @@ message ResizeInstructionComplete {
string Error = 3;
}
message SetCoordinatorMessage {
Node New = 1;
}
message UpdateCoordinatorMessage {
Node New = 1;
}
message Topology {
string ClusterID = 1;
repeated string NodeIDs = 2;
@ -216,6 +208,8 @@ message Topology {
message RecalculateCaches {}
message LoadSchemaMessage {}
message TransactionMessage {
string Action = 1;
Transaction Transaction = 2;
@ -230,4 +224,13 @@ message Transaction {
TransactionStats Stats = 6;
}
message TransactionStats {}
message TransactionStats {}
message ResizeAbortMessage {
}
message ResizeNodeMessage {
string NodeID = 1;
string Action = 2;
}

190
internal/public.pb.go
View file

@ -6771,7 +6771,10 @@ func (m *Row) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6856,7 +6859,10 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6979,7 +6985,10 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7138,7 +7147,10 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7265,7 +7277,10 @@ func (m *IDList) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7369,7 +7384,10 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7486,7 +7504,10 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7569,7 +7590,10 @@ func (m *KeyList) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7783,7 +7807,10 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7920,7 +7947,10 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8035,7 +8065,10 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8154,7 +8187,10 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8275,7 +8311,10 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8394,7 +8433,10 @@ func (m *PairField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8511,7 +8553,10 @@ func (m *PairsField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8581,7 +8626,10 @@ func (m *Int64) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8751,7 +8799,10 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8874,7 +8925,10 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9010,7 +9064,10 @@ func (m *ValCount) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9099,7 +9156,10 @@ func (m *Decimal) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9235,7 +9295,10 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9419,7 +9482,10 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9504,7 +9570,10 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9797,7 +9866,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9948,7 +10020,10 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10561,7 +10636,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11045,7 +11123,10 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11507,7 +11588,10 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11677,7 +11761,10 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11760,7 +11847,10 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11927,7 +12017,10 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12054,7 +12147,10 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12245,7 +12341,10 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12328,7 +12427,10 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12445,7 +12547,10 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12659,7 +12764,10 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -12920,7 +13028,10 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -13037,7 +13148,10 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {

17
main_test.go
View file

@ -16,19 +16,28 @@ package pilosa_test
import (
"fmt"
"net"
"net/http"
"testing"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/testhook"
_ "net/http/pprof"
"github.com/pilosa/pilosa/v2/testhook"
)
func TestMain(m *testing.M) {
port := pilosa.GetAvailPort()
l, err := net.Listen("tcp", ":0")
if err != nil {
panic(err)
}
port := l.Addr().(*net.TCPAddr).Port
fmt.Printf("pilosa/ TestMain: online stack-traces: curl http://localhost:%v/debug/pprof/goroutine?debug=2\n", port)
go func() {
_ = http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", port), nil)
err := http.Serve(l, nil)
if err != nil {
panic(err)
}
}()
testhook.RunTestsWithHooks(m)
}

2
mmap_test.go
View file

@ -30,7 +30,7 @@ type cv struct {
}
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint(6)
depth := uint64(6)
f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0)
tx.Rollback()
f.Logger = logger.NewLogfLogger(t)

22
mock/translator.go
View file

@ -37,13 +37,6 @@ type TranslateStore struct {
EntryReaderFunc func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error)
}
func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) {
return
}
func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) {
return
}
func (s *TranslateStore) Close() error {
return s.CloseFunc()
}
@ -104,14 +97,6 @@ func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) {
return 0, nil
}
func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) {
return
}
func (s *TranslateStore) GetStorePath() string {
return ""
}
var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil)
type TranslateEntryReader struct {
@ -126,10 +111,3 @@ func (r *TranslateEntryReader) Close() error {
func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
return r.ReadEntryFunc(entry)
}
func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error {
panic("TODO")
}
func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error {
panic("TODO")
}

17
uri.go → net/uri.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package net
import (
"encoding/json"
@ -54,6 +54,11 @@ func (u *URI) URL() url.URL {
return url.URL{Scheme: u.Scheme, Host: net.JoinHostPort(u.Host, strconv.Itoa(int(u.Port)))}
}
// DefaultURI creates and returns the default URI.
func DefaultURI() *URI {
return defaultURI()
}
// defaultURI creates and returns the default URI.
func defaultURI() *URI {
return &URI{
@ -79,7 +84,7 @@ func (u URIs) HostPortStrings() []string {
// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
uri := defaultURI()
err := uri.setHost(host)
err := uri.SetHost(host)
if err != nil {
return nil, errors.Wrap(err, "setting uri host")
}
@ -92,8 +97,8 @@ func NewURIFromAddress(address string) (*URI, error) {
return parseAddress(address)
}
// setScheme sets the scheme of this URI.
func (u *URI) setScheme(scheme string) error {
// SetScheme sets the scheme of this URI.
func (u *URI) SetScheme(scheme string) error {
m := schemeRegexp.FindStringSubmatch(scheme)
if m == nil {
return errors.New("invalid scheme")
@ -102,8 +107,8 @@ func (u *URI) setScheme(scheme string) error {
return nil
}
// setHost sets the host of this URI.
func (u *URI) setHost(host string) error {
// SetHost sets the host of this URI.
func (u *URI) SetHost(host string) error {
m := hostRegexp.FindStringSubmatch(host)
if m == nil {
return errors.New("invalid host")

10
uri_internal_test.go → net/uri_internal_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package net
import "testing"
@ -79,7 +79,7 @@ func TestURIPath(t *testing.T) {
func TestSetScheme(t *testing.T) {
uri := defaultURI()
target := "fun"
err := uri.setScheme(target)
err := uri.SetScheme(target)
if err != nil {
t.Fatal(err)
}
@ -91,7 +91,7 @@ func TestSetScheme(t *testing.T) {
func TestSetHost(t *testing.T) {
uri := defaultURI()
target := "10.20.30.40"
err := uri.setHost(target)
err := uri.SetHost(target)
if err != nil {
t.Fatal(err)
}
@ -111,7 +111,7 @@ func TestSetPort(t *testing.T) {
func TestSetInvalidScheme(t *testing.T) {
uri := defaultURI()
err := uri.setScheme("?invalid")
err := uri.SetScheme("?invalid")
if err == nil {
t.Fatalf("Should have failed")
}
@ -119,7 +119,7 @@ func TestSetInvalidScheme(t *testing.T) {
func TestSetInvalidHost(t *testing.T) {
uri := defaultURI()
err := uri.setHost("index?.pilosa.com")
err := uri.SetHost("index?.pilosa.com")
if err == nil {
t.Fatalf("Should have failed")
}

55
pg/pgtest/server.go
View file

@ -16,6 +16,7 @@ package pgtest
import (
"context"
"fmt"
"net"
"testing"
@ -36,13 +37,8 @@ func (f ShutdownFunc) Finish(tb testing.TB, name string) {
}
}
// ServeTCP creates a TCP listener and serves postgres wire protocol on it.
func ServeTCP(addr string, server *pg.Server) (net.Addr, ShutdownFunc, error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, nil, errors.Wrap(err, "listening on TCP")
}
// ServeListener serves postgres wire protocol on a listener.
func ServeListener(listener net.Listener, server *pg.Server) (net.Addr, ShutdownFunc, error) {
laddr := listener.Addr()
ctx, cancel := context.WithCancel(context.Background())
@ -57,6 +53,37 @@ func ServeTCP(addr string, server *pg.Server) (net.Addr, ShutdownFunc, error) {
nil
}
// ServeTCP creates a TCP listener and serves postgres wire protocol on it.
func ServeTCP(addr string, server *pg.Server) (net.Addr, ShutdownFunc, error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, nil, errors.Wrap(err, "listening on TCP")
}
return ServeListener(listener, server)
}
// ServeTLSListener sets up TLS on the server and invokes ServeListener.
func ServeTLSListener(listener net.Listener, server *pg.Server) (net.Addr, ShutdownFunc, error) {
err := SetupTLS(server)
if err != nil {
return nil, nil, errors.Wrap(err, "server TLS setup failed")
}
var tries int = 5
var netAddr net.Addr
var shutdown ShutdownFunc
for i := 0; i < tries; i++ {
if i > 0 {
fmt.Printf("--- try serving TLS again: %d\n", i)
}
if netAddr, shutdown, err = ServeListener(listener, server); err == nil {
break
}
}
return netAddr, shutdown, err
}
// ServeTLS sets up TLS on the server and invokes ServeTCP.
func ServeTLS(addr string, server *pg.Server) (net.Addr, ShutdownFunc, error) {
err := SetupTLS(server)
@ -64,7 +91,19 @@ func ServeTLS(addr string, server *pg.Server) (net.Addr, ShutdownFunc, error) {
return nil, nil, errors.Wrap(err, "server TLS setup failed")
}
return ServeTCP(addr, server)
var tries int = 5
var netAddr net.Addr
var shutdown ShutdownFunc
for i := 0; i < tries; i++ {
if i > 0 {
fmt.Printf("--- try serving TLS again: %d\n", i)
}
if netAddr, shutdown, err = ServeTCP(addr, server); err == nil {
break
}
}
return netAddr, shutdown, err
}
// ConnectFunc is a function to connect to a server.

4
pg/server_test.go
View file

@ -109,6 +109,7 @@ func TestPQConnect(t *testing.T) {
StartupTimeout: time.Second,
Logger: logger.NopLogger,
}
addr, shutdown, err := pgtest.ServeTCP(":0", server)
if err != nil {
t.Fatalf("starting postgres server: %v", err)
@ -140,6 +141,7 @@ func TestPQConnectSSL(t *testing.T) {
StartupTimeout: time.Second,
Logger: logger.NopLogger,
}
addr, shutdown, err := pgtest.ServeTLS(":0", server)
if err != nil {
t.Fatalf("starting postgres server: %v", err)
@ -204,6 +206,7 @@ func TestPSQLQuery(t *testing.T) {
StartupTimeout: time.Second,
Logger: logger.NopLogger,
}
addr, shutdown, err := pgtest.ServeTCP(":0", server)
if err != nil {
t.Fatalf("starting postgres server: %v", err)
@ -265,6 +268,7 @@ func TestPSQLQuery(t *testing.T) {
Logger: logger.NopLogger,
CancellationManager: pg.NewLocalCancellationManager(rand.Reader),
}
addr, shutdown, err := pgtest.ServeTCP(":0", server)
if err != nil {
t.Fatalf("starting postgres server: %v", err)

67
pilosa.go
View file

@ -16,9 +16,13 @@ package pilosa
import (
"encoding/json"
"os"
"regexp"
"time"
"github.com/pilosa/pilosa/v2/disco"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pkg/errors"
)
@ -27,15 +31,17 @@ var (
ErrHostRequired = errors.New("host required")
ErrIndexRequired = errors.New("index required")
ErrIndexExists = errors.New("index already exists")
ErrIndexExists = disco.ErrIndexExists
ErrIndexNotFound = errors.New("index not found")
ErrInvalidSchema = errors.New("invalid schema")
ErrForeignIndexNotFound = errors.New("foreign index not found")
// ErrFieldRequired is returned when no field is specified.
ErrFieldRequired = errors.New("field required")
ErrColumnRequired = errors.New("column required")
ErrFieldExists = errors.New("field already exists")
ErrFieldExists = disco.ErrFieldExists
ErrFieldNotFound = errors.New("field not found")
ErrBSIGroupNotFound = errors.New("bsigroup not found")
@ -50,6 +56,8 @@ var (
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrDecimalOutOfRange = errors.New("decimal value out of range")
ErrViewRequired = errors.New("view required")
ErrViewExists = disco.ErrViewExists
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
@ -69,10 +77,10 @@ var (
// ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match
ErrPreconditionFailed = errors.New("precondition failed")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
ErrResizeNoReplicas = errors.New("not enough data to perform resize (replica factor may need to be increased)")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotPrimary = errors.New("node is not the primary")
ErrResizeNotRunning = errors.New("no resize job currently running")
ErrResizeNoReplicas = errors.New("not enough data to perform resize (replica factor may need to be increased)")
ErrNotImplemented = errors.New("not implemented")
ErrFieldsArgumentRequired = errors.New("fields argument required")
@ -178,39 +186,32 @@ func validateName(name string) error {
return nil
}
// stringSlicesAreEqual determines if two string slices are equal.
func stringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
}
if a == nil || b == nil {
return false
}
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func timestamp() int64 {
return time.Now().UnixNano()
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {
func AddressWithDefaults(addr string) (*pnet.URI, error) {
if addr == "" {
return defaultURI(), nil
return pnet.DefaultURI(), nil
}
return NewURIFromAddress(addr)
return pnet.NewURIFromAddress(addr)
}
// CurrentBackend is one step in an attempt to centralize (and either minimize
// or completely remove), the calls to environment variables throughout the
// tests. Ideally we could get rid of this and rely completely on the
// configuration parameters.
func CurrentBackend() string {
return os.Getenv("PILOSA_STORAGE_BACKEND")
}
// CurrentBackendOrDefault tries the environment variable first, but falls back
// to the default backend if the environment variable is empty.
func CurrentBackendOrDefault() string {
if backend := os.Getenv("PILOSA_STORAGE_BACKEND"); backend != "" {
return backend
}
return storage.DefaultBackend
}

26
pprof.go
View file

@ -22,16 +22,20 @@ import (
"time"
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"github.com/pilosa/pilosa/v2/storage"
)
// CPUProfileForDur (where "Dur" is short for "Duration"), is used for
// performance tuning during development. It's only called—but is currently
// commented out—in holder.go.
func CPUProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc == "" {
txsrc = DefaultTxsrc
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
path := outpath + "." + txsrc
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)
@ -48,14 +52,16 @@ func CPUProfileForDur(dur time.Duration, outpath string) {
}()
}
// MemProfileForDur (where "Dur" is short for "Duration"), is used for
// performance tuning during development. It's only called—but is currently
// commented out—in holder.go.
func MemProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc == "" {
txsrc = DefaultTxsrc
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
path := outpath + "." + txsrc
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)

8
pql/ast.go
View file

@ -436,6 +436,14 @@ var callInfoByFunc = map[string]callInfo{
"field": "",
},
},
"Percentile": {
allowUnknown: false,
prototypes: map[string]interface{}{
"field": "",
"filter": nil,
"nth": nil,
},
},
// special cases:
"Clear": {
allowUnknown: true,

1
pql/pql.peg
View file

@ -14,6 +14,7 @@ Call <- "Set" {p.startCall("Set")} open col comma args (comma timestamp)? close
/ "Store" {p.startCall("Store")} open Call comma arg close {p.endCall()}
/ "TopN" {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ "TopK" {p.startCall("TopK")} open posfield (comma allargs)? close {p.endCall()}
/ "Percentile" {p.startCall("Percentile")} open posfield (comma allargs)? close {p.endCall()}
/ "Rows" {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
/ "Range" {p.startCall("Range")} open field eq value comma 'from='? {p.addField("from")} timestampfmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(text)} close {p.endCall()}
/ < IDENT > { p.startCall(text) } open allargs comma? close { p.endCall() }

2536
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

25
rbf.go
View file

@ -30,6 +30,8 @@ import (
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
)
@ -90,6 +92,14 @@ type rbfDBRegistrar struct {
mp map[*RbfDBWrapper]bool
path2db map[string]*RbfDBWrapper
rbfConfig *rbfcfg.Config
}
func (r *rbfDBRegistrar) SetRBFConfig(cfg *rbfcfg.Config) {
r.mu.Lock()
defer r.mu.Unlock()
r.rbfConfig = cfg
}
func (r *rbfDBRegistrar) Size() int {
@ -148,7 +158,7 @@ func rbfPath(path string) string {
// if one does not exist for its path. Otherwise it returns
// the existing instance. This insures only one RbfDBWrapper
// per bpath in this pilosa node.
func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *rbfcfg.Config) (DBWrapper, error) {
func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
path := rbfPath(path0)
r.mu.Lock()
defer r.mu.Unlock()
@ -157,11 +167,12 @@ func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *rbfc
// creates the effect of having only one DB open per pilosa node.
return w, nil
}
if cfg == nil {
cfg = rbfcfg.NewDefaultConfig()
cfg.DoAllocZero = doAllocZero
if r.rbfConfig == nil {
r.rbfConfig = rbfcfg.NewDefaultConfig()
r.rbfConfig.DoAllocZero = doAllocZero
r.rbfConfig.FsyncEnabled = cfg.FsyncEnabled
}
db := rbf.NewDB(path, cfg)
db := rbf.NewDB(path, r.rbfConfig)
w = &RbfDBWrapper{
reg: r,
@ -169,7 +180,7 @@ func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *rbfc
db: db,
doAllocZero: doAllocZero,
openTx: make(map[*RBFTx]bool),
cfg: cfg,
cfg: r.rbfConfig,
}
r.unprotectedRegister(w)
@ -423,7 +434,7 @@ func (tx *RBFTx) UseRowCache() bool {
// the rowCache without first making a copy.
// So we only use the rowCache if the copy is
// enabled.
return rbf.EnableRowCache()
return storage.EnableRowCache()
}
func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {

25
rbf/cfg/cfg.go
View file

@ -28,26 +28,26 @@ const (
type Config struct {
// The maximum allowed database size. Required by mmap.
MaxSize int64
MaxSize int64 `toml:"max-db-size"`
// The maximum allowed WAL size. Required by mmap.
MaxWALSize int64
MaxWALSize int64 `toml:"max-wal-size"`
// The minimum WAL size before the WAL is copied to the DB.
MinWALCheckpointSize int64
MinWALCheckpointSize int64 `toml:"min-wal-checkpoint-size"`
// The maximum WAL size before transactions are halted to allow a checkpoint.
MaxWALCheckpointSize int64
MaxWALCheckpointSize int64 `toml:"max-wal-checkpoint-size"`
// Set before calling db.Open()
FsyncEnabled bool
FsyncEnabled bool `toml:"fsync"`
// for mmap correctness testing.
DoAllocZero bool
DoAllocZero bool `toml:"do-alloc-zero"`
// CursorCacheSize is the number of copies of Cursor{} to keep in our
// readyCursorCh arena to avoid GC pressure.
CursorCacheSize int64
CursorCacheSize int64 `toml:"cursor-cache-size"`
}
func NewDefaultConfig() *Config {
@ -66,13 +66,12 @@ func NewDefaultConfig() *Config {
func (cfg *Config) DefineFlags(flags *pflag.FlagSet) {
default0 := NewDefaultConfig()
flags.Int64Var(&cfg.MaxSize, "rbf-max-db-size", default0.MaxSize, "RBF maximum size in bytes of a database file (distinct from a WAL file)")
flags.Int64Var(&cfg.MaxWALSize, "rbf-max-wal-size", default0.MaxWALSize, "RBF maximum size in bytes of a WAL file (distinct from a DB file)")
flags.Int64Var(&cfg.MinWALCheckpointSize, "rbf-min-wal-checkpoint-size", default0.MinWALCheckpointSize, "RBF minimum size in bytes of a WAL file before attempting checkpoint")
flags.Int64Var(&cfg.MaxWALCheckpointSize, "rbf-max-wal-checkpoint-size", default0.MaxWALCheckpointSize, "RBF maximum size in bytes of a WAL file before forcing checkpoint")
flags.Int64Var(&cfg.MaxSize, "rbf.max-db-size", default0.MaxSize, "RBF maximum size in bytes of a database file (distinct from a WAL file)")
flags.Int64Var(&cfg.MaxWALSize, "rbf.max-wal-size", default0.MaxWALSize, "RBF maximum size in bytes of a WAL file (distinct from a DB file)")
flags.Int64Var(&cfg.MinWALCheckpointSize, "rbf.min-wal-checkpoint-size", default0.MinWALCheckpointSize, "RBF minimum size in bytes of a WAL file before attempting checkpoint")
flags.Int64Var(&cfg.MaxWALCheckpointSize, "rbf.max-wal-checkpoint-size", default0.MaxWALCheckpointSize, "RBF maximum size in bytes of a WAL file before forcing checkpoint")
// renamed from --rbf-fsync to just --fsync because now it applies to all Tx backends.
flags.BoolVar(&cfg.FsyncEnabled, "fsync", default0.FsyncEnabled, "enable fsync fully safe flush-to-disk")
flags.Int64Var(&cfg.CursorCacheSize, "rbf-cursor-cache", default0.CursorCacheSize, "how big a Cursor arena to maintain. 0 means use sync.Pool with dynamic sizing. Note that <= 20 is needed to pass CI. Controls the memory footprint of rbf.")
flags.Int64Var(&cfg.CursorCacheSize, "rbf.cursor-cache-size", default0.CursorCacheSize, "how big a Cursor arena to maintain. 0 means use sync.Pool with dynamic sizing. Note that <= 20 is needed to pass CI. Controls the memory footprint of rbf.")
}

28
rbf/cursorx.go
View file

@ -19,31 +19,13 @@ import (
"io"
"math"
"os"
"sync/atomic"
"unsafe"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pkg/errors"
)
// if enableRowCache, then we must not return mmap-ed memory
// directly, but only a copy.
var enableRowcache int64 = 1
// SetEnableRowCache should only be called in NewHolder before
// all other reads.
func SetRowcacheOn(on bool) {
if on {
atomic.StoreInt64(&enableRowcache, 1)
} else {
atomic.StoreInt64(&enableRowcache, 0)
}
}
func EnableRowCache() bool {
return atomic.LoadInt64(&enableRowcache) == 1
}
//probably should just implement the container interface
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
@ -192,7 +174,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by
orig := l.Data
var cpMaybe []byte
var mapped bool
if EnableRowCache() || tx.db.cfg.DoAllocZero {
if storage.EnableRowCache() || tx.db.cfg.DoAllocZero {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = target[:len(orig)]
@ -209,7 +191,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by
case ContainerTypeBitmapPtr:
_, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe))
cloneMaybe := bm
if EnableRowCache() {
if storage.EnableRowCache() {
cloneMaybe = (*[1024]uint64)(unsafe.Pointer(&target[0]))[:1024]
copy(cloneMaybe, bm)
}
@ -235,7 +217,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) {
orig := l.Data
var cpMaybe []byte
var mapped bool
if EnableRowCache() || tx.db.cfg.DoAllocZero {
if storage.EnableRowCache() || tx.db.cfg.DoAllocZero {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = make([]byte, len(orig))
@ -252,7 +234,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) {
case ContainerTypeBitmapPtr:
_, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe))
cloneMaybe := bm
if EnableRowCache() {
if storage.EnableRowCache() {
cloneMaybe = make([]uint64, len(bm))
copy(cloneMaybe, bm)
}

21
rbf/db_test.go
View file

@ -24,10 +24,11 @@ import (
"testing"
"time"
_ "net/http/pprof"
"github.com/pilosa/pilosa/v2/rbf"
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"golang.org/x/sync/errgroup"
_ "net/http/pprof"
)
func TestDB_Open(t *testing.T) {
@ -350,17 +351,17 @@ func TestDB_MultiTx(t *testing.T) {
// better diagnosis of deadlocks/hung situations versus just really slow "Quick" tests.
func TestMain(m *testing.M) {
port := getAvailPort()
l, err := net.Listen("tcp", ":0")
if err != nil {
panic(err)
}
port := l.Addr().(*net.TCPAddr).Port
fmt.Printf("rbf/ TestMain: online stack-traces: curl http://localhost:%v/debug/pprof/goroutine?debug=2\n", port)
go func() {
_ = http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", port), nil)
err := http.Serve(l, nil)
if err != nil {
panic(err)
}
}()
os.Exit(m.Run())
}
func getAvailPort() int {
l, _ := net.Listen("tcp", ":0")
r := l.Addr()
l.Close()
return r.(*net.TCPAddr).Port
}

8
rrtx.go
View file

@ -26,10 +26,9 @@ import (
"sync"
"sync/atomic"
"github.com/pilosa/pilosa/v2/rbf"
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
@ -68,7 +67,7 @@ func (tx *RoaringTx) Dump(short bool, shard uint64) {
}
func (tx *RoaringTx) UseRowCache() bool {
return rbf.EnableRowCache()
return storage.EnableRowCache()
}
// based on view.openFragments()
@ -641,8 +640,7 @@ func (r *roaringRegistrar) unregister(w *RoaringWrapper) {
// openRoaringDB will check the registry and make a new instance only
// if one does not exist for its path0. Otherwise it returns
// the existing instance.
func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *rbfcfg.Config) (DBWrapper, error) {
func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool, _ *storage.Config) (DBWrapper, error) {
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[path]

9
rrtx_internal_test.go
View file

@ -15,17 +15,14 @@
package pilosa
import (
"os"
"testing"
)
func TestRoaring_HasData(t *testing.T) {
holder := newHolderWithTempPath(t, "roaring")
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
os.Setenv("PILOSA_TXSRC", "roaring")
idx := newIndexWithTempPath(t, "i")
idx, err := holder.CreateIndex("i", IndexOptions{})
panicOn(err)
defer idx.Close()
db, err := globalRoaringReg.OpenDBWrapper(idx.path, false, nil)

4
scripts/bench_read.sh
View file

@ -29,13 +29,13 @@ do
# Execute RBF/Roaring benchmark.
STARTTIME=$(date +%s)
RBF_PATH=gloat/data/query/${TYPE}/rbf/${DATE}.tar.gz
TXSRC=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH
STORAGE_BACKEND=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH
RBF_ELAPSED=$(($(date +%s) - $STARTTIME))
RBF_LATENCY=$(gloat metric -n -name request_avg_latency "$RBF_PATH")
STARTTIME=$(date +%s)
ROARING_PATH=gloat/data/query/${TYPE}/roaring/${DATE}.tar.gz
TXSRC=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH
STORAGE_BACKEND=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH
ROARING_ELAPSED=$(($(date +%s) - $STARTTIME))
ROARING_LATENCY=$(gloat metric -n -name request_avg_latency "$ROARING_PATH")

Some files were not shown because too many files have changed in this diff Show more