scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into alisharawal-patch-1

Browse source
This commit is contained in:
alisharawal 2020-06-15 11:15:14 -05:00 • committed by GitHub
commit 4a68b46597
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 3611 additions and 1496 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.circleci/config.yml
View file

@ -6,8 +6,12 @@ executors:
version:
type: string
default: "1.14"
resource_class:
type: string
default: medium
docker:
- image: circleci/golang:<< parameters.version >>
resource_class: << parameters.resource_class >>
working_directory: /go/src/github.com/pilosa/pilosa
environment:
GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12.
@ -64,6 +68,9 @@ jobs:
- run: make build GOOS=linux GOARCH=arm64
test:
parameters:
resource_class:
type: string
default: medium
golang_version:
type: string
default: "1.14"
@ -82,6 +89,7 @@ jobs:
executor:
name: golang
version: << parameters.golang_version >>
resource_class: << parameters.resource_class >>
steps:
- attach_workspace:
at: .
@ -191,6 +199,7 @@ workflows:
- test:
name: test-race
test_make_target: test-race
resource_class: xlarge
requires:
- setup
- test:

147
api.go
View file

@ -45,6 +45,7 @@ type API struct {
holder *Holder
cluster *cluster
server *Server
tracker *queryTracker
importWorkersWG sync.WaitGroup
importWorkerPoolSize int
@ -95,6 +96,8 @@ func NewAPI(opts ...apiOption) (*API, error) {
}()
}
api.tracker = newQueryTracker()
return api, nil
}
@ -130,6 +133,7 @@ func (api *API) validate(f apiMethod) error {
func (api *API) Close() error {
close(api.importWork)
api.importWorkersWG.Wait()
api.tracker.Stop()
return nil
}
@ -146,6 +150,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if err != nil {
return QueryResponse{}, errors.Wrap(err, "parsing")
}
defer api.tracker.Finish(api.tracker.Start(req.Query))
execOpts := &execOptions{
Remote: req.Remote,
Profile: req.Profile,
@ -181,11 +186,18 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
if err != nil {
return nil, errors.Wrap(err, "creating index")
}
createdAt := timestamp()
index.mu.Lock()
index.createdAt = createdAt
index.mu.Unlock()
// Send the create index message to all nodes.
err = api.server.SendSync(
&CreateIndexMessage{
Index: indexName,
Meta: &options,
Index: indexName,
CreatedAt: createdAt,
Meta: &options,
})
if err != nil {
return nil, errors.Wrap(err, "sending CreateIndex message")
@ -269,14 +281,18 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
if err != nil {
return nil, errors.Wrap(err, "creating field")
}
createdAt := timestamp()
field.mu.Lock()
field.createdAt = createdAt
field.mu.Unlock()
// Send the create field message to all nodes.
err = api.server.SendSync(
&CreateFieldMessage{
Index: indexName,
Field: fieldName,
Meta: &fo,
})
err = api.server.SendSync(&CreateFieldMessage{
Index: indexName,
Field: fieldName,
CreatedAt: createdAt,
Meta: &fo,
})
if err != nil {
api.server.logger.Printf("problem sending CreateField message: %s", err)
return nil, errors.Wrap(err, "sending CreateField message")
@ -413,14 +429,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return errors.Wrap(err, "validating api method")
}
nodes := api.cluster.shardNodes(indexName, shard)
field := api.holder.Field(indexName, fieldName)
if field == nil {
index, field, err := api.indexField(indexName, fieldName, shard)
if index == nil || field == nil {
return newNotFoundError(ErrFieldNotFound)
}
errCh := make(chan error, len(nodes))
if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return newPreconditionFailedError(err)
}
nodes := api.cluster.shardNodes(indexName, shard)
errCh := make(chan error, len(nodes))
for _, node := range nodes {
node := node
if node.ID == api.server.nodeID {
@ -803,6 +822,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
return errors.Wrap(err, "validating api method")
}
// set CreatedAt for indexes and fields (if empty), and then apply schema.
for _, index := range s.Indexes {
if index.CreatedAt == 0 {
index.CreatedAt = timestamp()
}
for _, field := range index.Fields {
if field.CreatedAt == 0 {
field.CreatedAt = timestamp()
}
}
}
if !remote {
nodes := api.cluster.Nodes()
for i, node := range nodes {
@ -813,7 +844,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
}
}
return api.holder.applySchema(s)
return errors.Wrap(api.holder.applySchema(s), "applying schema")
}
// Views returns the views in the given field.
@ -999,16 +1030,23 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
return errors.Wrap(err, "validating api method")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
// Set up import options.
options, err := setUpImportOptions(opts...)
if err != nil {
return errors.Wrap(err, "setting up import options")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
span.LogKV(
"index", req.Index,
"field", req.Field)
// Unless explicitly ignoring key validation (meaning keys have been
// translated to ids in a previous step at the coordinator node), then
@ -1016,6 +1054,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
if !options.IgnoreKeyCheck {
// Translate row keys.
if field.Keys() {
span.LogKV("rowKeys", true)
if len(req.RowIDs) != 0 {
return errors.New("row ids cannot be used because field uses string keys")
}
@ -1026,6 +1065,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
// Translate column keys.
if index.Keys() {
span.LogKV("columnKeys", true)
if len(req.ColumnIDs) != 0 {
return errors.New("column ids cannot be used because index uses string keys")
}
@ -1110,7 +1150,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
return errors.Wrap(err, "validating api method")
}
if err := req.Validate(); err != nil {
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
}
@ -1120,17 +1165,20 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
return errors.Wrap(err, "setting up import options")
}
index, field, err := api.indexField(req.Index, req.Field, req.Shard)
index, field, err = api.indexField(req.Index, req.Field, req.Shard)
if err != nil {
return errors.Wrap(err, "getting index and field")
}
span.LogKV(
"index", req.Index,
"field", req.Field)
// Unless explicitly ignoring key validation (meaning keys have been
// translate to ids in a previous step at the coordinator node), then
// check to see if keys need translation.
if !options.IgnoreKeyCheck {
// Translate column keys.
if index.Keys() {
span.LogKV("columnKeys", true)
if len(req.ColumnIDs) != 0 {
return errors.New("column ids cannot be used because index uses string keys")
}
@ -1144,6 +1192,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
// the field has a ForeignIndex with keys).
if field.Keys() {
// Perform translation.
span.LogKV("rowKeys", true)
uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues)
if err != nil {
return err
@ -1257,6 +1306,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq
return errors.Wrap(err, "validating shard ownership")
}
if req.IndexCreatedAt != 0 {
if index.CreatedAt() != req.IndexCreatedAt {
return ErrPreconditionFailed
}
}
bulkAttrs := make(map[uint64]map[string]interface{})
for n := 0; n < len(req.ColumnIDs); n++ {
bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}
@ -1594,14 +1649,41 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
if err := api.validate(apiStartTransaction); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
return api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
if exclusive {
switch err {
case nil:
api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
case ErrTransactionExclusive:
api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
}
if t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
}
} else {
switch err {
case nil:
api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
case ErrTransactionExclusive:
api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
}
}
return t, err
}
func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) {
if err := api.validate(apiFinishTransaction); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
return api.server.FinishTransaction(ctx, id, remote)
t, err := api.server.FinishTransaction(ctx, id, remote)
if err == nil {
if t.Exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
} else {
api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
}
}
return t, err
}
func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) {
@ -1615,7 +1697,20 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr
if err := api.validate(apiGetTransaction); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
return api.server.GetTransaction(ctx, id, remote)
t, err := api.server.GetTransaction(ctx, id, remote)
if err == nil {
if t.Exclusive && t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
}
}
return t, err
}
func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) {
if err := api.validate(apiActiveQueries); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
return api.tracker.ActiveQueries(), nil
}
type serverInfo struct {
@ -1669,6 +1764,7 @@ const (
apiFinishTransaction
apiTransactions
apiGetTransaction
apiActiveQueries
)
var methodsCommon = map[apiMethod]struct{}{
@ -1708,4 +1804,5 @@ var methodsNormal = map[apiMethod]struct{}{
apiFinishTransaction: {},
apiTransactions: {},
apiGetTransaction: {},
apiActiveQueries: {},
}

81
api_test.go
View file

@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
m1 := c[1]
t.Run("ImportColumnAttrs", func(t *testing.T) {
ctx := context.Background()
index := "i"
field := "f"
indexName := "i"
fieldName := "f"
attrKey := "k"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field)
_, err = m0.API.CreateField(ctx, indexName, fieldName)
if err != nil {
t.Fatalf("creating field: %v", err)
}
@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
columnIDs0 = append(columnIDs0, uint64(n))
val0 := attrFun(uint64(n))
attrVals0 = append(attrVals0, val0)
setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field)
if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil {
setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
t.Fatal(err)
}
columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
val1 := attrFun(uint64(n + ShardWidth))
attrVals1 = append(attrVals1, val1)
setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field)
if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil {
setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
t.Fatal(err)
}
}
// send shard0 to node1
req := &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs0,
AttrVals: attrVals0,
Shard: 0,
Index: index,
AttrKey: attrKey,
ColumnIDs: columnIDs0,
AttrVals: attrVals0,
Shard: 0,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
// send shard1 to node0
req = &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs1,
AttrVals: attrVals1,
Shard: 1,
Index: index,
AttrKey: attrKey,
ColumnIDs: columnIDs1,
AttrVals: attrVals1,
Shard: 1,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
}
// Query node0.
pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
}
}
// Query node1.
pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field)
res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
@ -186,17 +188,24 @@ func TestAPI_Import(t *testing.T) {
t.Run("RowIDColumnKey", func(t *testing.T) {
ctx := context.Background()
index := "rick"
field := "f"
indexName := "rick"
fieldName := "f"
_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true})
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if index.CreatedAt() == 0 {
t.Fatal("index createdAt is empty")
}
field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if field.CreatedAt() == 0 {
t.Fatal("field createdAt is empty")
}
rowID := uint64(1)
timestamp := int64(0)
@ -215,21 +224,23 @@ func TestAPI_Import(t *testing.T) {
// Import data with keys to the coordinator (node0) and verify that it gets
// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
req := &pilosa.ImportRequest{
Index: index,
Field: field,
Shard: 0,
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
Field: fieldName,
FieldCreatedAt: field.CreatedAt(),
Shard: 0,
RowIDs: rowIDs,
ColumnKeys: colKeys,
Timestamps: timestamps,
}
if err := m0.API.Import(ctx, req); err != nil {
t.Fatal(err)
}
pql := fmt.Sprintf("Row(%s=%d)", field, rowID)
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
t.Fatal(err)
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
t.Fatalf("unexpected column keys: %#v", keys)
@ -237,7 +248,7 @@ func TestAPI_Import(t *testing.T) {
// Query node1.
if err := test.RetryUntil(5*time.Second, func() error {
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil {
return err
} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
return fmt.Errorf("unexpected column keys: %#v", keys)

66
cluster.go
View file

@ -62,9 +62,8 @@ const (
resizeJobActionAdd = "ADD"
resizeJobActionRemove = "REMOVE"
confirmDownRetries = 10
confirmDownSleep = 1
confirmDownTimeout = 2
defaultConfirmDownRetries = 10
defaultConfirmDownSleep = 1 * time.Second
)
// Node represents a node in the cluster.
@ -239,6 +238,9 @@ type cluster struct { // nolint: maligned
logger logger.Logger
InternalClient InternalClient
confirmDownRetries int
confirmDownSleep time.Duration
}
// newCluster returns a new instance of Cluster with defaults.
@ -258,6 +260,9 @@ func newCluster() *cluster {
InternalClient: newNopInternalClient(),
logger: logger.NopLogger,
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
}
}
@ -610,6 +615,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus {
ClusterID: c.id,
State: c.state,
Nodes: c.nodes,
Schema: &Schema{Indexes: c.holder.Schema()},
}
}
@ -659,6 +665,7 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool {
n.State = node.State
n.IsCoordinator = node.IsCoordinator
n.URI = node.URI
n.GRPCURI = node.GRPCURI
return true
}
return false
@ -1042,6 +1049,11 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
func (c *cluster) ownsPartition(nodeID string, partition int) bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.unprotectedOwnsPartition(nodeID, partition)
}
// unprotectedOwnsPartition returns true if a host owns a partition.
func (c *cluster) unprotectedOwnsPartition(nodeID string, partition int) bool {
return Nodes(c.partitionNodes(partition)).ContainsID(nodeID)
}
@ -1271,9 +1283,7 @@ func (c *cluster) listenForJoins() {
// Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events.
// We use a bool `setNormal` to indicate when at least one node has joined.
var setNormal bool
for {
// Handle all pending joins before changing state back to NORMAL.
select {
case nodeAction := <-c.joiningLeavingNodes:
@ -1745,8 +1755,9 @@ func (j *resizeJob) distributeResizeInstructions() error {
// Because the node may not be in the cluster yet, create
// a dummy node object to use in the SendTo() method.
node := &Node{
ID: instr.Node.ID,
URI: instr.Node.URI,
ID: instr.Node.ID,
URI: instr.Node.URI,
GRPCURI: instr.Node.GRPCURI,
}
j.Logger.Printf("send resize instructions: %v", instr)
if err := j.Broadcaster.SendTo(node, instr); err != nil {
@ -1917,7 +1928,7 @@ func (c *cluster) considerTopology() error {
// band aid to protect against false nodeLeave events from memberlist
// the test is the lightest weight endpoint of the node in question /version
// TODO provide more robust solution to false nodeLeave events
func confirmNodeDown(uri URI, log logger.Logger) bool {
func (c *cluster) confirmNodeDown(uri URI) bool {
u := url.URL{
Scheme: uri.Scheme,
Host: uri.HostPort(),
@ -1925,11 +1936,11 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
log.Printf("bad request:%s %s", u.String(), err)
c.logger.Printf("bad request:%s %s", u.String(), err)
return false
}
for i := 0; i < confirmDownRetries; i++ {
ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second)
for i := 0; i < c.confirmDownRetries; i++ {
ctx, cancel := context.WithTimeout(context.Background(), c.confirmDownSleep*2)
defer cancel()
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
var bod []byte
@ -1940,8 +1951,8 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
}
}
log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
time.Sleep(confirmDownSleep * time.Second)
c.logger.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
time.Sleep(c.confirmDownSleep)
}
return true
}
@ -1969,7 +1980,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
// not already removed by a removeNode request. We treat this as the
// host being temporarily unavailable, and expect it to come back
// up.
if confirmNodeDown(e.Node.URI, c.logger) {
if c.confirmNodeDown(e.Node.URI) {
if c.removeNodeBasicSorted(e.Node.ID) {
c.Topology.nodeStates[e.Node.ID] = nodeStateDown
// put the cluster into STARTING if we've lost a number of nodes
@ -2046,6 +2057,9 @@ func (c *cluster) nodeJoin(node *Node) error {
c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI)
cnode.URI = node.URI
}
if cnode.GRPCURI != node.GRPCURI {
cnode.GRPCURI = node.GRPCURI
}
return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
}
@ -2141,7 +2155,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
}
var availableShards *roaring.Bitmap
for _, idx := range ns.Schema.Indexes {
is := &IndexStatus{Name: idx.Name}
is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
for _, f := range idx.Fields {
if field := c.holder.Field(idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards()
@ -2150,6 +2164,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
}
is.Fields = append(is.Fields, &FieldStatus{
Name: f.Name,
CreatedAt: f.CreatedAt,
AvailableShards: availableShards,
})
}
@ -2445,6 +2460,7 @@ type ClusterStatus struct {
ClusterID string
State string
Nodes []*Node
Schema *Schema
}
// ResizeInstruction contains the instruction provided to a node
@ -2486,7 +2502,7 @@ type translationResizeNode struct {
// Schema contains information about indexes and their configuration.
type Schema struct {
Indexes []*IndexInfo
Indexes []*IndexInfo `json:"indexes"`
}
func encodeTopology(topology *Topology) *internal.Topology {
@ -2524,8 +2540,9 @@ type CreateShardMessage struct {
// CreateIndexMessage is an internal message indicating index creation.
type CreateIndexMessage struct {
Index string
Meta *IndexOptions
Index string
CreatedAt int64
Meta *IndexOptions
}
// DeleteIndexMessage is an internal message indicating index deletion.
@ -2535,9 +2552,10 @@ type DeleteIndexMessage struct {
// CreateFieldMessage is an internal message indicating field creation.
type CreateFieldMessage struct {
Index string
Field string
Meta *FieldOptions
Index string
Field string
CreatedAt int64
Meta *FieldOptions
}
// DeleteFieldMessage is an internal message indicating field deletion.
@ -2600,13 +2618,15 @@ type NodeStatus struct {
// IndexStatus is an internal message representing the contents of an index.
type IndexStatus struct {
Name string
Fields []*FieldStatus
Name string
CreatedAt int64
Fields []*FieldStatus
}
// FieldStatus is an internal message representing the contents of a field.
type FieldStatus struct {
Name string
CreatedAt int64
AvailableShards *roaring.Bitmap
}

21
cluster_internal_test.go
View file

@ -943,18 +943,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
t.Error(err)
}
uri.Port = uint16(iport)
if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
c := newCluster()
c.logger = logger.NewVerboseLogger(os.Stdout)
if c.confirmNodeDown(uri) {
t.Errorf("expected node to be up")
}
}
func TestCluster_confirmNodeDownTimeout(t *testing.T) {
sleep := 50 * time.Millisecond
retries := 5
if testing.Short() {
t.Skip()
}
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
time.Sleep(sleep * time.Duration(retries))
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
@ -973,8 +977,11 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
t.Error(err)
}
uri.Port = uint16(iport)
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
c := newCluster()
c.confirmDownSleep = sleep
c.confirmDownRetries = retries
c.logger = logger.NewVerboseLogger(os.Stdout)
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}
@ -987,8 +994,12 @@ func TestCluster_confirmNodeDownDown(t *testing.T) {
uri.Scheme = "http"
uri.Host = "DoesntMatter"
uri.Port = 6666
c := newCluster()
c.confirmDownSleep = 50 * time.Millisecond
c.confirmDownRetries = 5
c.logger = logger.NewVerboseLogger(os.Stdout)
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}

176
docs/api-reference.md
View file

@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user
```
``` response
{
"fields": [
{
"name": "event",
"options": {
"keys": false,
"timeQuantum": "YMD",
"type": "time"
}
}
],
"name": "user",
"options": {
"keys": false,
"trackExistence": true
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
}
],
"shardWidth": 1048576
}
```
@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option
curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}'
```
``` response
{"success":true}
{"success":true,"name":"user","createdAt":1591179042178854000}
```
### Remove index
@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they
exist must also contain the same number of items as rows and columns. The
column IDs must all be in the shard specified in the request.
Some endpoints and data structures include a `CreatedAt` fields.
This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field,
but to serve as a unique identifier for use in cache invalidation.
The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk))
can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached.
This is true except in cases where an index or field gets deleted and then recreated,
or if Pilosa is restored from a backup.
So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted),
and the ingester will know that it needs to drop its cache.
```
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
repeated int64 Timestamps = 6;
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated int64 Timestamps = 6;
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
int64 IndexCreatedAt = 9;
int64 FieldCreatedAt = 10;
}
```
### Create field
`POST /index/<index-name>/field/<field-name>`
@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \
-d '{"options": {"type": "int", "min": -1000, "max":2000}}'
```
``` response
{"success":true}
{"success":true,"name":"quantity","createdAt":1591180110914425000}
```
Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`.
@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit,
curl localhost:10101/index/user/field/language -X POST
```
``` response
{"success":true}
{"success":true,"name":"language","createdAt":1591180128294321000}
```
``` request
curl localhost:10101/index/repository/field/stats \
-X POST \
-d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}'
-d '{"options":{"type": "int", "min": 0, "max": 1000000}}'
```
``` response
{"success":true}
{"success":true,"name":"stats","createdAt":1591180737881627000}
```
### Remove field
@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema
```
``` response
{
"indexes": [
"indexes": [
{
"name": "user",
"createdAt": 1591178953061239000,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"fields": [
{
"name": "event",
"options": {
"keys": false,
"timeQuantum": "YMD",
"type": "time"
}
},
{
"name": "language",
"options": {
"cacheSize": 50000,
"cacheType": "ranked",
"keys": false,
"type": "set"
}
}
],
"name": "user",
"options": {
"keys": false,
"trackExistence": true
}
"name": "event",
"createdAt": 1591178962332452000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "language",
"createdAt": 1591180128294321000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": false
}
},
{
"name": "quantity",
"createdAt": 1591180110914425000,
"options": {
"type": "int",
"base": 0,
"bitDepth": 0,
"min": -1000,
"max": 2000,
"keys": false,
"foreignIndex": ""
}
}
]
],
"shardWidth": 1048576
}
]
}
```
@ -304,7 +340,7 @@ Returns the version of the Pilosa server.
curl -XGET localhost:10101/version
```
``` response
{"version":"v0.6.0"}
{"version":"2.0.0-alpha.20-6-gb9d8d6b4"}
```
### Get status
@ -318,19 +354,25 @@ curl -XGET localhost:10101/status
```
```response
{
"localID": "d3369125-29d8-4305-a351-b4474d14a542",
"nodes": [
{
"id": "d3369125-29d8-4305-a351-b4474d14a542",
"isCoordinator": true,
"uri": {
"host": "localhost",
"port": 10101,
"scheme": "http"
}
}
],
"state": "NORMAL"
"state": "NORMAL",
"nodes": [
{
"id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9",
"uri": {
"scheme": "http",
"host": "localhost",
"port": 10101
},
"grpc-uri": {
"scheme": "http",
"host": "localhost",
"port": 20101
},
"isCoordinator": true,
"state": "READY"
}
],
"localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9"
}
```

107
encoding/proto/proto.go
View file

@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp
func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest {
return &internal.ImportRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
RowIDs: m.RowIDs,
ColumnIDs: m.ColumnIDs,
RowKeys: m.RowKeys,
ColumnKeys: m.ColumnKeys,
Timestamps: m.Timestamps,
Index: m.Index,
Field: m.Field,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Shard: m.Shard,
RowIDs: m.RowIDs,
ColumnIDs: m.ColumnIDs,
RowKeys: m.RowKeys,
ColumnKeys: m.ColumnKeys,
Timestamps: m.Timestamps,
}
}
func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest {
return &internal.ImportValueRequest{
Index: m.Index,
Field: m.Field,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
StringValues: m.StringValues,
Index: m.Index,
Field: m.Field,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Shard: m.Shard,
ColumnIDs: m.ColumnIDs,
ColumnKeys: m.ColumnKeys,
Values: m.Values,
FloatValues: m.FloatValues,
StringValues: m.StringValues,
}
}
@ -438,20 +442,23 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *
i++
}
return &internal.ImportRoaringRequest{
Clear: m.Clear,
Action: m.Action,
Block: uint64(m.Block),
Views: views,
IndexCreatedAt: m.IndexCreatedAt,
FieldCreatedAt: m.FieldCreatedAt,
Clear: m.Clear,
Action: m.Action,
Block: uint64(m.Block),
Views: views,
}
}
func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest {
return &internal.ImportColumnAttrsRequest{
Index: m.Index,
Shard: m.Shard,
AttrKey: m.AttrKey,
AttrVals: m.AttrVals,
ColumnIDs: m.ColumnIDs,
Index: m.Index,
IndexCreatedAt: m.IndexCreatedAt,
Shard: m.Shard,
AttrKey: m.AttrKey,
AttrVals: m.AttrVals,
ColumnIDs: m.ColumnIDs,
}
}
@ -593,9 +600,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index
func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index {
return &internal.Index{
Name: idx.Name,
Options: s.encodeIndexMeta(&idx.Options),
Fields: s.encodeFieldInfos(idx.Fields),
Name: idx.Name,
CreatedAt: idx.CreatedAt,
Options: s.encodeIndexMeta(&idx.Options),
Fields: s.encodeFieldInfos(idx.Fields),
}
}
@ -609,9 +617,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field {
func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field {
ifield := &internal.Field{
Name: f.Name,
Meta: s.encodeFieldOptions(&f.Options),
Views: make([]string, 0, len(f.Views)),
Name: f.Name,
CreatedAt: f.CreatedAt,
Meta: s.encodeFieldOptions(&f.Options),
Views: make([]string, 0, len(f.Views)),
}
for _, viewinfo := range f.Views {
@ -655,6 +664,7 @@ func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node {
URI: s.encodeURI(n.URI),
IsCoordinator: n.IsCoordinator,
State: n.State,
GRPCURI: s.encodeURI(n.GRPCURI),
}
}
@ -671,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust
State: m.State,
ClusterID: m.ClusterID,
Nodes: s.encodeNodes(m.Nodes),
Schema: s.encodeSchema(m.Schema),
}
}
@ -684,8 +695,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte
func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage {
return &internal.CreateIndexMessage{
Index: m.Index,
Meta: s.encodeIndexMeta(m.Meta),
Index: m.Index,
CreatedAt: m.CreatedAt,
Meta: s.encodeIndexMeta(m.Meta),
}
}
@ -704,9 +716,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte
func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage {
return &internal.CreateFieldMessage{
Index: m.Index,
Field: m.Field,
Meta: s.encodeFieldOptions(m.Meta),
Index: m.Index,
Field: m.Field,
CreatedAt: m.CreatedAt,
Meta: s.encodeFieldOptions(m.Meta),
}
}
@ -785,8 +798,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus
func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus {
return &internal.IndexStatus{
Name: m.Name,
Fields: s.encodeFieldStatuses(m.Fields),
Name: m.Name,
CreatedAt: m.CreatedAt,
Fields: s.encodeFieldStatuses(m.Fields),
}
}
@ -801,6 +815,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind
func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus {
return &internal.FieldStatus{
Name: m.Name,
CreatedAt: m.CreatedAt,
AvailableShards: m.AvailableShards.Slice(),
}
}
@ -937,6 +952,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo)
func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) {
m.Name = idx.Name
m.CreatedAt = idx.CreatedAt
m.Options = pilosa.IndexOptions{}
s.decodeIndexMeta(idx.Options, &m.Options)
m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields))
@ -952,6 +968,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) {
func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) {
m.Name = f.Name
m.CreatedAt = f.CreatedAt
m.Options = pilosa.FieldOptions{}
s.decodeFieldOptions(f.Meta, &m.Options)
m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views))
@ -991,11 +1008,14 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl
m.ClusterID = cs.ClusterID
m.Nodes = make([]*pilosa.Node, len(cs.Nodes))
s.decodeNodes(cs.Nodes, m.Nodes)
m.Schema = &pilosa.Schema{}
s.decodeSchema(cs.Schema, m.Schema)
}
func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) {
m.ID = node.ID
s.decodeURI(node.URI, &m.URI)
s.decodeURI(node.GRPCURI, &m.GRPCURI)
m.IsCoordinator = node.IsCoordinator
m.State = node.State
}
@ -1014,6 +1034,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m
func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) {
m.Index = pb.Index
m.CreatedAt = pb.CreatedAt
m.Meta = &pilosa.IndexOptions{}
s.decodeIndexMeta(pb.Meta, m.Meta)
}
@ -1032,6 +1053,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m
func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) {
m.Index = pb.Index
m.Field = pb.Field
m.CreatedAt = pb.CreatedAt
m.Meta = &pilosa.FieldOptions{}
s.decodeFieldOptions(pb.Meta, m.Meta)
}
@ -1105,6 +1127,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind
func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) {
m.Name = pb.Name
m.CreatedAt = pb.CreatedAt
m.Fields = s.decodeFieldStatuses(pb.Fields)
}
@ -1119,6 +1142,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie
func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) {
m.Name = pb.Name
m.CreatedAt = pb.CreatedAt
m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...)
}
@ -1147,6 +1171,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im
m.RowKeys = pb.RowKeys
m.ColumnKeys = pb.ColumnKeys
m.Timestamps = pb.Timestamps
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) {
@ -1158,6 +1184,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m
m.Values = pb.Values
m.FloatValues = pb.FloatValues
m.StringValues = pb.StringValues
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) {
@ -1169,10 +1197,13 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest
m.Action = pb.Action
m.Block = int(pb.Block)
m.Views = views
m.IndexCreatedAt = pb.IndexCreatedAt
m.FieldCreatedAt = pb.FieldCreatedAt
}
func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) {
m.Index = pb.Index
m.IndexCreatedAt = pb.IndexCreatedAt
m.Shard = pb.Shard
m.AttrKey = pb.AttrKey
m.AttrVals = pb.AttrVals

264
executor.go
View file

@ -59,6 +59,8 @@ type executor struct {
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
shutdown bool
workMu sync.RWMutex
workersWG sync.WaitGroup
workerPoolSize int
work chan job
@ -115,6 +117,9 @@ func newExecutor(opts ...executorOption) *executor {
}
func (e *executor) Close() error {
e.workMu.Lock()
defer e.workMu.Unlock()
e.shutdown = true
close(e.work)
e.workersWG.Wait()
return nil
@ -155,6 +160,7 @@ func (e *executor) registerOps(ops []ext.BitmapOp) error {
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
span.LogKV("pql", q.String())
defer span.Finish()
resp := QueryResponse{}
@ -357,7 +363,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call
row = r.Pos
default:
return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v)
}
if err := ctx.Err(); err != nil {
return err
}
c.Children = []*pql.Call{}
c.Name = "Precomputed"
@ -376,6 +384,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call
// handlePreCallChildren handles any pre-calls in the children of a given call.
func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
for i := range c.Children {
if err := ctx.Err(); err != nil {
return err
}
if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil {
return err
}
@ -383,6 +394,9 @@ func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *p
for _, val := range c.Args {
// Handle Call() operations which exist inside named arguments, too.
if call, ok := val.(*pql.Call); ok {
if err := ctx.Err(); err != nil {
return err
}
if err := e.handlePreCalls(ctx, index, call, shards, opt); err != nil {
return err
}
@ -649,12 +663,12 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string,
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeIncludesColumnCallShard(ctx, index, c, shard, col)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(bool)
return other || v.(bool)
}
@ -704,12 +718,12 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p
shard := colID / ShardWidth
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeFieldValueCallShard(ctx, field, colID, shard)
}
// Select single returned result at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
if other.Count == 1 {
return other
@ -833,12 +847,15 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call
// using the executor.mapReduce() method.
func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeAllCallShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
if err := ctx.Err(); err != nil {
return err
}
other, _ := prev.(*Row)
if other == nil {
other = NewRow()
@ -888,12 +905,12 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeSumCountShard(ctx, index, c, nil, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.add(v.(ValCount))
}
@ -940,13 +957,16 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeGenericFieldShard(ctx, index, c, op, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(SignedRow)
if err := ctx.Err(); err != nil {
return err
}
return other.union(v.(SignedRow))
}
@ -973,12 +993,12 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeMinShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.smaller(v.(ValCount))
}
@ -1009,12 +1029,12 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeMaxShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(ValCount)
return other.larger(v.(ValCount))
}
@ -1041,12 +1061,12 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call,
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeMinRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
// if minRowID exists, and if it is smaller than the other one return it.
// otherwise return the minRowID of the one which exists.
if prev == nil {
@ -1080,12 +1100,12 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call,
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeMaxRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
// if minRowID exists, and if it is smaller than the other one return it.
// otherwise return the minRowID of the one which exists.
if prev == nil {
@ -1124,6 +1144,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, index string, c *
// executeBitmapCall executes a call that returns a bitmap.
func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
span.LogKV("pqlCallName", c.Name)
defer span.Finish()
indexTag := "index:" + index
@ -1136,16 +1157,19 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeBitmapCallShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(*Row)
if other == nil {
other = NewRow()
}
if err := ctx.Err(); err != nil {
return err
}
other.Merge(v.(*Row))
return other
}
@ -1503,12 +1527,12 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C
defer span.Finish()
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeTopNShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(*PairsField)
vpf, _ := v.(*PairsField)
if other == nil {
@ -1516,6 +1540,9 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C
} else if vpf == nil {
return other
}
if err := ctx.Err(); err != nil {
return err
}
other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs)
return other
}
@ -1743,10 +1770,16 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
return nil, err
}
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
// perform necessary Rows queries (any that have limit or columns args) -
// TODO, call async? would only help if multiple Rows queries had a column
// or limit arg.
// TODO support TopN in here would be really cool - and pretty easy I think.
bases := make(map[int]int64)
childRows := make([]RowIDs, len(c.Children))
for i, child := range c.Children {
// Check "field" first for backwards compatibility, then set _field.
@ -1766,6 +1799,19 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
if err != nil {
return nil, errors.Wrap(err, "getting column")
}
fieldName, ok := child.Args["_field"].(string)
if !ok {
return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", child.Name, child.Args["_field"])
}
f := idx.Field(fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
switch f.Type() {
case FieldTypeInt:
bases[i] = f.bsiGroup(f.name).Base
}
if hasLimit || hasCol { // we need to perform this query cluster-wide ahead of executeGroupByShard
childRows[i], err = e.executeRows(ctx, index, child, shards, opt)
if err != nil {
@ -1778,12 +1824,15 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
return e.executeGroupByShard(ctx, index, c, filter, shard, childRows)
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeGroupByShard(ctx, index, c, filter, shard, childRows, bases)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.([]GroupCount)
if err := ctx.Err(); err != nil {
return err
}
return mergeGroupCounts(other, v.([]GroupCount), limit)
}
// Get full result set.
@ -2127,7 +2176,7 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit
return gcs[:i]
}
func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) {
func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard")
defer span.Finish()
@ -2175,6 +2224,13 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql
}
}
// Apply bases.
for i, base := range bases {
for _, r := range results {
*r.Group[i].Value += base
}
}
return results, nil
}
@ -2197,7 +2253,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeRowsShard(ctx, index, fieldName, c, shard)
}
@ -2210,8 +2266,11 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(RowIDs)
if err := ctx.Err(); err != nil {
return err
}
return other.merge(v.(RowIDs), limit)
}
// Get full result set.
@ -2223,7 +2282,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s
return results, nil
}
func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) {
func (e *executor) executeRowsShard(ctx context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) {
// Fetch index.
idx := e.Holder.Index(index)
if idx == nil {
@ -2333,12 +2392,15 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
}
for _, view := range views {
if err := ctx.Err(); err != nil {
return nil, err
}
frag := e.Holder.fragment(index, fieldName, view, shard)
if frag == nil {
continue
}
viewRows := frag.rows(start, filters...)
viewRows := frag.rows(ctx, start, filters...)
rowIDs = rowIDs.merge(viewRows, limit)
}
@ -2463,21 +2525,15 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c
return nil, ErrFieldNotFound
}
// EQ null (not implemented: flip frag.NotNull with max ColumnID)
// EQ null _exists - frag.NotNull()
// NEQ null frag.NotNull()
// BETWEEN a,b(in) BETWEEN/frag.RowBetween()
// BETWEEN a,b(out) BETWEEN/frag.NotNull()
// EQ <int> frag.RangeOp
// NEQ <int> frag.RangeOp
// Handle `!= null`.
// Handle `!= null` and `== null`.
if cond.Op == pql.NEQ && cond.Value == nil {
// Find bsiGroup.
bsig := f.bsiGroup(fieldName)
if bsig == nil {
return nil, ErrBSIGroupNotFound
}
// Retrieve fragment.
frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if frag == nil {
@ -2486,6 +2542,37 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c
return frag.notNull()
} else if cond.Op == pql.EQ && cond.Value == nil {
// Make sure the index supports existence tracking.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
} else if idx.existenceField() == nil {
return nil, errors.Errorf("index does not support existence tracking: %s", index)
}
var existenceRow *Row
existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard)
if existenceFrag == nil {
existenceRow = NewRow()
} else {
existenceRow = existenceFrag.row(0)
}
var notNull *Row
var err error
// Retrieve notNull from fragment if it exists.
if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil {
if notNull, err = frag.notNull(); err != nil {
return nil, errors.Wrap(err, "getting fragment not null")
}
} else {
notNull = NewRow()
}
return existenceRow.Difference(notNull), nil
} else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT ||
cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE {
predicates, err := getCondIntSlice(f, cond)
@ -2787,7 +2874,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return 0, err
@ -2796,7 +2883,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
@ -2822,7 +2909,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call,
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
if err != nil {
return 0, err
@ -2831,7 +2918,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call,
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
@ -2944,12 +3031,12 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeClearRowShard(ctx, index, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
val := v.(bool)
if prev == nil {
return val
@ -3032,12 +3119,12 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(shard uint64) (interface{}, error) {
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeSetRowShard(ctx, indexName, c, shard)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
val := v.(bool)
if prev == nil {
return val
@ -3046,7 +3133,15 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C
}
result, err := e.mapReduce(ctx, indexName, shards, c, opt, mapFn, reduceFn)
return result.(bool), err
if err != nil {
return false, err
}
b, ok := result.(bool)
if !ok {
return false, errors.New("unsupported result type")
}
return b, nil
}
// executeSetRowShard executes a SetRow() call for a single shard.
@ -3566,7 +3661,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
}
// Start mapping across all primary owners.
if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil {
if err := e.mapper(ctx, cancel, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil {
return nil, errors.Wrap(err, "starting mapper")
}
@ -3586,7 +3681,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
if err := e.mapper(ctx, cancel, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable {
return nil, resp.err
} else if err != nil {
return nil, errors.Wrap(err, "calling mapper")
@ -3595,7 +3690,11 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64,
}
// Reduce value.
result = reduceFn(result, resp.result)
result = reduceFn(ctx, result, resp.result)
if err, ok := result.(error); ok {
cancel()
return nil, err
}
// If all shards have been processed then return.
shardN += len(resp.shards)
@ -3643,9 +3742,10 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row {
return newRows
}
func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper")
defer span.Finish()
done := ctx.Done()
// Group shards together by nodes.
m, err := e.shardsByNode(nodes, index, shards)
@ -3672,11 +3772,16 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
}
resp.err = err
}
// Return response to the channel.
select {
case <-ctx.Done():
case <-done:
case ch <- resp:
// The cancel coming after the above send is intentional.
// We want to report the actual error that happened
// before we cause anything to return "context canceled".
if resp.err != nil {
cancel()
}
}
}(n, nodeShards)
}
@ -3693,7 +3798,7 @@ type job struct {
func worker(work chan job) {
for j := range work {
result, err := j.mapFn(j.shard)
result, err := j.mapFn(j.ctx, j.shard)
select {
case <-j.ctx.Done():
@ -3702,10 +3807,21 @@ func worker(work chan job) {
}
}
var errShutdown = errors.New("executor has shut down")
// mapperLocal performs map & reduce entirely on the local node.
func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal")
defer span.Finish()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
done := ctx.Done()
e.workMu.RLock()
defer e.workMu.RUnlock()
if e.shutdown {
return nil, errShutdown
}
ch := make(chan mapResponse, len(shards))
@ -3723,13 +3839,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu
var result interface{}
for {
select {
case <-ctx.Done():
case <-done:
return nil, ctx.Err()
case resp := <-ch:
if resp.err != nil {
return nil, resp.err
}
result = reduceFn(result, resp.result)
result = reduceFn(ctx, result, resp.result)
if err, ok := result.(error); ok {
cancel()
return nil, err
}
maxShard++
}
@ -4241,9 +4361,9 @@ func validateQueryContext(ctx context.Context) error {
// errShardUnavailable is a marker error if no nodes are available.
var errShardUnavailable = errors.New("shard unavailable")
type mapFunc func(shard uint64) (interface{}, error)
type mapFunc func(ctx context.Context, shard uint64) (interface{}, error)
type reduceFunc func(prev, v interface{}) interface{}
type reduceFunc func(ctx context.Context, prev, v interface{}) interface{}
type mapResponse struct {
node *Node
@ -4743,18 +4863,21 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal
// nextAtIdx is a recursive helper method for getting the next row for the field
// at index i, and then updating the rows in the "higher" fields if it wraps.
func (gbi *groupByIterator) nextAtIdx(i int) {
func (gbi *groupByIterator) nextAtIdx(ctx context.Context, i int) (err error) {
// loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed.
for {
if err = ctx.Err(); err != nil {
return err
}
nr, rowID, value, wrapped := gbi.rowIters[i].Next()
if nr == nil {
gbi.done = true
return
return nil
}
if wrapped && i != 0 {
gbi.nextAtIdx(i - 1)
if gbi.done {
return
err = gbi.nextAtIdx(ctx, i-1)
if gbi.done || err != nil {
return err
}
}
if i == 0 && gbi.filter != nil {
@ -4771,6 +4894,7 @@ func (gbi *groupByIterator) nextAtIdx(i int) {
break
}
}
return nil
}
// Next returns a GroupCount representing the next group by record. When there
@ -4778,6 +4902,9 @@ func (gbi *groupByIterator) nextAtIdx(i int) {
func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool, err error) {
// loop until we find a result with count > 0
for {
if err := ctx.Err(); err != nil {
return ret, false, err
}
if gbi.done {
return ret, true, nil
}
@ -4805,7 +4932,10 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
}
}
if ret.Count == 0 {
gbi.nextAtIdx(len(gbi.rows) - 1)
err := gbi.nextAtIdx(ctx, len(gbi.rows)-1)
if err != nil {
return ret, false, err
}
continue
}
break
@ -4820,9 +4950,9 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
}
// set up for next call
gbi.nextAtIdx(len(gbi.rows) - 1)
err = gbi.nextAtIdx(ctx, len(gbi.rows)-1)
return ret, false, nil
return ret, false, err
}
// getCondIntSlice looks at the field, the cond op type (which is

83
executor_test.go
View file

@ -2371,7 +2371,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
@ -2414,6 +2414,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) {
}
t.Run("EQ", func(t *testing.T) {
// EQ null
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{1,
50,
ShardWidth,
ShardWidth + 1,
ShardWidth + 2,
(5 * ShardWidth) + 100,
}, result.Results[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns())
}
// EQ <int>
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil {
t.Fatal(err)
} else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) {
@ -2983,6 +2996,37 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
test.CheckGroupBy(t, expected, results)
}
})
t.Run("groupBy on ints with offset regression", func(t *testing.T) {
_, err = c[0].API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000))
if err != nil {
t.Fatalf("creating field: %v", err)
}
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set(0, hint=1)
Set(1, hint=2)
Set(2, hint=3)
`}); err != nil {
t.Fatalf("querying remote: %v", err)
}
if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i",
Query: `GroupBy(Rows(hint))`,
}); err != nil {
t.Fatalf("GroupBy querying: %v", err)
} else {
var a, b, c int64 = 1, 2, 3
expected := []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "hint", Value: &a}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "hint", Value: &b}}, Count: 1},
{Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1},
}
results := res.Results[0].([]pilosa.GroupCount)
test.CheckGroupBy(t, expected, results)
}
})
}
// Ensure executor returns an error if too many writes are in a single request.
@ -3463,15 +3507,15 @@ func TestExecutor_Execute_All(t *testing.T) {
t.Fatal(err)
}
// Create an import request that sets a full shard,
// plus a couple bits set on either side of it, and
// a final bit set in a fourth shard.
// Create an import request that sets things on either end
// of a shard, plus a couple bits set on either side of it,
// and a final bit set in a fourth shard.
//
// shard0 shard1 shard2 shard3
// |----------|----------|----------|----------|
// | **|**********|** | *
// | **|** **|** | *
//
bitCount := ShardWidth + 5
bitCount := 100 + 5
req := &pilosa.ImportRequest{
Index: index.Name(),
Field: fld.Name(),
@ -3479,10 +3523,14 @@ func TestExecutor_Execute_All(t *testing.T) {
RowIDs: make([]uint64, bitCount),
ColumnIDs: make([]uint64, bitCount),
}
for i := 0; i < bitCount-1; i++ {
for i := 0; i < bitCount/2; i++ {
req.RowIDs[i] = 10
req.ColumnIDs[i] = uint64(i + ShardWidth - 2)
}
for i := bitCount / 2; i < bitCount-1; i++ {
req.RowIDs[i] = 10
req.ColumnIDs[i] = uint64(i + (ShardWidth * 2) - bitCount + 5)
}
req.RowIDs[bitCount-1] = 10
req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2)
@ -3508,7 +3556,7 @@ func TestExecutor_Execute_All(t *testing.T) {
{qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2},
{qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2},
{qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1},
{qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth},
{qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)},
}
for i, test := range tests {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil {
@ -4001,6 +4049,25 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Err_Store(Distinct)", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name())
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil {
t.Fatalf("expected 'unsupported result type' error, got: %+v", res)
}
})
}
func benchmarkExistence(nn bool, b *testing.B) {

24
field.go
View file

@ -86,10 +86,11 @@ var availableShardFileFlushDuration = &protected{
// Field represents a container for views.
type Field struct {
mu sync.RWMutex
path string
index string
name string
mu sync.RWMutex
createdAt int64
path string
index string
name string
viewMap map[string]*view
@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) {
// Name returns the name the field was initialized with.
func (f *Field) Name() string { return f.name }
// CreatedAt is an timestamp for a specific version of field.
func (f *Field) CreatedAt() int64 {
f.mu.RLock()
defer f.mu.RUnlock()
return f.createdAt
}
// Index returns the index name the field was initialized with.
func (f *Field) Index() string { return f.index }
@ -1971,9 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FieldInfo represents schema information for a field.
type FieldInfo struct {
Name string `json:"name"`
Options FieldOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
Name string `json:"name"`
CreatedAt int64 `json:"createdAt,omitempty"`
Options FieldOptions `json:"options"`
Views []*ViewInfo `json:"views,omitempty"`
}
type fieldInfoSlice []*FieldInfo

341
fragment.go
View file

@ -1159,14 +1159,24 @@ func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row,
}
}
func absInt64(v int64) uint64 {
switch {
case v > 0:
return uint64(v)
case v == -9223372036854775808:
return 9223372036854775808
default:
return uint64(-v)
}
}
func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) {
// Start with set of columns with values set.
b := f.row(bsiExistsBit)
// Filter to only positive/negative numbers.
upredicate := uint64(predicate)
upredicate := absInt64(predicate)
if predicate < 0 {
upredicate = uint64(-predicate)
b = b.Intersect(f.row(bsiSignBit)) // only negatives
} else {
b = b.Difference(f.row(bsiSignBit)) // only positives
@ -1204,27 +1214,42 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) {
}
func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
if predicate == 1 && !allowEquality {
predicate, allowEquality = 0, true
}
// Start with set of columns with values set.
b := f.row(bsiExistsBit)
// Create predicate without sign bit.
upredicate := uint64(predicate)
if predicate < 0 {
upredicate = uint64(-predicate)
}
// Get the sign bit row.
sign := f.row(bsiSignBit)
// If predicate is positive, return all positives less than predicate and all negatives.
if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) {
pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality)
// Create predicate without sign bit.
upredicate := absInt64(predicate)
switch {
case predicate == 0 && !allowEquality:
// Match all negative integers.
return b.Intersect(sign), nil
case predicate == 0 && allowEquality:
// Match all integers that are either negative or 0.
zeroes, err := f.rangeEQ(bitDepth, 0)
if err != nil {
return nil, err
}
neg := f.row(bsiSignBit)
return neg.Union(pos), nil
return b.Intersect(sign).Union(zeroes), nil
case predicate < 0:
// Match all every negative number beyond the predicate.
return f.rangeGTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality)
default:
// Match positive numbers less than the predicate, and all negatives.
pos, err := f.rangeLTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality)
if err != nil {
return nil, err
}
neg := b.Intersect(sign)
return pos.Union(neg), nil
}
// Otherwise if predicate is negative, return all negatives greater than upredicate.
return f.rangeGTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality)
}
// msb gives the 1-indexed position (counting from lsb) of the most
@ -1237,110 +1262,118 @@ func msb(x uint64) uint {
// rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit.
func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
keep := NewRow()
// if the predicate is larger than all representable numbers given
// our bitDepth... then just return everything.
if msb(predicate) > bitDepth {
switch {
case msb(predicate) > bitDepth:
fallthrough
case predicate == (1<<bitDepth)-1 && allowEquality:
// This query matches all possible values.
return filter, nil
case predicate == (1<<bitDepth)-1 && !allowEquality:
// This query matches everything that is not (1<<bitDepth)-1.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
row := f.row(uint64(bsiOffsetBit + i))
matches = matches.Union(filter.Difference(row))
}
return matches, nil
case allowEquality:
predicate++
}
// Filter any bits that don't match the current bit value.
leadingZeros := true
for i := int(bitDepth - 1); i >= 0; i-- {
// Compare intermediate bits.
matched := NewRow()
remaining := filter
for i := int(bitDepth - 1); i >= 0 && predicate > 0 && remaining.Any(); i-- {
row := f.row(uint64(bsiOffsetBit + i))
bit := (predicate >> uint(i)) & 1
// Remove any columns with higher bits set.
if leadingZeros {
if bit == 0 {
filter = filter.Difference(row)
continue
} else {
leadingZeros = false
}
}
// Handle last bit differently.
// If bit is zero then return only already kept columns.
// If bit is one then remove any one columns.
if i == 0 && !allowEquality {
if bit == 0 {
return keep, nil
}
return filter.Difference(row.Difference(keep)), nil
}
// If bit is zero then remove all set columns not in excluded bitmap.
if bit == 0 {
filter = filter.Difference(row.Difference(keep))
continue
}
// If bit is set then add columns for set bits to exclude.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(filter.Difference(row))
zeroes := remaining.Difference(row)
switch (predicate >> uint(i)) & 1 {
case 1:
// Match everything with a zero bit here.
matched = matched.Union(zeroes)
predicate &^= 1 << uint(i)
case 0:
// Discard everything with a one bit here.
remaining = zeroes
}
}
return filter, nil
return matched, nil
}
func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) {
if predicate == -1 && !allowEquality {
predicate, allowEquality = 0, true
}
b := f.row(bsiExistsBit)
// Create predicate without sign bit.
upredicate := uint64(predicate)
if predicate < 0 {
upredicate = uint64(-predicate)
}
upredicate := absInt64(predicate)
// If predicate is positive, return all positives greater than predicate.
if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) {
return f.rangeGTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality)
}
sign := f.row(bsiSignBit)
// If predicate is negative, return all negatives less than than upredicate and all positives.
neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality)
if err != nil {
return nil, err
switch {
case predicate == 0 && !allowEquality:
// Match all positive numbers except zero.
nonzero, err := f.rangeNEQ(bitDepth, 0)
if err != nil {
return nil, err
}
b = nonzero
fallthrough
case predicate == 0 && allowEquality:
// Match all positive numbers.
return b.Difference(sign), nil
case predicate >= 0:
// Match all positive numbers greater than the predicate.
return f.rangeGTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality)
default:
// Match all positives and greater negatives.
neg, err := f.rangeLTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality)
if err != nil {
return nil, err
}
pos := b.Difference(sign)
return pos.Union(neg), nil
}
pos := b.Difference(f.row(bsiSignBit))
return pos.Union(neg), nil
}
func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
keep := NewRow()
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
switch {
case predicate == 0 && allowEquality:
// This query matches all possible values.
return filter, nil
case predicate == 0 && !allowEquality:
// This query matches everything that is not 0.
matches := NewRow()
for i := uint(0); i < bitDepth; i++ {
row := f.row(uint64(bsiOffsetBit + i))
matches = matches.Union(filter.Intersect(row))
}
return matches, nil
case allowEquality:
predicate--
}
// Compare intermediate bits.
matched := NewRow()
remaining := filter
predicate |= (^uint64(0)) << bitDepth
for i := int(bitDepth - 1); i >= 0 && predicate < ^uint64(0) && remaining.Any(); i-- {
row := f.row(uint64(bsiOffsetBit + i))
bit := (predicate >> uint(i)) & 1
// Handle last bit differently.
// If bit is one then return only already kept columns.
// If bit is zero then remove any unset columns.
if i == 0 && !allowEquality {
if bit == 1 {
return keep, nil
}
return filter.Difference(filter.Difference(row, keep)), nil
}
// If bit is set then remove all unset columns not already kept.
if bit == 1 {
filter = filter.Difference(filter.Difference(row, keep))
continue
}
// If bit is unset then add columns with set bit to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(filter.Intersect(row))
ones := remaining.Intersect(row)
switch (predicate >> uint(i)) & 1 {
case 1:
// Discard everything with a zero bit here.
remaining = ones
case 0:
// Match everything with a one bit here.
matched = matched.Union(ones)
predicate |= 1 << uint(i)
}
}
return filter, nil
return matched, nil
}
// notNull returns the exists row.
@ -1353,73 +1386,65 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64)
b := f.row(bsiExistsBit)
// Convert predicates to unsigned values.
upredicateMin, upredicateMax := uint64(predicateMin), uint64(predicateMax)
if predicateMin < 0 {
upredicateMin = uint64(-predicateMin)
}
if predicateMax < 0 {
upredicateMax = uint64(-predicateMax)
}
upredicateMin, upredicateMax := absInt64(predicateMin), absInt64(predicateMax)
// Handle positive-only values.
if predicateMin >= 0 {
switch {
case predicateMin == predicateMax:
return f.rangeEQ(bitDepth, predicateMin)
case predicateMin >= 0:
// Handle positive-only values.
return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax)
}
// Handle negative-only values. Swap unsigned min/max predicates.
if predicateMax < 0 {
case predicateMax < 0:
// Handle negative-only values. Swap unsigned min/max predicates.
return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin)
default:
// If predicate crosses positive/negative boundary then handle separately and union.
pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true)
if err != nil {
return nil, err
}
neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true)
if err != nil {
return nil, err
}
return pos.Union(neg), nil
}
// If predicate crosses positive/negative boundary then handle separately and union.
pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true)
if err != nil {
return nil, err
}
neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true)
if err != nil {
return nil, err
}
return pos.Union(neg), nil
}
// rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit.
func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
keep1 := NewRow() // GTE
keep2 := NewRow() // LTE
switch {
case predicateMax > (1<<bitDepth)-1:
// The upper bound cannot be violated.
return f.rangeGTUnsigned(filter, bitDepth, predicateMin, true)
case predicateMin == 0:
// The lower bound cannot be violated.
return f.rangeLTUnsigned(filter, bitDepth, predicateMax, true)
}
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
// Compare any upper bits which are equal.
firstDiff := int(msb(predicateMax^predicateMin)) - 1
remaining := filter
for i := int(bitDepth - 1); i > firstDiff; i-- {
row := f.row(uint64(bsiOffsetBit + i))
bit1 := (predicateMin >> uint(i)) & 1
bit2 := (predicateMax >> uint(i)) & 1
// GTE predicateMin
// If bit is set then remove all unset columns not already kept.
if bit1 == 1 {
filter = filter.Difference(filter.Difference(row, keep1))
} else {
// If bit is unset then add columns with set bit to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep1 = keep1.Union(filter.Intersect(row))
}
}
// LTE predicateMax
// If bit is zero then remove all set bits not in excluded bitmap.
if bit2 == 0 {
filter = filter.Difference(row.Difference(keep2))
} else {
// If bit is set then add columns for set bits to exclude.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep2 = keep2.Union(filter.Difference(row))
}
switch (predicateMin >> uint(i)) & 1 {
case 1:
remaining = remaining.Intersect(row)
case 0:
remaining = remaining.Difference(row)
}
}
return filter, nil
var err error
remaining, err = f.rangeGTUnsigned(remaining, uint(firstDiff+1), predicateMin, true)
if err != nil {
return nil, err
}
remaining, err = f.rangeLTUnsigned(remaining, uint(firstDiff+1), predicateMax, true)
if err != nil {
return nil, err
}
return remaining, nil
}
// pos translates the row ID and column ID into a position in the storage bitmap.
@ -2607,14 +2632,14 @@ func filterWithRows(rows []uint64) rowFilter {
// returning done == true will cause processing to stop after all filters for
// this container have been processed. The rows accumulated up to this point
// (including this row if all filters passed) will be returned.
func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 {
func (f *fragment) rows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
return f.unprotectedRows(start, filters...)
return f.unprotectedRows(ctx, start, filters...)
}
// unprotectedRows calls rows without grabbing the mutex.
func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64 {
func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 {
startKey := rowToKey(start)
i, _ := f.storage.Containers.Iterator(startKey)
rows := make([]uint64, 0)
@ -2622,6 +2647,10 @@ func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64
// Loop over the existing containers.
for i.Next() {
// caller doesn't need a result anymore.
if ctx.Err() != nil {
return nil
}
key, c := i.Value()
// virtual row for the current container
@ -2861,7 +2890,7 @@ type setRowIterator struct {
func (f *fragment) setRowIterator(wrap bool, filters ...rowFilter) rowIterator {
return &setRowIterator{
f: f,
rowIDs: f.rows(0, filters...), // TODO: this may be memory intensive in high cardinality cases
rowIDs: f.rows(context.Background(), 0, filters...), // TODO: this may be memory intensive in high cardinality cases
wrap: wrap,
}
}
@ -3282,7 +3311,7 @@ func newRowsVector(f *fragment) *rowsVector {
// otherwise it returns false. Ensure that you already
// have the mutex before calling this.
func (v *rowsVector) Get(colID uint64) (uint64, bool, error) {
rows := v.f.unprotectedRows(0, filterColumn(colID))
rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID))
if len(rows) > 1 {
return 0, false, errors.New("found multiple row values for column")
} else if len(rows) == 1 {
@ -3316,7 +3345,7 @@ func newBoolVector(f *fragment) *boolVector {
// otherwise it returns false. Ensure that you already
// have the fragment mutex before calling this.
func (v *boolVector) Get(colID uint64) (uint64, bool, error) {
rows := v.f.unprotectedRows(0, filterColumn(colID))
rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID))
if len(rows) > 1 {
return 0, false, errors.New("found multiple row values for column")
} else if len(rows) == 1 {

60
fragment_internal_test.go
View file

@ -111,6 +111,11 @@ func TestFragment_ClearBit(t *testing.T) {
func TestFragment_RowcacheMap(t *testing.T) {
var done int64
f := mustOpenFragment("i", "f", viewStandard, 0, "")
// Under -race, this test turns out to take a fairly long time
// to run with larger OpN, because we write 50,000 bits to
// the bitmap, and everything is being race-detected, and we don't
// actually need that many to get the result we care about.
f.MaxOpN = 2000
defer f.Clean(t)
ch := make(chan struct{})
@ -619,6 +624,23 @@ func TestFragment_Range(t *testing.T) {
}
})
t.Run("LTMaxRegression", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean(t)
if _, err := f.setValue(1, 2, 3); err != nil {
t.Fatal(err)
} else if _, err := f.setValue(2, 2, 0); err != nil {
t.Fatal(err)
}
if b, err := f.rangeLTUnsigned(NewRow(1, 2), 2, 3, false); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2}) {
t.Fatalf("unepxected coulmns: %+v", b.Columns())
}
})
t.Run("GT", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean(t)
@ -667,6 +689,23 @@ func TestFragment_Range(t *testing.T) {
}
})
t.Run("GTMinRegression", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean(t)
if _, err := f.setValue(1, 2, 0); err != nil {
t.Fatal(err)
} else if _, err := f.setValue(2, 2, 1); err != nil {
t.Fatal(err)
}
if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 0, false); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Columns(), []uint64{2}) {
t.Fatalf("unepxected coulmns: %+v", b.Columns())
}
})
t.Run("BETWEEN", func(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
defer f.Clean(t)
@ -2412,8 +2451,9 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) {
if err != nil {
t.Fatalf("importing roaring: %v", err)
}
if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
t.Fatalf("unexpected rows: %v", f.rows(0))
rows := f.rows(context.Background(), 0)
if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
t.Fatalf("unexpected rows: %v", rows)
}
for i := uint64(1); i < 10; i++ {
if f.row(i).Count() >= f.row(i-1).Count() {
@ -2714,12 +2754,12 @@ func TestFragment_RowsIteration(t *testing.T) {
}
}
ids := f.rows(0)
ids := f.rows(context.Background(), 0)
if !reflect.DeepEqual(expectedAll, ids) {
t.Fatalf("Do not match %v %v", expectedAll, ids)
}
ids = f.rows(0, filterColumn(1))
ids = f.rows(context.Background(), 0, filterColumn(1))
if !reflect.DeepEqual(expectedOdd, ids) {
t.Fatalf("Do not match %v %v", expectedOdd, ids)
}
@ -2738,12 +2778,12 @@ func TestFragment_RowsIteration(t *testing.T) {
t.Fatal(err)
}
ids := f.rows(0)
ids := f.rows(context.Background(), 0)
if !reflect.DeepEqual(expected, ids) {
t.Fatalf("Do not match %v %v", expected, ids)
}
ids = f.rows(0, filterColumn(66000))
ids = f.rows(context.Background(), 0, filterColumn(66000))
if !reflect.DeepEqual(expected, ids) {
t.Fatalf("Do not match %v %v", expected, ids)
}
@ -2754,18 +2794,18 @@ func TestFragment_RowsIteration(t *testing.T) {
defer f.Clean(t)
expectedRows := make([]uint64, 0)
for r := uint64(1); r < uint64(10000); r += 100 {
for r := uint64(1); r < uint64(10000); r += 250 {
expectedRows = append(expectedRows, r)
for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 {
for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) {
if _, err := f.setBit(r, c); err != nil {
t.Fatal(err)
}
ids := f.rows(0)
ids := f.rows(context.Background(), 0)
if !reflect.DeepEqual(expectedRows, ids) {
t.Fatalf("Do not match %v %v", expectedRows, ids)
}
ids = f.rows(0, filterColumn(c))
ids = f.rows(context.Background(), 0, filterColumn(c))
if !reflect.DeepEqual(expectedRows, ids) {
t.Fatalf("Do not match %v %v", expectedRows, ids)
}

51
generator/slice.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package generator
import (
"math/rand"
"sort"
)
// Uint64Slice generates between [0, n) random uint64 numbers between min and max.
func Uint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 {
a := make([]uint64, rand.Intn(n))
for i := range a {
a[i] = min + uint64(rand.Int63n(int64(max-min)))
}
if sorted {
sort.Sort(uint64Slice(a))
}
return a
}
// Uint64SetSlice returns the values in a uint64 set.
func Uint64SetSlice(m map[uint64]struct{}) []uint64 {
a := make([]uint64, 0, len(m))
for v := range m {
a = append(a, v)
}
sort.Sort(uint64Slice(a))
return a
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
func (u uint64Slice) Swap(i, j int) { u[i], u[j] = u[j], u[i] }
func (u uint64Slice) Len() int { return len(u) }
func (u uint64Slice) Less(i, j int) bool { return u[i] < u[j] }

8
gopsutil/systeminfo.go
View file

@ -15,6 +15,7 @@
package gopsutil
import (
"math"
"runtime"
"strings"
@ -114,6 +115,13 @@ func (s *systemInfo) collectPlatformInfo() error {
}
s.cpuModel = infos[0].ModelName
s.cpuMHz = computeMHz(s.cpuModel)
if s.cpuMHz < 0 {
s.cpuMHz = int(math.Round(infos[0].Mhz))
}
if s.cpuMHz < 0 {
// This is supposed to be unsigned.
s.cpuMHz = 0
}
// gopsutil reports core and clock speed info inconsistently
// by OS

10
gossip/gossip.go
View file

@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte {
Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())},
}
for _, idx := range m.Schema.Indexes {
is := &pilosa.IndexStatus{Name: idx.Name}
is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt}
for _, f := range idx.Fields {
availableShards := roaring.NewBitmap()
if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil {
availableShards = field.AvailableShards()
}
is.Fields = append(is.Fields, &pilosa.FieldStatus{
fs := &pilosa.FieldStatus{
Name: f.Name,
CreatedAt: f.CreatedAt,
AvailableShards: availableShards,
})
}
is.Fields = append(is.Fields, fs)
}
m.Indexes = append(m.Indexes, is)
}

75
handler.go
View file

@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{}
// ImportValueRequest describes the import request structure
// for a value (BSI) import.
type ImportValueRequest struct {
Index string
Field string
Index string
IndexCreatedAt int64
Field string
FieldCreatedAt int64
// if Shard is MaxUint64 (an impossible shard value), this
// indicates that the column IDs may come from multiple shards.
Shard uint64
@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) {
// Validate ensures that the payload of the request is valid.
func (ivr *ImportValueRequest) Validate() error {
return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt)
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ivr.Index == "" || ivr.Field == "" {
return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field)
}
@ -160,30 +167,48 @@ func (ivr *ImportValueRequest) Validate() error {
if valueSetCount > 1 {
return errors.Errorf("must pass ints, floats, or strings but not multiple")
}
if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 {
if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
// ImportColumnAttrsRequest describes the import request structure
// for a ColumnAttr import
type ImportColumnAttrsRequest struct {
AttrKey string
ColumnIDs []uint64
AttrVals []string
Shard int64
Index string
AttrKey string
ColumnIDs []uint64
AttrVals []string
Shard int64
Index string
IndexCreatedAt int64
}
// ImportRequest describes the import request structure
// for an import.
type ImportRequest struct {
Index string
Field string
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
Index string
IndexCreatedAt int64
Field string
FieldCreatedAt int64
Shard uint64
RowIDs []uint64
ColumnIDs []uint64
RowKeys []string
ColumnKeys []string
Timestamps []int64
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 {
if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
const (
@ -195,10 +220,22 @@ const (
// ImportRoaringRequest describes the import request structure
// for an import containing roaring-encoded data.
type ImportRoaringRequest struct {
Clear bool
Action string // [set, clear, overwrite]
Block int
Views map[string][]byte
IndexCreatedAt int64
FieldCreatedAt int64
Clear bool
Action string // [set, clear, overwrite]
Block int
Views map[string][]byte
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 {
if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt {
return ErrPreconditionFailed
}
}
return nil
}
// ImportResponse is the structured response of an import.

87
holder.go
View file

@ -234,7 +234,13 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "opening index")
}
if err := index.Open(); err != nil {
if h.isCoordinator() {
index.createdAt = timestamp()
err = index.OpenWithTimestamp()
} else {
err = index.Open()
}
if err != nil {
if err == ErrName {
h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
continue
@ -380,11 +386,16 @@ func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{
Name: index.Name(),
Options: index.Options(),
Name: index.Name(),
CreatedAt: index.CreatedAt(),
Options: index.Options(),
}
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
fi := &FieldInfo{
Name: field.Name(),
CreatedAt: field.CreatedAt(),
Options: field.Options(),
}
for _, view := range field.views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo {
for _, index := range h.Indexes() {
di := &IndexInfo{
Name: index.Name(),
CreatedAt: index.CreatedAt(),
Options: index.Options(),
ShardWidth: ShardWidth,
}
@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo {
if strings.HasPrefix(field.name, "_") {
continue
}
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
fi := &FieldInfo{
Name: field.Name(),
CreatedAt: field.CreatedAt(),
Options: field.Options(),
}
di.Fields = append(di.Fields, fi)
}
sort.Sort(fieldInfoSlice(di.Fields))
@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo {
// applySchema applies an internal Schema to Holder.
func (h *Holder) applySchema(schema *Schema) error {
// Create indexes that don't exist.
for _, index := range schema.Indexes {
idx, err := h.CreateIndexIfNotExists(index.Name, index.Options)
for _, i := range schema.Indexes {
idx, err := h.CreateIndexIfNotExists(i.Name, i.Options)
if err != nil {
return errors.Wrap(err, "creating index")
}
if i.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = i.CreatedAt
idx.mu.Unlock()
}
// Create fields that don't exist.
for _, f := range index.Fields {
field, err := idx.createFieldIfNotExists(f.Name, &f.Options)
for _, f := range i.Fields {
fld, err := idx.createFieldIfNotExists(f.Name, &f.Options)
if err != nil {
return errors.Wrap(err, "creating field")
}
if f.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = f.CreatedAt
fld.mu.Unlock()
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := field.createViewIfNotExists(v.Name)
_, err := fld.createViewIfNotExists(v.Name)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -447,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error {
return nil
}
func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
for _, ii := range indexes {
idx := h.Index(ii.Name)
if idx == nil {
continue
}
if ii.CreatedAt != 0 {
idx.mu.Lock()
idx.createdAt = ii.CreatedAt
idx.mu.Unlock()
}
for _, fi := range ii.Fields {
fld := idx.Field(fi.Name)
if fld == nil {
continue
}
if fi.CreatedAt != 0 {
fld.mu.Lock()
fld.createdAt = fi.CreatedAt
fld.mu.Unlock()
}
}
}
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
@ -652,6 +706,13 @@ func (h *Holder) recalculateCaches() {
}
}
func (h *Holder) isCoordinator() bool {
if s, ok := h.broadcaster.(*Server); ok {
return s.isCoordinator
}
return false
}
// setFileLimit attempts to set the open file limit to the FileLimit constant defined above.
func (h *Holder) setFileLimit() {
oldLimit := &syscall.Rlimit{}
@ -1059,7 +1120,8 @@ func (s *holderSyncer) stopTranslationSync() error {
// writing new translation keys. Index stores are writable if the node owns the
// partition. Field stores are writable if the node is the coordinator.
func (s *holderSyncer) setTranslateReadOnlyFlags() {
isCoordinator := s.Cluster.isCoordinator()
s.Cluster.mu.RLock()
isCoordinator := s.Cluster.unprotectedIsCoordinator()
for _, index := range s.Holder.Indexes() {
// There is a race condition here:
@ -1079,7 +1141,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
// done using it.
index.mu.RLock()
for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID)
ownsPartition := s.Cluster.unprotectedOwnsPartition(s.Node.ID, partitionID)
if ts := index.TranslateStore(partitionID); ts != nil {
ts.SetReadOnly(!ownsPartition)
}
@ -1090,6 +1152,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() {
field.TranslateStore().SetReadOnly(!isCoordinator)
}
}
s.Cluster.mu.RUnlock()
}
// initializeIndexTranslateReplication connects to each node that is the

334
http/handler.go
View file

@ -15,14 +15,15 @@
package http
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"expvar"
"fmt"
"io"
"io/ioutil"
"math"
"mime"
"net"
"net/http"
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
@ -358,6 +359,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/transaction/{id}", handler.handlePostTransaction).Methods("POST").Name("PostTransaction")
router.HandleFunc("/transaction/{id}/finish", handler.handlePostFinishTransaction).Methods("POST").Name("PostFinishTransaction")
router.HandleFunc("/transactions", handler.handleGetTransactions).Methods("GET").Name("GetTransactions")
router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
// /internal endpoints are for internal use only; they may change at any time.
@ -401,9 +403,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// successResponse is a general success/error struct for http responses.
type successResponse struct {
h *Handler
Success bool `json:"success"`
Error *Error `json:"error,omitempty"`
h *Handler
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
Error *Error `json:"error,omitempty"`
}
// check determines success or failure based on the error.
@ -473,9 +477,33 @@ func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) {
// headers are present, but none of them are "application/json"
// (or any matching wildcard). Otherwise returns true.
func validHeaderAcceptJSON(header http.Header) bool {
return validHeaderAcceptType(header, "application", "json")
}
func validHeaderAcceptType(header http.Header, typ, subtyp string) bool {
if v, found := header["Accept"]; found {
for _, v := range v {
if v == "application/json" || v == "*/*" || v == "*/json" || v == "application/*" {
t, _, err := mime.ParseMediaType(v)
if err != nil {
switch err {
case mime.ErrInvalidMediaParameter:
// This is an optional feature, so we can keep going anyway.
default:
continue
}
}
spl := strings.SplitN(t, "/", 2)
if len(spl) < 2 {
continue
}
switch {
case spl[0] == typ && spl[1] == subtyp:
return true
case spl[0] == "*" && spl[1] == subtyp:
return true
case spl[0] == typ && spl[1] == "*":
return true
case spl[0] == "*" && spl[1] == "*":
return true
}
}
@ -504,7 +532,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
schema := h.api.Schema(r.Context())
if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here?
if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil {
h.logger.Printf("write schema response error: %s", err)
}
}
@ -773,7 +801,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
return
}
resp := successResponse{h: h}
resp := successResponse{h: h, Name: indexName}
// Decode request.
req := postIndexRequest{
@ -787,8 +815,15 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
return
}
_, err = h.api.CreateIndex(r.Context(), indexName, req.Options)
index, err := h.api.CreateIndex(r.Context(), indexName, req.Options)
if index != nil {
resp.CreatedAt = index.CreatedAt()
} else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok {
if index, _ = h.api.Index(r.Context(), indexName); index != nil {
resp.CreatedAt = index.CreatedAt()
}
}
resp.write(w, err)
}
@ -826,6 +861,52 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
}
}
func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) {
var rtype string
switch {
case validHeaderAcceptType(r.Header, "text", "plain"):
rtype = "text/plain"
case validHeaderAcceptJSON(r.Header):
rtype = "application/json"
default:
http.Error(w, "no acceptable response type selected", http.StatusNotAcceptable)
return
}
queries, err := h.api.ActiveQueries(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", rtype)
switch rtype {
case "text/plain":
durations := make([]string, len(queries))
for i, q := range queries {
durations[i] = q.Age.String()
}
var maxlen int
for _, l := range durations {
if len(l) > maxlen {
maxlen = len(l)
}
}
for i, q := range queries {
_, err := fmt.Fprintf(w, "%*s%q\n", -(maxlen + 2), durations[i], q.Query)
if err != nil {
h.logger.Printf("sending GetActiveQueries response: %s", err)
return
}
}
if _, err := w.Write([]byte{'\n'}); err != nil {
h.logger.Printf("sending GetActiveQueries response: %s", err)
}
case "application/json":
if err := json.NewEncoder(w).Encode(queries); err != nil {
h.logger.Printf("encoding GetActiveQueries response: %s", err)
}
}
}
type postIndexAttrDiffRequest struct {
Blocks []pilosa.AttrBlock `json:"blocks"`
}
@ -853,7 +934,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
return
}
resp := successResponse{h: h}
resp := successResponse{h: h, Name: fieldName}
// Decode request.
var req postFieldRequest
@ -925,11 +1006,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex))
}
_, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...)
field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...)
if _, ok := err.(pilosa.BadRequestError); ok {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if field != nil {
resp.CreatedAt = field.CreatedAt()
} else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok {
if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil {
resp.CreatedAt = field.CreatedAt()
}
}
resp.write(w, err)
}
@ -1240,7 +1328,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error
// readProtobufQueryRequest parses query parameters in protobuf from r.
func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) {
// Slurp the body.
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1258,7 +1346,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
q := r.URL.Query()
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
buf, err := readBody(r)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1319,95 +1407,14 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse
return json.NewEncoder(w).Encode(resp)
}
// handlePostImport handles /import requests.
func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
func validateProtobufHeader(r *http.Request) (error string, code int) {
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
return
return "Unsupported media type", http.StatusUnsupportedMediaType
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
// If the clear flag is true, treat the import as clear bits.
q := r.URL.Query()
doClear := q.Get("clear") == "true"
doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true"
opts := []pilosa.ImportOption{
pilosa.OptImportOptionsClear(doClear),
pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck),
}
// Get index and field type to determine how to handle the
// import data.
field, err := h.api.Field(r.Context(), indexName, fieldName)
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrIndexNotFound:
fallthrough
case pilosa.ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Read entire body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Unmarshal request based on field type.
if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
// Field type: Int
// Marshal into request object.
req := &pilosa.ImportValueRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.ImportValue(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
} else {
// Field type: set, time, mutex
// Marshal into request object.
req := &pilosa.ImportRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.Import(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
}
// Write response.
_, err = w.Write(importOk)
if err != nil {
h.logger.Printf("writing import response: %v", err)
if r.Header.Get("Accept") != "application/x-protobuf" {
return "Not acceptable", http.StatusNotAcceptable
}
return
}
// handleGetExport handles /export requests.
@ -1885,6 +1892,95 @@ func GetHTTPClient(t *tls.Config) *http.Client {
return &http.Client{Transport: transport}
}
// handlePostImport handles /import requests.
func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
// Get index and field type to determine how to handle the
// import data.
indexName := mux.Vars(r)["index"]
index, err := h.api.Index(r.Context(), indexName)
if err != nil {
if errors.Cause(err) == pilosa.ErrIndexNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
fieldName := mux.Vars(r)["field"]
field := index.Field(fieldName)
if field == nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// If the clear flag is true, treat the import as clear bits.
q := r.URL.Query()
doClear := q.Get("clear") == "true"
doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true"
opts := []pilosa.ImportOption{
pilosa.OptImportOptionsClear(doClear),
pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck),
}
// Read entire body.
body, err := readBody(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Unmarshal request based on field type.
if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal {
// Field type: Int
// Marshal into request object.
req := &pilosa.ImportValueRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.ImportValue(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
} else {
// Field type: set, time, mutex
// Marshal into request object.
req := &pilosa.ImportRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.Import(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
}
// Write response.
_, err = w.Write(importOk)
if err != nil {
h.logger.Printf("writing import response: %v", err)
}
}
// handlePostImportColumnAttrs
func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
@ -1898,7 +1994,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
opts := []pilosa.ImportOption{}
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -1911,7 +2007,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
}
if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
@ -1922,16 +2023,16 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req
}
}
// handlPostRoaringImport
// handlePostImportRoaring
func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
// Get index and field type to determine how to handle the
// import data.
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
@ -1946,7 +2047,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
body, err := ioutil.ReadAll(r.Body)
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
if err != nil {
@ -1976,6 +2077,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
resp.Err = err.Error()
if _, ok := err.(pilosa.BadRequestError); ok {
w.WriteHeader(http.StatusBadRequest)
} else if _, ok := err.(pilosa.NotFoundError); ok {
w.WriteHeader(http.StatusNotFound)
} else if _, ok := err.(pilosa.PreconditionFailedError); ok {
w.WriteHeader(http.StatusPreconditionFailed)
} else {
w.WriteHeader(http.StatusInternalServerError)
}
@ -2040,3 +2145,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request)
h.logger.Printf("writing translate keys response: %v", err)
}
}
// Read entire request body.
func readBody(r *http.Request) ([]byte, error) {
var contentLength int64 = bytes.MinRead
if r.ContentLength > 0 {
contentLength = r.ContentLength
}
buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength))
if _, err := buf.ReadFrom(r.Body); err != nil {
return nil, err
}
return buf.Bytes(), nil
}

31
index.go
View file

@ -36,10 +36,11 @@ import (
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
path string
name string
keys bool // use string keys
mu sync.RWMutex
createdAt int64
path string
name string
keys bool // use string keys
// Existence tracking.
trackExistence bool
@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) {
}, nil
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()
defer i.mu.RUnlock()
return i.createdAt
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions {
}
// Open opens and initializes the index.
func (i *Index) Open() (err error) {
func (i *Index) Open() error { return i.open(false) }
// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (i *Index) OpenWithTimestamp() error { return i.open(true) }
func (i *Index) open(withTimestamp bool) (err error) {
// Ensure the path exists.
i.logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
@ -154,7 +167,7 @@ func (i *Index) Open() (err error) {
}
i.logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(); err != nil {
if err := i.openFields(withTimestamp); err != nil {
return errors.Wrap(err, "opening fields")
}
@ -197,7 +210,7 @@ func (i *Index) Open() (err error) {
var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields() error {
func (i *Index) openFields(withTimestamp bool) error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -229,6 +242,9 @@ fileLoop:
i.logger.Debugf("open field: %s", fi.Name())
mu.Lock()
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
fld.createdAt = timestamp()
}
mu.Unlock()
if err != nil {
return errors.Wrapf(ErrName, "'%s'", fi.Name())
@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
CreatedAt int64 `json:"createdAt,omitempty"`
Options IndexOptions `json:"options"`
Fields []*FieldInfo `json:"fields"`
ShardWidth uint64 `json:"shardWidth"`

530
internal/private.pb.go
View file

@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string {
type CreateIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta {
return nil
}
func (m *CreateIndexMessage) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type CreateFieldMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"`
CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions {
return nil
}
func (m *CreateFieldMessage) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type DeleteFieldMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -853,6 +869,7 @@ type Field struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"`
Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"`
CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -912,6 +929,13 @@ func (m *Field) GetViews() []string {
return nil
}
func (m *Field) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type Schema struct {
Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index {
type Index struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"`
Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -1008,6 +1033,13 @@ func (m *Index) GetName() string {
return ""
}
func (m *Index) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
func (m *Index) GetOptions() *IndexMeta {
if m != nil {
return m.Options
@ -1090,6 +1122,7 @@ type Node struct {
URI *URI `protobuf:"bytes,2,opt,name=URI,proto3" json:"URI,omitempty"`
IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"`
State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"`
GRPCURI *URI `protobuf:"bytes,5,opt,name=GRPCURI,proto3" json:"GRPCURI,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1156,6 +1189,13 @@ func (m *Node) GetState() string {
return ""
}
func (m *Node) GetGRPCURI() *URI {
if m != nil {
return m.GRPCURI
}
return nil
}
type NodeStateMessage struct {
NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
@ -1332,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus {
type IndexStatus struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1384,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus {
return nil
}
func (m *IndexStatus) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type FieldStatus struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"`
CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1439,10 +1488,18 @@ func (m *FieldStatus) GetAvailableShards() []uint64 {
return nil
}
func (m *FieldStatus) GetCreatedAt() int64 {
if m != nil {
return m.CreatedAt
}
return 0
}
type ClusterStatus struct {
ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"`
Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1502,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node {
return nil
}
func (m *ClusterStatus) GetSchema() *Schema {
if m != nil {
return m.Schema
}
return nil
}
type BSIGroup struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"`
@ -2413,95 +2477,99 @@ func init() {
func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) }
var fileDescriptor_d2a91b51c7bdc125 = []byte{
// 1395 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x17, 0xcd, 0x72, 0xdb, 0x44,
0x18, 0x59, 0x8e, 0x63, 0x7f, 0x8e, 0x53, 0x67, 0xdb, 0xa6, 0x6a, 0x60, 0x82, 0x59, 0x3a, 0xd4,
0x74, 0x86, 0xd0, 0x69, 0x99, 0xe1, 0xb7, 0x33, 0x6d, 0xe2, 0xb4, 0x98, 0x92, 0xb4, 0x5d, 0xa7,
0xbd, 0x71, 0xd8, 0xc8, 0x3b, 0x8d, 0x26, 0xb2, 0x64, 0xa4, 0x55, 0xea, 0xf4, 0xc0, 0x15, 0x66,
0x78, 0x01, 0x8e, 0xbc, 0x07, 0x2f, 0xc0, 0x91, 0x47, 0x60, 0xca, 0x53, 0x70, 0x63, 0xf6, 0xdb,
0x5d, 0x49, 0x76, 0x1c, 0x52, 0x52, 0x6e, 0xfb, 0xfd, 0xff, 0x7f, 0x9f, 0x04, 0xad, 0x71, 0x12,
0x1c, 0x71, 0x29, 0x36, 0xc6, 0x49, 0x2c, 0x63, 0x52, 0x0f, 0x22, 0x29, 0x92, 0x88, 0x87, 0x6b,
0x4b, 0xe3, 0x6c, 0x3f, 0x0c, 0x7c, 0x8d, 0xa7, 0x0f, 0xa0, 0xd1, 0x8f, 0x86, 0x62, 0xb2, 0x23,
0x24, 0x27, 0x04, 0xaa, 0x0f, 0xc5, 0x71, 0xea, 0xb9, 0x1d, 0xa7, 0x5b, 0x67, 0xf8, 0x26, 0x1f,
0xc0, 0xf2, 0x5e, 0xc2, 0xfd, 0xc3, 0xed, 0x49, 0x90, 0x4a, 0x11, 0xf9, 0xc2, 0xab, 0x22, 0x75,
0x06, 0x4b, 0x7f, 0x75, 0x61, 0xe9, 0x7e, 0x20, 0xc2, 0xe1, 0xa3, 0xb1, 0x0c, 0xe2, 0x28, 0x55,
0xca, 0xf6, 0x8e, 0xc7, 0xc2, 0xab, 0x77, 0x9c, 0x6e, 0x83, 0xe1, 0x9b, 0xbc, 0x03, 0x8d, 0x2d,
0xee, 0x1f, 0x08, 0x24, 0xb8, 0x48, 0x28, 0x10, 0x39, 0x75, 0x10, 0xbc, 0xd4, 0x56, 0x5a, 0xac,
0x40, 0x90, 0x0e, 0x34, 0xf7, 0x82, 0x91, 0x78, 0x92, 0xf1, 0x48, 0x66, 0x23, 0x6f, 0x01, 0xa5,
0xcb, 0x28, 0xb2, 0x0a, 0xb5, 0x47, 0xe1, 0x70, 0x27, 0x88, 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc,
0x40, 0x16, 0xcf, 0x27, 0x1e, 0x14, 0x78, 0x3e, 0xc9, 0xc3, 0x6d, 0x4e, 0x87, 0xbb, 0x1b, 0x0f,
0x24, 0x8f, 0x86, 0x3c, 0x19, 0x3e, 0x0b, 0xc4, 0x0b, 0x6f, 0x49, 0x87, 0x3b, 0x8d, 0x55, 0xb2,
0x9b, 0x3c, 0x15, 0x5e, 0x0b, 0x35, 0xe2, 0x9b, 0xac, 0x41, 0x7d, 0x33, 0x90, 0x3d, 0x31, 0x96,
0x07, 0xde, 0x72, 0xc7, 0xe9, 0x56, 0x59, 0x0e, 0x93, 0x4b, 0xb0, 0x30, 0xf0, 0x79, 0x28, 0xbc,
0x0b, 0x28, 0xa0, 0x01, 0x42, 0x61, 0xe9, 0x7e, 0x9c, 0x88, 0xe0, 0x79, 0x84, 0x45, 0xf0, 0xda,
0x18, 0xd4, 0x14, 0x8e, 0xbc, 0x0f, 0xae, 0x0a, 0x69, 0xa5, 0xe3, 0x74, 0x9b, 0xb7, 0x56, 0x36,
0x6c, 0x1d, 0x37, 0x7a, 0xc2, 0x0f, 0x46, 0x3c, 0x64, 0x8a, 0x8a, 0x4c, 0x7c, 0xe2, 0x91, 0xd3,
0x99, 0xf8, 0x84, 0x52, 0x58, 0xee, 0x8f, 0xc6, 0x71, 0x22, 0x99, 0x48, 0xc7, 0x71, 0x94, 0x0a,
0xd2, 0x06, 0x77, 0x3b, 0x49, 0x3c, 0x07, 0xcd, 0xaa, 0x27, 0xfd, 0x01, 0xda, 0x9b, 0x61, 0xec,
0x1f, 0xf6, 0xb8, 0xe4, 0x4c, 0x7c, 0x9f, 0x89, 0x54, 0x2a, 0xdf, 0xb5, 0x7b, 0x9a, 0x4f, 0x03,
0x0a, 0x8b, 0xf5, 0xf6, 0x2a, 0x1a, 0x8b, 0x80, 0xca, 0x0b, 0x66, 0x4d, 0x97, 0x07, 0xdf, 0x18,
0xfb, 0x01, 0x4f, 0x86, 0x58, 0xd3, 0x2a, 0xd3, 0x80, 0xc2, 0xa2, 0x25, 0xec, 0x83, 0x2a, 0xd3,
0x00, 0xed, 0xc3, 0x4a, 0xc9, 0xbe, 0x71, 0x73, 0x15, 0x6a, 0x2c, 0x7e, 0xd1, 0xef, 0xa5, 0x9e,
0xd3, 0x71, 0xbb, 0x55, 0x66, 0x20, 0x6c, 0x98, 0x38, 0xcc, 0x46, 0x91, 0x22, 0x55, 0x90, 0x54,
0x20, 0xe8, 0x55, 0x58, 0xc0, 0xee, 0x51, 0x51, 0x16, 0xb2, 0xea, 0x49, 0x7f, 0x74, 0xa0, 0xb1,
0xc3, 0x27, 0xe8, 0x48, 0x4a, 0xee, 0x40, 0xdd, 0xd6, 0x16, 0x99, 0x9a, 0xb7, 0xde, 0x2b, 0x32,
0x98, 0xb3, 0x6d, 0x58, 0x9e, 0xed, 0x48, 0x26, 0xc7, 0x2c, 0x17, 0x59, 0xfb, 0x12, 0x5a, 0x53,
0x24, 0x65, 0xef, 0x50, 0x1c, 0xdb, 0xac, 0x1e, 0x8a, 0x63, 0x15, 0xeb, 0x11, 0x0f, 0x33, 0x81,
0xb9, 0xaa, 0x32, 0x0d, 0x7c, 0x51, 0xf9, 0xcc, 0xa1, 0xcf, 0x80, 0x6c, 0x25, 0x82, 0x4b, 0x81,
0x46, 0x76, 0x44, 0x9a, 0xf2, 0xe7, 0xe2, 0xac, 0x8c, 0xbb, 0xe5, 0x8c, 0xe7, 0xd9, 0xad, 0x94,
0xb2, 0x4b, 0x6f, 0x00, 0xe9, 0x89, 0x50, 0x48, 0x61, 0xa6, 0xfb, 0x5f, 0xf4, 0xd2, 0x81, 0xf5,
0xe1, 0x6c, 0x5e, 0x72, 0x1d, 0xaa, 0x6a, 0x55, 0xa0, 0xb1, 0xe6, 0xad, 0x8b, 0x45, 0x9e, 0xf2,
0x2d, 0xc2, 0x90, 0x81, 0x86, 0x56, 0x29, 0x7a, 0xf9, 0x9a, 0x81, 0x4d, 0xb5, 0xd2, 0x0d, 0x63,
0xca, 0x45, 0x53, 0xab, 0x85, 0xa9, 0xf2, 0x9a, 0x31, 0xd6, 0xee, 0xda, 0x70, 0xcf, 0x6b, 0x8d,
0xfa, 0xf0, 0xb6, 0xd6, 0x70, 0xef, 0x88, 0x07, 0x21, 0xdf, 0x0f, 0xff, 0x53, 0x45, 0xa6, 0x1c,
0xf7, 0x60, 0x11, 0x65, 0xfb, 0x3d, 0xd3, 0xdb, 0x16, 0xa4, 0xdf, 0x41, 0x31, 0x26, 0xbb, 0x7c,
0x24, 0x8c, 0x36, 0x7c, 0xe7, 0xf1, 0x56, 0xce, 0x8e, 0x57, 0x19, 0x56, 0xa3, 0xa5, 0x56, 0xb5,
0xab, 0x0c, 0x23, 0x40, 0x6f, 0x43, 0x6d, 0xe0, 0x1f, 0x88, 0x11, 0x27, 0x1f, 0xc2, 0x22, 0x7a,
0x28, 0x52, 0xd3, 0xd1, 0x17, 0x66, 0x2a, 0xc5, 0x2c, 0x9d, 0xa6, 0x26, 0xb2, 0xb9, 0x3e, 0x7d,
0x04, 0x8b, 0xc6, 0x30, 0x4e, 0xf4, 0x29, 0x15, 0xb7, 0x3c, 0xe4, 0x3a, 0xd4, 0xd0, 0xd9, 0xd4,
0xab, 0xce, 0x5a, 0x45, 0x3c, 0x33, 0x64, 0xba, 0x0d, 0xee, 0x53, 0xd6, 0x57, 0x83, 0x8d, 0x0e,
0x5b, 0xa3, 0x06, 0x52, 0xae, 0x7c, 0x1d, 0xa7, 0xd2, 0xa4, 0x15, 0xdf, 0x0a, 0xf7, 0x38, 0x4e,
0x24, 0xa6, 0xb4, 0xc5, 0xf0, 0x4d, 0x53, 0xa8, 0xee, 0xc6, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef,
0x19, 0x1d, 0x95, 0x7e, 0x8f, 0xbc, 0x8b, 0xea, 0x4d, 0x26, 0x5b, 0x85, 0x13, 0x4f, 0x59, 0x9f,
0xa1, 0xe1, 0x6b, 0xd0, 0xea, 0xa7, 0x5b, 0x71, 0x9c, 0x0c, 0x83, 0x88, 0xcb, 0x38, 0x31, 0x27,
0x6f, 0x1a, 0x89, 0xa3, 0x25, 0xb9, 0xd4, 0xc7, 0xa8, 0xc1, 0x34, 0x40, 0xef, 0x42, 0x5b, 0x19,
0x45, 0xc0, 0xb6, 0xc7, 0x2a, 0xd4, 0x14, 0x2e, 0x77, 0xc2, 0x40, 0x85, 0x86, 0x4a, 0x59, 0xc3,
0xb7, 0x5a, 0xc3, 0xf6, 0x91, 0x88, 0x64, 0xa9, 0xc1, 0x10, 0x46, 0x05, 0x2d, 0xa6, 0x01, 0x42,
0x75, 0x80, 0x26, 0x92, 0xe5, 0x22, 0x12, 0x85, 0x65, 0x48, 0xa3, 0x3f, 0x3b, 0x00, 0xd6, 0xa1,
0x2c, 0xcd, 0x45, 0x9c, 0xd3, 0x45, 0x48, 0xd7, 0x36, 0x8a, 0x19, 0xae, 0x76, 0xc1, 0xa5, 0xf1,
0xcc, 0x36, 0xd2, 0xc7, 0x45, 0x23, 0xe9, 0x92, 0x5e, 0x9e, 0x69, 0x00, 0x6d, 0xb5, 0x68, 0xa7,
0xc7, 0xd0, 0x2c, 0xe1, 0x4f, 0x69, 0x2a, 0xdb, 0x25, 0x95, 0x59, 0x95, 0x88, 0x37, 0x2a, 0x6d,
0xaf, 0x3c, 0x84, 0x66, 0x09, 0x3d, 0x57, 0x63, 0x17, 0x2e, 0x4c, 0x8f, 0xad, 0x3d, 0x07, 0xb3,
0x68, 0x1a, 0x40, 0x6b, 0x2b, 0xcc, 0x52, 0x29, 0x12, 0xa3, 0x4e, 0xdd, 0x10, 0x8d, 0xc8, 0x8b,
0x57, 0x20, 0xe6, 0xd7, 0x8f, 0x5c, 0x83, 0x05, 0x95, 0x46, 0x3d, 0x7d, 0x27, 0x73, 0xac, 0x89,
0xf4, 0x19, 0xd4, 0x37, 0x07, 0xfd, 0x07, 0x49, 0x9c, 0x8d, 0xe7, 0x3a, 0x6d, 0x3f, 0x90, 0x2a,
0xa5, 0x0f, 0xa4, 0xb6, 0x3e, 0xf6, 0x2e, 0x7e, 0x24, 0xe0, 0x65, 0x6f, 0xeb, 0xcb, 0x5e, 0x35,
0x18, 0xae, 0xd6, 0xf5, 0x8a, 0xde, 0xac, 0x6a, 0xe8, 0xcf, 0xb3, 0x9f, 0xec, 0x8d, 0x76, 0x8b,
0x1b, 0xad, 0x94, 0xea, 0xf5, 0xf7, 0x7f, 0x2a, 0xfd, 0xbb, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x4b,
0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x12, 0x4a, 0xfe, 0x9b, 0x78, 0xdf, 0x64, 0xdb, 0x65,
0x1a, 0x78, 0x9d, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, 0xb3, 0x27, 0x59, 0xcb, 0x2c, 0xe4, 0x26,
0x2c, 0x0e, 0xe2, 0x2c, 0xf1, 0xf3, 0xf6, 0x2d, 0xad, 0x55, 0xed, 0x99, 0x26, 0x33, 0xcb, 0x46,
0x9e, 0x00, 0xd9, 0x4b, 0x78, 0x94, 0x86, 0x5c, 0x39, 0x6b, 0x85, 0xeb, 0xb3, 0x9f, 0x05, 0x25,
0x9e, 0x29, 0x3d, 0x73, 0x84, 0xc9, 0x27, 0xe5, 0xf9, 0xf4, 0x16, 0xd1, 0xeb, 0x4b, 0xd3, 0x5e,
0x9b, 0x96, 0x2f, 0xcf, 0xf1, 0x9d, 0x99, 0x4e, 0xf5, 0x6a, 0x28, 0x78, 0xa5, 0x10, 0x9c, 0x22,
0xb3, 0x69, 0x6e, 0xfa, 0x93, 0x03, 0x4b, 0x65, 0xcf, 0x5e, 0x6b, 0x2f, 0xe4, 0x05, 0xaf, 0x9c,
0xfd, 0xdd, 0x61, 0x0b, 0x5e, 0x9d, 0xf7, 0xa5, 0xb7, 0x50, 0xfe, 0x16, 0xc9, 0xe0, 0xca, 0x29,
0xe9, 0x7a, 0x03, 0xa7, 0x3a, 0xd0, 0x7c, 0xcc, 0x13, 0x19, 0x28, 0x95, 0xe6, 0xd0, 0x2e, 0xb0,
0x32, 0x8a, 0x1e, 0xc2, 0xd5, 0x13, 0xcd, 0xb7, 0x15, 0x8f, 0xc6, 0xaa, 0xcb, 0xdf, 0xa0, 0x09,
0xd5, 0xa2, 0x4e, 0x12, 0xd3, 0x7e, 0x0d, 0xa6, 0x01, 0xfa, 0x39, 0x5c, 0x1e, 0x08, 0x59, 0x6a,
0x3d, 0x3b, 0x43, 0x1d, 0x70, 0x77, 0xc5, 0x8b, 0x53, 0x02, 0x54, 0x24, 0xfa, 0x15, 0x78, 0x4f,
0xc7, 0x43, 0x2e, 0xc5, 0xb9, 0xa4, 0x37, 0xa1, 0xbe, 0x17, 0x8f, 0xe3, 0x30, 0x7e, 0x7e, 0x7c,
0xc6, 0x2e, 0xf3, 0x60, 0x51, 0x5f, 0x25, 0xbd, 0x1c, 0x1b, 0xcc, 0x82, 0xf4, 0xa2, 0x1a, 0x53,
0x9f, 0x87, 0x7e, 0x16, 0x2a, 0x37, 0xd4, 0x47, 0x73, 0x4a, 0x85, 0x19, 0x04, 0x8e, 0x89, 0x2b,
0x1d, 0xba, 0x7b, 0x88, 0xb0, 0x87, 0x4e, 0x43, 0xe4, 0x53, 0x68, 0x96, 0xb8, 0x4d, 0x02, 0x2f,
0xcf, 0xcc, 0x8b, 0x26, 0xb2, 0x32, 0x27, 0xfd, 0xcd, 0x99, 0x92, 0x3c, 0x71, 0xca, 0x8d, 0xc1,
0x23, 0x5d, 0x94, 0x3a, 0x33, 0x90, 0x8a, 0x75, 0x7b, 0xe2, 0x87, 0x59, 0xaa, 0x48, 0xfa, 0x7a,
0x17, 0x08, 0x15, 0xab, 0xfa, 0x33, 0x8c, 0x33, 0x69, 0x36, 0xa7, 0x05, 0xd5, 0x4f, 0x5a, 0x4f,
0xf0, 0x61, 0x18, 0x44, 0x02, 0xbb, 0xd4, 0x65, 0x39, 0x4c, 0x6e, 0xea, 0x6d, 0x6f, 0x47, 0x6d,
0x6d, 0xae, 0xfb, 0xc8, 0xa1, 0x2f, 0x41, 0x4a, 0x09, 0xb4, 0x67, 0x49, 0x9b, 0xed, 0xdf, 0x5f,
0xad, 0x3b, 0x7f, 0xbc, 0x5a, 0x77, 0xfe, 0x7c, 0xb5, 0xee, 0xfc, 0xf2, 0xd7, 0xfa, 0x5b, 0xfb,
0x35, 0xfc, 0xd7, 0xbe, 0xfd, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x76, 0xf5, 0x59, 0x94,
0x0f, 0x00, 0x00,
// 1458 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45,
0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf,
0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0,
0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa,
0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0,
0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97,
0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75,
0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e,
0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec,
0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17,
0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8,
0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba,
0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91,
0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47,
0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee,
0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27,
0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78,
0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93,
0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3,
0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17,
0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29,
0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb,
0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38,
0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0,
0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45,
0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21,
0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29,
0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c,
0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c,
0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96,
0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37,
0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2,
0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87,
0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4,
0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf,
0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89,
0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a,
0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3,
0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b,
0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e,
0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf,
0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3,
0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce,
0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e,
0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce,
0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15,
0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0,
0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae,
0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79,
0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64,
0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47,
0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78,
0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82,
0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68,
0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5,
0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5,
0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b,
0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce,
0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2,
0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44,
0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae,
0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27,
0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d,
0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02,
0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56,
0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0,
0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91,
0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07,
0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38,
0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c,
0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04,
0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92,
0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2,
0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59,
0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98,
0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01,
0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e,
0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10,
0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19,
0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2,
0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d,
0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe,
0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3,
0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a,
0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9,
0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c,
0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f,
0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b,
0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb,
0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05,
0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10,
0x00, 0x00,
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
@ -3034,6 +3102,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if m.Meta != nil {
{
size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i])
@ -3080,6 +3153,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x20
}
if m.Meta != nil {
{
size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i])
@ -3220,6 +3298,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x20
}
if len(m.Views) > 0 {
for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.Views[iNdEx])
@ -3342,6 +3425,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) {
dAtA[i] = 0x22
}
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x10
}
if len(m.Name) > 0 {
i -= len(m.Name)
copy(dAtA[i:], m.Name)
@ -3422,6 +3510,18 @@ func (m *Node) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.GRPCURI != nil {
{
size, err := m.GRPCURI.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintPrivate(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x2a
}
if len(m.State) > 0 {
i -= len(m.State)
copy(dAtA[i:], m.State)
@ -3635,6 +3735,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if len(m.Fields) > 0 {
for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- {
{
@ -3683,21 +3788,26 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.CreatedAt != 0 {
i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt))
i--
dAtA[i] = 0x18
}
if len(m.AvailableShards) > 0 {
dAtA18 := make([]byte, len(m.AvailableShards)*10)
var j17 int
dAtA19 := make([]byte, len(m.AvailableShards)*10)
var j18 int
for _, num := range m.AvailableShards {
for num >= 1<<7 {
dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80)
dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j17++
j18++
}
dAtA18[j17] = uint8(num)
j17++
dAtA19[j18] = uint8(num)
j18++
}
i -= j17
copy(dAtA[i:], dAtA18[:j17])
i = encodeVarintPrivate(dAtA, i, uint64(j17))
i -= j18
copy(dAtA[i:], dAtA19[:j18])
i = encodeVarintPrivate(dAtA, i, uint64(j18))
i--
dAtA[i] = 0x12
}
@ -3735,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.Schema != nil {
{
size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintPrivate(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x22
}
if len(m.Nodes) > 0 {
for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- {
{
@ -4738,6 +4860,9 @@ func (m *CreateIndexMessage) Size() (n int) {
l = m.Meta.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4762,6 +4887,9 @@ func (m *CreateFieldMessage) Size() (n int) {
l = m.Meta.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4831,6 +4959,9 @@ func (m *Field) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4865,6 +4996,9 @@ func (m *Index) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if len(m.Fields) > 0 {
for _, e := range m.Fields {
l = e.Size()
@ -4925,6 +5059,10 @@ func (m *Node) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.GRPCURI != nil {
l = m.GRPCURI.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5012,6 +5150,9 @@ func (m *IndexStatus) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5035,6 +5176,9 @@ func (m *FieldStatus) Size() (n int) {
}
n += 1 + sovPrivate(uint64(l)) + l
}
if m.CreatedAt != 0 {
n += 1 + sovPrivate(uint64(m.CreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -5061,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if m.Schema != nil {
l = m.Schema.Size()
n += 1 + l + sovPrivate(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -6996,6 +7144,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7150,6 +7317,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7559,6 +7745,25 @@ func (m *Field) Unmarshal(dAtA []byte) error {
}
m.Views = append(m.Views, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7733,6 +7938,25 @@ func (m *Index) Unmarshal(dAtA []byte) error {
}
m.Name = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType)
@ -8114,6 +8338,42 @@ func (m *Node) Unmarshal(dAtA []byte) error {
}
m.State = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field GRPCURI", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthPrivate
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.GRPCURI == nil {
m.GRPCURI = &URI{}
}
if err := m.GRPCURI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8621,6 +8881,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8783,6 +9062,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error {
} else {
return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType)
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType)
}
m.CreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.CreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -8935,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthPrivate
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Schema == nil {
m.Schema = &Schema{}
}
if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])

8
internal/private.proto
View file

@ -64,12 +64,14 @@ message DeleteIndexMessage {
message CreateIndexMessage {
string Index = 1;
IndexMeta Meta = 2;
int64 CreatedAt = 3;
}
message CreateFieldMessage {
string Index = 1;
string Field = 2;
FieldOptions Meta = 3;
int64 CreatedAt = 4;
}
message DeleteFieldMessage {
@ -87,6 +89,7 @@ message Field {
string Name = 1;
FieldOptions Meta = 2;
repeated string Views = 3;
int64 CreatedAt = 4;
}
message Schema {
@ -95,6 +98,7 @@ message Schema {
message Index {
string Name = 1;
int64 CreatedAt = 2;
IndexMeta Options = 5;
repeated Field Fields = 4;
}
@ -110,6 +114,7 @@ message Node {
URI URI = 2;
bool IsCoordinator = 3;
string State = 4;
URI GRPCURI = 5;
}
message NodeStateMessage {
@ -131,17 +136,20 @@ message NodeStatus {
message IndexStatus {
string Name = 1;
repeated FieldStatus Fields = 2;
int64 CreatedAt = 3;
}
message FieldStatus {
string Name = 1;
repeated uint64 AvailableShards = 2;
int64 CreatedAt = 3;
}
message ClusterStatus {
string ClusterID = 1;
string State = 2;
repeated Node Nodes = 3;
Schema Schema = 4;
}
message BSIGroup {

400
internal/public.pb.go
View file

@ -1183,6 +1183,8 @@ type ImportRequest struct {
RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"`
ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"`
Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 {
return nil
}
func (m *ImportRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type ImportValueRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -1286,6 +1302,8 @@ type ImportValueRequest struct {
Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"`
FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"`
StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string {
return nil
}
func (m *ImportValueRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportValueRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type TranslateKeysRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct {
Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"`
Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"`
Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1726,12 +1760,27 @@ func (m *ImportRoaringRequest) GetBlock() uint64 {
return 0
}
func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 {
if m != nil {
return m.FieldCreatedAt
}
return 0
}
type ImportColumnAttrsRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"`
AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"`
AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"`
IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1805,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 {
return nil
}
func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 {
if m != nil {
return m.IndexCreatedAt
}
return 0
}
func init() {
proto.RegisterType((*Row)(nil), "internal.Row")
proto.RegisterType((*SignedRow)(nil), "internal.SignedRow")
@ -1837,83 +1893,86 @@ func init() {
func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) }
var fileDescriptor_413a91106d7bcce8 = []byte{
// 1207 bytes of a gzipped FileDescriptorProto
// 1258 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45,
0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b,
0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b,
0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00,
0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d,
0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa,
0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a,
0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b,
0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e,
0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf,
0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92,
0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a,
0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85,
0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23,
0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42,
0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3,
0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5,
0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87,
0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54,
0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b,
0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87,
0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a,
0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e,
0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36,
0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75,
0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72,
0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae,
0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c,
0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde,
0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68,
0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2,
0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c,
0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47,
0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0,
0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3,
0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e,
0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17,
0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97,
0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61,
0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb,
0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29,
0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef,
0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c,
0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30,
0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea,
0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0,
0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5,
0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c,
0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d,
0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b,
0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43,
0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6,
0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b,
0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c,
0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac,
0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2,
0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9,
0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1,
0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67,
0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe,
0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90,
0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e,
0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc,
0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d,
0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9,
0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6,
0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed,
0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42,
0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde,
0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6,
0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc,
0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6,
0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf,
0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7,
0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc,
0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00,
0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01,
0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b,
0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c,
0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55,
0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5,
0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10,
0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c,
0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda,
0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6,
0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82,
0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39,
0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1,
0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c,
0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c,
0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a,
0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f,
0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc,
0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21,
0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b,
0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4,
0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c,
0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06,
0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35,
0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40,
0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce,
0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87,
0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19,
0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72,
0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a,
0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65,
0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07,
0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54,
0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b,
0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33,
0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9,
0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c,
0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39,
0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce,
0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d,
0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90,
0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7,
0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a,
0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98,
0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5,
0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c,
0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a,
0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42,
0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81,
0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5,
0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9,
0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c,
0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41,
0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06,
0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11,
0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd,
0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad,
0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b,
0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99,
0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4,
0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2,
0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0,
0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3,
0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35,
0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a,
0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b,
0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51,
0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71,
0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1,
0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0,
0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea,
0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88,
0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0,
0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda,
0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2,
0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9,
0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb,
0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00,
0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00,
}
func (m *Row) Marshal() (dAtA []byte, err error) {
@ -2985,6 +3044,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x50
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x48
}
if len(m.ColumnKeys) > 0 {
for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.ColumnKeys[iNdEx])
@ -3104,6 +3173,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x58
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x50
}
if len(m.StringValues) > 0 {
for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.StringValues[iNdEx])
@ -3446,6 +3525,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.FieldCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt))
i--
dAtA[i] = 0x30
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x28
}
if m.Block != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.Block))
i--
@ -3509,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.IndexCreatedAt != 0 {
i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt))
i--
dAtA[i] = 0x30
}
if len(m.ColumnIDs) > 0 {
dAtA36 := make([]byte, len(m.ColumnIDs)*10)
var j35 int
@ -4080,6 +4174,12 @@ func (m *ImportRequest) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4132,6 +4232,12 @@ func (m *ImportValueRequest) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4270,6 +4376,12 @@ func (m *ImportRoaringRequest) Size() (n int) {
if m.Block != 0 {
n += 1 + sovPublic(uint64(m.Block))
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.FieldCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.FieldCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -4306,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) {
}
n += 1 + sovPublic(uint64(l)) + l
}
if m.IndexCreatedAt != 0 {
n += 1 + sovPublic(uint64(m.IndexCreatedAt))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
@ -7525,6 +7640,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
}
m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 9:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -7932,6 +8085,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 11:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -8771,6 +8962,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType)
}
m.FieldCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FieldCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -9016,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error {
} else {
return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType)
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType)
}
m.IndexCreatedAt = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.IndexCreatedAt |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])

9
internal/public.proto
View file

@ -116,6 +116,7 @@ message QueryResult {
message ImportRequest {
string Index = 1;
string Field = 2;
uint64 Shard = 3;
repeated uint64 RowIDs = 4;
@ -123,6 +124,8 @@ message ImportRequest {
repeated string RowKeys = 7;
repeated string ColumnKeys = 8;
repeated int64 Timestamps = 6;
int64 IndexCreatedAt = 9;
int64 FieldCreatedAt = 10;
}
message ImportValueRequest {
@ -134,6 +137,8 @@ message ImportValueRequest {
repeated int64 Values = 6;
repeated double FloatValues = 8;
repeated string StringValues = 9;
int64 IndexCreatedAt = 10;
int64 FieldCreatedAt = 11;
}
message TranslateKeysRequest {
@ -166,6 +171,9 @@ message ImportRoaringRequest {
repeated ImportRoaringRequestView views = 2;
string Action = 3;
uint64 Block = 4;
int64 IndexCreatedAt = 5;
int64 FieldCreatedAt = 6;
}
message ImportColumnAttrsRequest {
@ -174,4 +182,5 @@ message ImportColumnAttrsRequest {
string AttrKey = 3;
repeated string AttrVals = 4;
repeated uint64 ColumnIDs = 5;
int64 IndexCreatedAt = 6;
}

7
metrics.go
View file

@ -58,4 +58,11 @@ const (
MetricStackInuse = "stack_inuse"
MetricMallocs = "mallocs"
MetricFrees = "frees"
MetricTransactionStart = "transaction_start"
MetricTransactionEnd = "trasaction_end"
MetricTransactionBlocked = "transaction_blocked"
MetricExclusiveTransactionRequest = "transaction_exclusive_request"
MetricExclusiveTransactionActive = "transaction_exclusive_active"
MetricExclusiveTransactionEnd = "trasaction_exclusive_end"
MetricExclusiveTransactionBlocked = "transaction_exclusive_blocked"
)

17
pilosa.go
View file

@ -17,6 +17,7 @@ package pilosa
import (
"encoding/json"
"regexp"
"time"
"github.com/pkg/errors"
)
@ -65,6 +66,9 @@ var (
// we won't need this error at all by 2.0 though.
ErrClusterDoesNotOwnShard = errors.New("node does not own shard")
// ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match
ErrPreconditionFailed = errors.New("precondition failed")
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
@ -124,6 +128,15 @@ func newNotFoundError(err error) NotFoundError {
return NotFoundError{err}
}
type PreconditionFailedError struct {
error
}
// newPreconditionFailedError returns err wrapped in a PreconditionFailedError.
func newPreconditionFailedError(err error) PreconditionFailedError {
return PreconditionFailedError{err}
}
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
@ -191,6 +204,10 @@ func stringSlicesAreEqual(a, b []string) bool {
return true
}
func timestamp() int64 {
return time.Now().UnixNano()
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {

7
pql/ast.go
View file

@ -134,6 +134,13 @@ func (q *Query) validateArgField(elem *callStackElem) {
}
func (q *Query) addVal(val interface{}) {
if vs, ok := val.(string); ok {
vsu, err := Unquote(vs)
if err != nil {
panic(err)
}
val = vsu
}
elem := q.lastCallStackElem()
if elem == nil || elem.lastField == "" {
panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))

78
pql/parser.go
View file

@ -18,7 +18,9 @@ import (
"fmt"
"io"
"io/ioutil"
"strconv"
"strings"
"unicode/utf8"
"github.com/pkg/errors"
)
@ -95,3 +97,79 @@ func (p *parser) Parse() (*Query, error) {
return &p.Query, nil
}
// Unquote interprets s as a single-quoted, double-quoted, or
// backquoted Go string literal, returning the string value that s
// quotes. It is a copy of stdlib's strconv.Unquote, but modified so
// that if s is single-quoted, it can still be a string rather than
// only character literal. This version of Unquote also accepts
// unquoted strings and passes them back unchanged.
func Unquote(s string) (string, error) {
n := len(s)
if n < 2 {
return s, nil
}
quote := s[0]
if quote != '"' && quote != '\'' && quote != '`' {
return s, nil
}
if quote != s[n-1] {
return "", strconv.ErrSyntax
}
s = s[1 : n-1]
if quote == '`' {
if contains(s, '`') {
return "", strconv.ErrSyntax
}
if contains(s, '\r') {
// -1 because we know there is at least one \r to remove.
buf := make([]byte, 0, len(s)-1)
for i := 0; i < len(s); i++ {
if s[i] != '\r' {
buf = append(buf, s[i])
}
}
return string(buf), nil
}
return s, nil
}
if quote != '"' && quote != '\'' {
return "", strconv.ErrSyntax
}
if contains(s, '\n') {
return "", strconv.ErrSyntax
}
// Is it trivial? Avoid allocation.
if !contains(s, '\\') && !contains(s, quote) {
switch quote {
case '"', '\'':
if utf8.ValidString(s) {
return s, nil
}
}
}
var runeTmp [utf8.UTFMax]byte
buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
for len(s) > 0 {
c, multibyte, ss, err := strconv.UnquoteChar(s, quote)
if err != nil {
return "", err
}
s = ss
if c < utf8.RuneSelf || !multibyte {
buf = append(buf, byte(c))
} else {
n := utf8.EncodeRune(runeTmp[:], c)
buf = append(buf, runeTmp[:n]...)
}
}
return string(buf), nil
}
// contains reports whether the string contains the byte c.
func contains(s string, c byte) bool {
return strings.ContainsRune(s, rune(c))
}

70
pql/parser_test.go
View file

@ -16,6 +16,7 @@ package pql_test
import (
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa/v2/pql"
@ -173,7 +174,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with condition arguments.
t.Run("WithCondition", func(t *testing.T) {
q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null, n == null)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -185,10 +186,77 @@ func TestParser_Parse(t *testing.T) {
"y": &pql.Condition{Op: pql.GTE, Value: int64(100)},
"z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
"m": &pql.Condition{Op: pql.NEQ, Value: nil},
"n": &pql.Condition{Op: pql.EQ, Value: nil},
},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
}
func TestUnquote(t *testing.T) {
tests := []struct {
name string
value string
exp string
expErr string
}{
{
name: "simple double",
value: `"hello"`,
exp: "hello",
},
{
name: "simple single",
value: `'hello'`,
exp: "hello",
},
{
name: "double with esc",
value: `"he\"llo"`,
exp: "he\"llo",
},
{
name: "single with esc",
value: `'he\'llo'`,
exp: "he'llo",
},
{
name: "single with backslash and esc",
value: `'he\\\'llo'`,
exp: `he\'llo`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := pql.Unquote(test.value)
if testErr(t, test.expErr, err) {
return
}
if got != test.exp {
t.Errorf("exp: '%s'\ngot: '%s'", test.exp, got)
}
})
}
}
func testErr(t *testing.T, exp string, actual error) (done bool) {
t.Helper()
if exp == "" && actual == nil {
return false
}
if exp == "" && actual != nil {
t.Fatalf("unexpected error: %v", actual)
}
if exp != "" && actual == nil {
t.Fatalf("expected error like '%s'", exp)
}
if !strings.Contains(actual.Error(), exp) {
t.Fatalf("unmatched errs exp/got\n%s\n%v", exp, actual)
}
return true
}

16
pql/pql.peg
View file

@ -64,8 +64,8 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) }
)
itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
/ < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }
/ '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
/ < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) }
/ < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) }
)
float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) }
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) }
@ -74,8 +74,8 @@ decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) }
)
doublequotedstring <- ( '\\"' / '\\\\' / [^"] )*
singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )*
doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )*
singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )*
fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
@ -83,12 +83,12 @@ reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }
uint <- [1-9] [0-9]* / '0'
col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
/ '\'' <singlequotedstring> '\'' {p.addPosStr("_col", buffer[begin:end])}
/ '"' <doublequotedstring> '"' {p.addPosStr("_col", buffer[begin:end])}
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])}
/ < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])}
)
row <- ( <uint> {p.addPosNum("_row", buffer[begin:end])}
/ '\'' <singlequotedstring> '\'' {p.addPosStr("_row", buffer[begin:end])}
/ '"' <doublequotedstring> '"' {p.addPosStr("_row", buffer[begin:end])}
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])}
/ < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])}
)
open <- '(' sp

811
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

8
pql/pqlpeg_test.go
View file

@ -238,8 +238,16 @@ func TestPEGWorking(t *testing.T) {
name: "RangeEQ",
input: "Row(a == 4)",
ncalls: 1},
{
name: "RangeEQNULL",
input: "Row(a == null)",
ncalls: 1},
{
name: "RangeNEQ",
input: "Row(a != 4)",
ncalls: 1},
{
name: "RangeNEQNull",
input: "Row(a != null)",
ncalls: 1},
{

9
roaring/btree.go
View file

@ -925,6 +925,15 @@ func (e *enumerator) Every(upd func(key uint64, oldV *Container, exists bool) (n
if write {
if nv == nil {
e.t.Delete(i.k)
f, _ := e.t.Seek(e.k)
*e = *f
f.Close()
// we don't want to e.next() here; we'll
// already be on an item with key >= i.k,
// and since we just deleted the item with
// key i.k, that means key is > i.k, which
// makes it the next item.
continue
} else {
e.q.d[e.i].v = nv
}

29
roaring/btree_test.go
View file

@ -11,6 +11,7 @@ import (
"math"
"math/rand"
"path"
"reflect"
"runtime"
"runtime/debug"
"strings"
@ -688,7 +689,7 @@ func TestBtreeDelete0(t *testing.T) {
}
func TestBtreeDelete1(t *testing.T) {
const N = 130000
const N = 13000
for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} {
r := treeNew()
set := r.Set
@ -787,7 +788,7 @@ func benchmarkDelRnd(b *testing.B, n int) {
}
func TestBtreeDelete2(t *testing.T) {
const N = 100000
const N = 10000
for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} {
r := treeNew()
set := r.Set
@ -998,6 +999,28 @@ func TestBtreeEnumeratorPrevSanity(t *testing.T) {
}
}
// TestBtreeEnumeratorEveryRegression is a regression test for a "use-after-free" bug.
// Previously, deleting a container would cause some values to be skipped (and sometimes trigger a race condition).
func TestBtreeEnumeratorEveryRegression(t *testing.T) {
r := treeNew()
r.Set(uint64(10), getDummyC(100))
r.Set(uint64(20), getDummyC(200))
r.Set(uint64(30), getDummyC(300))
e, _ := r.Seek(0)
expect := []uint64{10, 20, 30}
var found []uint64
_ = e.Every(func(key uint64, oldV *Container, exists bool) (*Container, bool) {
found = append(found, key)
return nil, true
})
if !reflect.DeepEqual(expect, found) { // Before the fix, this skipped the 20.
t.Errorf("had %v in bitmap; only found %v", expect, found)
}
}
func BenchmarkBtreeSeekSeq1e3(b *testing.B) {
benchmarkSeekSeq(b, 1e3)
}
@ -1445,7 +1468,7 @@ func TestBtreePut(t *testing.T) {
}
func TestBtreeSeek(t *testing.T) {
const N = 1 << 13
const N = 1 << 11
tr := treeNew()
for i := 0; i < N; i++ {
k := 2*i + 1

5
roaring/naive_test.go
View file

@ -15,6 +15,7 @@
package roaring
import (
"math/rand"
"reflect"
"testing"
)
@ -110,13 +111,15 @@ func TestUnionSlice(t *testing.T) {
}
func TestMaxInSlice(t *testing.T) {
// arbitrary, we just want to get the same values every time
r := rand.New(rand.NewSource(23))
a := []uint64{1, 4, 9, 5, 24, 13}
v := maxInSlice(a)
if uint64(24) != v {
t.Fatalf("expected %v, but got %v", uint64(24), v)
}
for i := uint64(1000); i <= uint64(100000); i++ {
for i := uint64(1000); i <= uint64(100000); i += uint64(r.Intn(35)) + 1 {
a = append(a, i)
if v = maxInSlice(a); v != i {
t.Fatalf("expected %v, but got %v", i, v)

225
roaring/roaring.go
View file

@ -160,7 +160,7 @@ type Containers interface {
// replace the given container.
UpdateEvery(fn func(uint64, *Container, bool) (*Container, bool))
// Iterator returns a Contiterator which after a call to Next(), a call to Value() will
// Iterator returns a ContainterIterator which after a call to Next(), a call to Value() will
// return the first container at or after key. found will be true if a
// container is found at key.
Iterator(key uint64) (citer ContainerIterator, found bool)
@ -1738,7 +1738,9 @@ type baseRoaringIterator struct {
currentN int
currentLen int
currentPointer *uint16
currentDataOffset uint32
currentDataOffset uint64
prevOffset32 uint32
chunkOffset uint64
lastDataOffset int64
lastErr error
}
@ -1752,6 +1754,8 @@ func (b *baseRoaringIterator) SilenceLint() {
_ = b.offsets
_ = b.headers
_ = b.currentIdx
_ = b.chunkOffset
_ = b.prevOffset32
}
type pilosaRoaringIterator struct {
@ -1787,14 +1791,14 @@ func newOfficialRoaringIterator(data []byte) (*officialRoaringIterator, error) {
r.headers = data[headerOffset:offsetOffset]
// note: offsets are only actually used with the no-run headers.
if r.haveRuns {
r.currentDataOffset = uint32(offsetOffset)
r.currentDataOffset = uint64(offsetOffset)
} else {
if len(r.data) < offsetOffset+int(r.keys*4) {
return nil, fmt.Errorf("insufficient data for offsets (need %d bytes, found %d)",
r.keys*4, len(r.data)-offsetOffset)
}
r.offsets = data[offsetOffset : offsetOffset+int(r.keys*4)]
r.currentDataOffset = uint32(offsetOffset)
r.currentDataOffset = uint64(offsetOffset)
}
// set key to -1; user should call Next first.
r.currentIdx = -1
@ -1838,7 +1842,11 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) {
// if there's no containers, we want to act as though data started at the end
// of the list of offsets, which was also empty, so we don't think the entire thing
// is actually a malformed op
r.currentDataOffset = uint32(offsetEnd)
r.prevOffset32 = uint32(offsetEnd)
r.currentDataOffset = uint64(offsetEnd)
// it's possible that there's so many headers that we're actually over
// 4GB into the file already.
r.chunkOffset = r.currentDataOffset &^ ((1 << 32) - 1)
// set key to -1; user should call Next first.
r.currentIdx = -1
r.currentKey = ^uint64(0)
@ -1895,7 +1903,12 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
r.currentKey = binary.LittleEndian.Uint64(header[0:8])
r.currentType = byte(binary.LittleEndian.Uint16(header[8:10]))
r.currentN = int(binary.LittleEndian.Uint16(header[10:12])) + 1
r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
offset32 := binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
if offset32 < r.prevOffset32 {
r.chunkOffset += (1 << 32)
}
r.prevOffset32 = offset32
r.currentDataOffset = r.chunkOffset + uint64(offset32)
// a run container keeps its data after an initial 2 byte length header
var runCount uint16
@ -1903,7 +1916,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize])
r.currentDataOffset += 2
}
if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize {
if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize {
r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d",
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data)))
return r.Current()
@ -1926,7 +1939,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data)))
return r.Current()
}
r.currentDataOffset += uint32(size)
r.currentDataOffset += uint64(size)
r.lastErr = nil
return r.Current()
}
@ -1949,7 +1962,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
// with runs, we can't actually look up offsets; the format just stores
// things sequentially. so we have to actually track the offset in that case.
if !r.haveRuns {
r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
r.currentDataOffset = uint64(binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]))
}
// a run container keeps its data after an initial 2 byte length header
var runCount uint16
@ -1962,7 +1975,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize])
r.currentDataOffset += 2
}
if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize {
if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize {
r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d",
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data)))
return r.Current()
@ -1994,7 +2007,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data)))
return r.Current()
}
r.currentDataOffset += uint32(size)
r.currentDataOffset += uint64(size)
r.lastErr = nil
return r.Current()
}
@ -2012,14 +2025,14 @@ func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut
if c.Mapped() {
mappedIn++
} else {
err = fmt.Errorf("container key %d, addr %x, inside %x+%d\n",
err = fmt.Errorf("container key %d, addr %x, inside %x+%d",
key, dptr, from, to-from)
errs++
unmappedIn++
}
} else {
if c.Mapped() {
err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped\n",
err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped",
key, dptr, from, to-from)
errs++
mappedOut++
@ -4539,50 +4552,95 @@ func (c *Container) bitmapZeroRange(i, j uint64) {
c.setN(n)
}
// equals reports whether two containers are equal.
func (c *Container) equals(c2 *Container) bool {
if c == nil || c2 == nil {
if c != c2 {
return false
func typePair(ct1, ct2 byte) int {
return int((ct1 << 4) | ct2)
}
// compareArrayBitmap actually only verifies that everything in the array
// is in the bitmap. It's used only after comparing the N for the containers,
// so if there's anything in the bitmap that's not in the array, either there's
// something in the array that's not in the bitmap, or we didn't get here.
func compareArrayBitmap(a []uint16, b []uint64) error {
for _, v := range a {
w, bit := b[v>>6], v&63
if w>>bit&1 == 0 {
return fmt.Errorf("value %d missing", v)
}
}
if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() {
return false
return nil
}
// compareArrayRuns determines whether an array matches a provided
// set of runs. As with compareArrayBitmap, it only verifies presence
// of the array's values in the run collection. the run collection
// can't be empty; if it were, N would have been 0, and we wouldn't
// have gotten here.
func compareArrayRuns(a []uint16, r []interval16) error {
ri := 0
ru := r[ri]
ri++
for _, v := range a {
if v < ru.start {
return fmt.Errorf("value %d missing", v)
}
if v > ru.last {
if ri >= len(r) {
return fmt.Errorf("value %d missing", v)
}
ru = r[ri]
ri++
// if they're identical, the array value must be
// the start of the next run.
if v != ru.start {
return fmt.Errorf("value %d missing", v)
}
}
}
if c.typ() == containerArray {
ca, c2a := c.array(), c2.array()
if len(ca) != len(c2a) {
return false
}
for i := 0; i < len(ca); i++ {
if ca[i] != c2a[i] {
return false
}
}
} else if c.typ() == containerBitmap {
cb, c2b := c.bitmap(), c2.bitmap()
if len(cb) != len(c2b) {
return false
}
for i := 0; i < len(cb); i++ {
if cb[i] != c2b[i] {
return false
}
}
} else if c.typ() == containerRun {
cr, c2r := c.runs(), c2.runs()
if len(cr) != len(c2r) {
return false
}
for i := 0; i < len(cr); i++ {
if cr[i] != c2r[i] {
return false
}
}
} else {
panic(fmt.Sprintf("unknown container type: %v", c.typ()))
return nil
}
// compareArrayArray reports whether everything in a1 is equal to everything
// in a2.
func compareArrayArray(a1, a2 []uint16) error {
if len(a1) != len(a2) {
return fmt.Errorf("unexpected length mismatch, %d vs %d", len(a1), len(a2))
}
return true
for i := range a1 {
if a1[i] != a2[i] {
return fmt.Errorf("item %d: %d vs %d", i, a1[i], a2[i])
}
}
return nil
}
// BitwiseCompare reports whether two containers are equal. It returns
// an error describing any difference it finds. This is mostly intended
// for use in tests that expect equality.
func (c *Container) BitwiseCompare(c2 *Container) error {
if c.N() != c2.N() {
return errors.New("containers are different lengths")
}
if c.N() == 0 {
return nil
}
switch typePair(c.typ(), c2.typ()) {
case typePair(containerArray, containerArray):
return compareArrayArray(c.array(), c2.array())
case typePair(containerArray, containerBitmap):
return compareArrayBitmap(c.array(), c2.bitmap())
case typePair(containerBitmap, containerArray):
return compareArrayBitmap(c2.array(), c.bitmap())
case typePair(containerArray, containerRun):
return compareArrayRuns(c.array(), c2.runs())
case typePair(containerRun, containerArray):
return compareArrayRuns(c2.array(), c.runs())
default:
c3 := xor(c, c2)
if c3.N() != 0 {
return fmt.Errorf("%d bits differenct between containers", c3.N())
}
}
return nil
}
func unionArrayBitmap(a, b *Container) *Container {
@ -5743,7 +5801,37 @@ func xorBitmapRun(a, b *Container) *Container {
return output
}
// CompareEquality is used mostly in test cases to confirm that two bitmaps came
// CompareBitmapSlice checks whether a bitmap has the same values in it
// that a provided slice does.
func CompareBitmapSlice(b *Bitmap, vals []uint64) (bool, error) {
count := b.Count()
if count != uint64(len(vals)) {
return false, fmt.Errorf("length mismatch: bitmap has %d bits, slice has %d", count, len(vals))
}
for _, v := range vals {
if !b.Contains(v) {
return false, fmt.Errorf("bitmap lacks expected value %d", v)
}
}
return true, nil
}
// CompareBitmapMap checks whether a bitmap has the same values in it
// that a provided map[uint64]struct{} has as keys.
func CompareBitmapMap(b *Bitmap, vals map[uint64]struct{}) (bool, error) {
count := b.Count()
if count != uint64(len(vals)) {
return false, fmt.Errorf("length mismatch: bitmap has %d bits, map has %d", count, len(vals))
}
for v := range vals {
if !b.Contains(v) {
return false, fmt.Errorf("bitmap lacks expected value %d", v)
}
}
return true, nil
}
// BitwiseEqual is used mostly in test cases to confirm that two bitmaps came
// out the same. It does not expect corresponding opN, or OpWriter, but expects
// identical bit contents. It does not expect identical representations; a bitmap
// container can be identical to an array container. It returns a boolean value,
@ -5812,35 +5900,6 @@ func (b *Bitmap) BitwiseEqual(c *Bitmap) (bool, error) {
return true, nil
}
func bitmapsEqual(b, c *Bitmap) error { // nolint: deadcode
statsHit("bitmapsEqual")
if b.OpWriter != c.OpWriter {
return errors.New("opWriters not equal")
}
if b.opN != c.opN {
return errors.New("opNs not equal")
}
biter, _ := b.Containers.Iterator(0)
citer, _ := c.Containers.Iterator(0)
bn, cn := biter.Next(), citer.Next()
for ; bn && cn; bn, cn = biter.Next(), citer.Next() {
bk, bc := biter.Value()
ck, cc := citer.Value()
if bk != ck {
return errors.New("keys not equal")
}
if !bc.equals(cc) {
return errors.New("containers not equal")
}
}
if bn && !cn || cn && !bn {
return errors.New("different numbers of containers")
}
return nil
}
func popcount(x uint64) uint64 {
return uint64(bits.OnesCount64(x))
}

89
roaring/roaring_helpers_test.go
View file

@ -14,6 +14,8 @@
package roaring
import "sync"
///////////////////////////////////////////////////////////////////////////
var containerWidth uint64 = 65536
@ -261,51 +263,56 @@ func doContainer(typ byte, data interface{}) *Container {
return nil
}
var makeCts sync.Once
var sampleTestContainers map[byte]map[string]*Container
func setupContainerTests() map[byte]map[string]*Container {
cts := make(map[byte]map[string]*Container)
makeCts.Do(func() {
sampleTestContainers = make(map[byte]map[string]*Container)
// array containers
cts[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
}
// array containers
sampleTestContainers[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
}
// bitmap containers
cts[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
}
// bitmap containers
sampleTestContainers[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
}
// run containers
cts[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
}
// run containers
sampleTestContainers[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
}
})
return cts
return sampleTestContainers
}

111
roaring/roaring_internal_test.go
View file

@ -19,11 +19,13 @@ import (
"encoding/hex"
"fmt"
"io/ioutil"
"math/rand"
"reflect"
"runtime"
"strings"
"testing"
"github.com/pilosa/pilosa/v2/generator"
"github.com/pkg/errors"
)
@ -2630,7 +2632,7 @@ func TestBitmapClone(t *testing.T) {
}
}
c := b.Clone()
if err := bitmapsEqual(b, c); err != nil {
if _, err := b.BitwiseEqual(c); err != nil {
t.Fatalf("Clone Objects not equal: %v\n", err)
}
d := func() *Bitmap { //anybody know how to declare a nil value?
@ -2733,34 +2735,46 @@ func getFunctionName(i interface{}) string {
return y[0]
}
// UnionInPlace is defined at the Bitmap level, but this wrapper lets us insert
// it into our ContainerCombinations tests so that it gets exercised on a wide
// variety of container data.
func unionInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.UnionInPlace(B)
return out.Containers.Get(0)
ret := a.Clone().unionInPlace(b)
ret.Repair()
return ret
}
func differenceInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.DifferenceInPlace(B)
return out.Containers.Get(0)
a = a.Clone()
// this should probably return its new value, but currently does not
a.differenceInPlace(b)
return a
}
func intersectInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.IntersectInPlace(B)
return out.Containers.Get(0)
return a.Clone().intersectInPlace(b)
}
func TestContainerBitwiseCompare(t *testing.T) {
cts := setupContainerTests()
for t1, containers := range cts {
for name, c := range containers {
for t2, other := range cts {
for otherName, otherC := range other {
err := c.BitwiseCompare(otherC)
if err != nil {
if otherName == name {
t.Fatalf("container types %d/%d, contents %s: unexpected error %v",
t1, t2, name, err)
}
} else {
if name != otherName {
t.Fatalf("container types %d/%d, unexpected %s == %s",
t1, t2, name, otherName)
}
}
}
}
}
}
}
func TestContainerCombinations(t *testing.T) {
@ -3560,51 +3574,8 @@ func TestContainerCombinations(t *testing.T) {
// Convert to all container types and check result.
for _, ct := range containerTypes {
clone := ret.Clone()
if ct == containerArray {
if clone == nil {
clone = NewContainerArray(nil)
} else if clone.isBitmap() {
clone = clone.bitmapToArray()
} else if clone.isRun() {
clone = clone.runToArray()
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
// Because xorRunRun resulting in an empty container returns an array container with a
// nil slice array, then we need to check len() on array first (look for 0).
if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) {
t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array())
}
} else if ct == containerBitmap {
if clone == nil {
clone = NewContainerBitmap(0, nil)
} else if clone.isArray() {
clone = clone.arrayToBitmap()
} else if clone.isRun() {
clone = clone.runToBitmap()
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) {
t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap())
}
} else if ct == containerRun {
if clone == nil {
clone = NewContainerRun(nil)
} else if clone.isArray() {
clone = clone.arrayToRun(0)
} else if clone.isBitmap() {
clone = clone.bitmapToRun(0)
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) {
t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs())
}
if err := ret.BitwiseCompare(cts[ct][exp]); err != nil {
t.Errorf("test %s: %v", desc, err)
}
}
}
@ -4026,7 +3997,11 @@ func TestDirectAddNVsAdd(t *testing.T) {
{9384932, 101000, 2, 1, 0},
{3489, 19230, 394, 0, 893982, 890283, 14, 7},
}
// TODO generate more tests and fuzz
// Add some randomly created tests.
rand := rand.New(rand.NewSource(1))
for i := 0; i < 100; i++ {
tests = append(tests, generator.Uint64Slice(1+rand.Intn(1000), 0, 10000000, i%2 == 0, rand))
}
testsCopy := make([][]uint64, len(tests))
copy(testsCopy, tests)
for i, test := range testsCopy {

148
roaring/roaring_test.go
View file

@ -20,12 +20,12 @@ import (
"math"
"math/rand"
"reflect"
"sort"
"testing"
"testing/quick"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/generator"
"github.com/pilosa/pilosa/v2/roaring"
_ "github.com/pilosa/pilosa/v2/test"
)
@ -280,11 +280,36 @@ func TestBitmap_Slice_Empty(t *testing.T) {
}
// Ensure a bitmap can return a slice of values within a range.
// TODO duplicate for all container types
func TestBitmap_SliceRange(t *testing.T) {
if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) {
t.Fatalf("unexpected slice: %+v", a)
}
t.Run("array", func(t *testing.T) {
if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) {
t.Fatalf("unexpected slice: %+v", a)
}
})
t.Run("bitmap", func(t *testing.T) {
bm := roaring.NewFileBitmap()
for i := uint64(10); i < 10000; i++ {
_, _ = bm.Add(i * 2)
}
bm.Optimize()
if a := bm.SliceRange(20, 30); !reflect.DeepEqual(a, []uint64{20, 22, 24, 26, 28}) {
t.Fatalf("unexpected slice: %+v", a)
}
})
t.Run("run", func(t *testing.T) {
bm := roaring.NewFileBitmap()
for i := uint64(0); i < 11; i++ {
_, _ = bm.Add(i)
}
bm.Optimize()
if a := bm.SliceRange(6, 10); !reflect.DeepEqual(a, []uint64{6, 7, 8, 9}) {
t.Fatalf("unexpected slice: %+v", a)
}
})
}
// Ensure a bitmap can loop over a set of values.
@ -841,7 +866,7 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) {
seed = time.Now().UnixNano()
source = rand.NewSource(seed)
rng = rand.New(source)
numTests = 100
numTests = 20
maxNumIntsPerBatch = 100
maxNumBatches = 100
maxRangePercent = 2
@ -1400,10 +1425,10 @@ func TestBitmap_Shift(t *testing.T) {
}
func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) }
func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) }
func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) }
func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 10000, 10000, 20000) }
func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxInt64) }
func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 1000, 0, 1000) }
func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 1000, 0, 10000) }
func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 1000, 10000, 20000) }
func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 1000, 0, math.MaxInt64) }
// Ensure a bitmap can perform basic operations on randomly generated values.
func testBitmapQuick(t *testing.T, n int, min, max uint64) {
@ -1442,7 +1467,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
// If `got` is nil and `exp` has zero length, don't perform the DeepEqual
// because when `a` is empty (`a = []uint64{}`) then `got` is a nil slice
// while `exp` is an empty slice. Therefore they will not be considered equal.
if got, exp := bm.Slice(), uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) {
if got, exp := bm.Slice(), generator.Uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) {
t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp)
}
@ -1467,7 +1492,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, false, rand))
values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, false, rand))
},
})
if err != nil {
@ -1478,22 +1503,30 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
func TestBitmap_Marshal_Quick_Array1(t *testing.T) {
testBitmapMarshalQuick(t, 1000, 1000, 2000, false)
}
func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000, false) }
func TestBitmap_Marshal_Quick_Array2(t *testing.T) {
testBitmapMarshalQuick(t, 1000, 0, 1000, false)
}
func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 0, 10000, false)
testBitmapMarshalQuick(t, 1000, 0, 10000, false)
}
func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 10000, 20000, false)
testBitmapMarshalQuick(t, 1000, 10000, 20000, false)
}
func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) {
testBitmapMarshalQuick(t, 100, 0, math.MaxInt64, false)
}
func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 0, 10000, true)
testBitmapMarshalQuick(t, 1000, 0, 10000, true)
}
// TODO update for RLE
// (travis) - it's not clear to me how to generate a run container
// using `testBitmapMarshalQuick`. Because it's randomly generated,
// even some of the "Bitmap" tests generate array containers. Also,
// I think in order for the container to be a run, we would need
// to call bm.Optimize() on the bitmap, and I'm hesitant to add that
// because it's not clear to me how that would affect the tests.
// Ensure a bitmap can be marshaled and unmarshaled.
func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
@ -1537,22 +1570,20 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
t.Fatal(err)
}
// Verify the original bitmap has the correct set of values.
if exp, got := uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) {
t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got)
if _, err := roaring.CompareBitmapMap(bm, set); err != nil {
t.Fatalf("source mismatch: %v", err)
}
// Verify the bitmap loaded with the ops log has the correct set of values.
if exp, got := uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) {
t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got)
if _, err := roaring.CompareBitmapMap(bm2, set); err != nil {
t.Fatalf("unmarshalled mismatch: %v", err)
}
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, sorted, rand))
values[1] = reflect.ValueOf(GenerateUint64Slice(100, min, max, sorted, rand))
values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, sorted, rand))
values[1] = reflect.ValueOf(generator.Uint64Slice(100, min, max, sorted, rand))
},
})
if err != nil {
@ -1561,9 +1592,8 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
}
// Ensure iterator can iterate over all the values on the bitmap.
// TODO duplicate for all container types
func TestIterator(t *testing.T) {
t.Run("bitmap", func(t *testing.T) {
t.Run("array", func(t *testing.T) {
itr := roaring.NewFileBitmap(1, 2, 3).Iterator()
itr.Seek(0)
@ -1577,6 +1607,29 @@ func TestIterator(t *testing.T) {
}
})
t.Run("bitmap", func(t *testing.T) {
bm := roaring.NewFileBitmap()
exp := []uint64{}
for i := uint64(0); i < 10000; i++ {
v := i * 2
_, _ = bm.Add(v)
exp = append(exp, v)
}
bm.Optimize()
itr := bm.Iterator()
itr.Seek(0)
var a []uint64
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
a = append(a, v)
}
if !reflect.DeepEqual(a, exp) {
t.Fatalf("unexpected values: %+v", a)
}
})
t.Run("run", func(t *testing.T) {
bm1 := roaring.NewFileBitmap()
for i := uint64(0); i < 11; i++ {
@ -1784,49 +1837,6 @@ func getBenchData(tb testing.TB) *benchmarkSampleData {
return data
}
// GenerateUint64Slice generates between [0, n) random uint64 numbers between min and max.
func GenerateUint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 {
a := make([]uint64, rand.Intn(n))
for i := range a {
a[i] = min + uint64(rand.Int63n(int64(max-min)))
}
if sorted {
sort.Sort(uint64Slice(a))
}
return a
}
// uint64SetSlice returns the values in a uint64 set.
func uint64SetSlice(m map[uint64]struct{}) []uint64 {
a := make([]uint64, 0, len(m))
for v := range m {
a = append(a, v)
}
sort.Sort(uint64Slice(a))
return a
}
// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func diff(a, b []uint64) string {
if len(a) != len(b) {
return fmt.Sprintf("len: %d != %d", len(a), len(b))
}
for i := range a {
if a[i] != b[i] {
return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i])
}
}
return ""
}
func TestBitmap_Intersect(t *testing.T) {
bm0 := testBM()
result := bm0.Intersect(bm0)

27
roaring/unmarshal_binary.go
View file

@ -159,8 +159,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
// Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)).
keyN := binary.LittleEndian.Uint32(data[3+1 : 8])
if uint32(len(data)) < headerBaseSize+keyN*12 {
return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", int(keyN)/12)
if int64(len(data)) < headerBaseSize+int64(keyN)*12 {
return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN)
}
headerSize := headerBaseSize
@ -173,14 +173,23 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
int(binary.LittleEndian.Uint16(buf[10:12]))+1,
true)
}
opsOffset := headerSize + int(keyN)*12
opsOffset := int64(headerSize) + int64(keyN)*12
// Read container offsets and attach data.
citer, _ := b.Containers.Iterator(0)
// if you have enough containers that the *headers alone* exceed 4GB, we
// need to start with a higher cycle offset.
cycleOffset := opsOffset &^ ((1 << 32) - 1)
prevOffset32 := uint32(opsOffset)
for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
offset := binary.LittleEndian.Uint32(buf[0:4])
offset32 := binary.LittleEndian.Uint32(buf[0:4])
if offset32 < prevOffset32 {
cycleOffset += (1 << 32)
}
prevOffset32 = offset32
offset := int64(offset32) + cycleOffset
// Verify the offset is within the bounds of the input data.
if int(offset) >= len(data) {
if offset >= int64(len(data)) {
return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
}
@ -201,13 +210,13 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
case containerRun:
runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount])
opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size
opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size
case containerArray:
c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32)
opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32)
case containerBitmap:
c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN])
opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64)
opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64)
}
}
@ -228,7 +237,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
// Increase the op count.
b.ops++
b.opN += opr.count()
opsOffset += opr.size()
opsOffset += int64(opr.size())
// Move the buffer forward.
buf = data[opsOffset:]
}

58
row.go
View file

@ -374,7 +374,7 @@ func (r *Row) Difference(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericUnary returns the results of a generic op on r.
// GenericUnaryOp returns the results of a generic op on r.
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
work := r
var segments []rowSegment
@ -510,14 +510,10 @@ func (r *Row) Columns() []uint64 {
// Includes returns true if the row contains the given column.
func (r *Row) Includes(col uint64) bool {
// TODO: improve the efficiency of this method by
// performing the column filter at the bitmap level
// rather than iterating through the results here.
shard := col / ShardWidth
for i := range r.segments {
for _, c := range r.segments[i].Columns() {
if c == col {
return true
}
if r.segments[i].shard == shard {
return r.segments[i].data.Contains(col)
}
}
return false
@ -574,13 +570,11 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 {
// Intersect returns the itersection of s and other.
func (s *rowSegment) Intersect(other *rowSegment) *rowSegment {
data := s.data.Intersect(other.data)
data = data.Freeze()
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
writable: true,
data: data,
shard: s.shard,
n: data.Count(),
}
}
@ -591,13 +585,11 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
datas[i] = other.data
}
data := s.data.Union(datas...)
data.Freeze()
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
writable: true,
data: data,
shard: s.shard,
n: data.Count(),
}
}
@ -635,49 +627,43 @@ func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
datas[i] = other.data
}
data := s.data.Difference(datas...)
data.Freeze()
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
writable: true,
data: data,
shard: s.shard,
n: data.Count(),
}
}
// Xor returns the xor of s and other.
func (s *rowSegment) Xor(other *rowSegment) *rowSegment {
data := s.data.Xor(other.data)
data = data.Freeze()
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
writable: true,
data: data,
shard: s.shard,
n: data.Count(),
}
}
// Shift returns s shifted by 1 bit.
func (s *rowSegment) Shift() (*rowSegment, error) {
//TODO deal with overflow
// TODO: deal with overflow
// See issue: https://github.com/molecula/pilosa/issues/403
data, err := s.data.Shift(1)
if err != nil {
return nil, errors.Wrap(err, "shifting roaring data")
}
data = data.Freeze()
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
writable: true,
data: data,
shard: s.shard,
n: data.Count(),
}, nil
}
// GenericUnary returns s subject to op.
// GenericUnaryOp returns s subject to op.
func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
//TODO deal with overflow
data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
return &rowSegment{

37
server.go
View file

@ -77,6 +77,8 @@ type Server struct { // nolint: maligned
metricInterval time.Duration
diagnosticInterval time.Duration
maxWritesPerRequest int
confirmDownSleep time.Duration
confirmDownRetries int
isCoordinator bool
syncer holderSyncer
@ -229,6 +231,17 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
}
}
// OptServerNodeDownRetries is a functional option on Server
// used to specify the retries and sleep duration for node down
// checks.
func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption {
return func(s *Server) error {
s.confirmDownRetries = retries
s.confirmDownSleep = sleep
return nil
}
}
// OptServerURI is a functional option on Server
// used to set the server URI.
func OptServerURI(uri *URI) ServerOption {
@ -330,6 +343,9 @@ func NewServer(opts ...ServerOption) (*Server, error) {
metricInterval: 0,
diagnosticInterval: 0,
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
resetTranslationSyncCh: make(chan struct{}),
logger: logger.NopLogger,
@ -356,14 +372,11 @@ func NewServer(opts ...ServerOption) (*Server, error) {
}
s.executor = newExecutor(executorOpts...)
// s.holder.translateFile.logger = s.logger
path, err := expandDirName(s.dataDir)
if err != nil {
return nil, err
}
s.holder.Path = path
// s.holder.translateFile.Path = filepath.Join(path, ".keys")
s.holder.Logger = s.logger
s.holder.Stats.SetLogger(s.logger)
@ -402,6 +415,8 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.cluster.confirmDownRetries = s.confirmDownRetries
s.cluster.confirmDownSleep = s.confirmDownSleep
s.holder.broadcaster = s
err = s.loadAllExtensions()
if err != nil {
@ -716,10 +731,13 @@ func (s *Server) receiveMessage(m Message) error {
}
case *CreateIndexMessage:
opt := obj.Meta
_, err := s.holder.CreateIndex(obj.Index, *opt)
idx, err := s.holder.CreateIndex(obj.Index, *opt)
if err != nil {
return err
}
idx.mu.Lock()
idx.createdAt = obj.CreatedAt
idx.mu.Unlock()
case *DeleteIndexMessage:
if err := s.holder.DeleteIndex(obj.Index); err != nil {
return err
@ -730,10 +748,13 @@ func (s *Server) receiveMessage(m Message) error {
return fmt.Errorf("local index not found: %s", obj.Index)
}
opt := obj.Meta
_, err := idx.createFieldIfNotExists(obj.Field, opt)
fld, err := idx.createFieldIfNotExists(obj.Field, opt)
if err != nil {
return err
}
fld.mu.Lock()
fld.createdAt = obj.CreatedAt
fld.mu.Unlock()
case *DeleteFieldMessage:
idx := s.holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
@ -767,6 +788,12 @@ func (s *Server) receiveMessage(m Message) error {
if err != nil {
return err
}
if !s.isCoordinator {
if obj.Schema != nil {
s.holder.applyCreatedAt(obj.Schema.Indexes)
}
}
case *ResizeInstruction:
err := s.cluster.followResizeInstruction(obj)
if err != nil {

197
server/handler_test.go
View file

@ -195,19 +195,81 @@ func TestHandler_Endpoints(t *testing.T) {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
body := w.Body.String()
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}
`, pilosa.ShardWidth)
body := strings.TrimSpace(w.Body.String())
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth)
if body != target {
t.Fatalf("%s != %s", target, body)
}
})
t.Run("Import", func(t *testing.T) {
indexInfo := cmd.API.Schema(context.Background())
err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false)
if err != nil {
t.Fatalf("applying schema: %v", err)
}
idx := indexInfo[0]
fld := indexInfo[0].Fields[0]
msg := pilosa.ImportRequest{
Index: idx.Name,
IndexCreatedAt: idx.CreatedAt,
Field: fld.Name,
FieldCreatedAt: fld.CreatedAt,
Shard: 0,
}
ser := proto.Serializer{}
data, err := ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name)
httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, httpReq)
if w.Code != 200 {
t.Fatalf(w.Body.String())
}
msg.IndexCreatedAt = -idx.CreatedAt
msg.FieldCreatedAt = -fld.CreatedAt
data, err = ser.Marshal(&msg)
if err != nil {
t.Fatal(err)
}
httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
w = httptest.NewRecorder()
h.ServeHTTP(w, httpReq)
if w.Code != 412 {
t.Fatal("expected: Precondition Failed, got:" + w.Body.String())
}
})
t.Run("ImportRoaring", func(t *testing.T) {
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
idx, err := cmd.API.Index(context.Background(), "i0")
if err != nil {
t.Fatal(err)
}
fld, err := cmd.API.Field(context.Background(), "i0", "f1")
if err != nil {
t.Fatal(err)
}
msg := pilosa.ImportRoaringRequest{
Clear: false,
IndexCreatedAt: idx.CreatedAt(),
FieldCreatedAt: fld.CreatedAt(),
Clear: false,
Views: map[string][]byte{
"": roaringData,
},
@ -217,10 +279,14 @@ func TestHandler_Endpoints(t *testing.T) {
if err != nil {
t.Fatal(err)
}
httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data))
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, httpReq)
if w.Code != 200 {
t.Fatalf("Unexpected response body: %s", w.Body.String())
}
resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"})
if err != nil {
t.Fatalf("querying: %v", err)
@ -761,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected response body: %s", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Verify index is gone.
if hldr.Index("i") != nil {
@ -779,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
})
@ -960,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create index again
@ -970,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusConflict {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Errorf("unexpected body: %q", w.Body.String())
}
}
// create field
@ -980,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create field again
@ -990,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusConflict {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
Name string `json:"name,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success || resp.Name == "" || resp.CreatedAt == 0 {
t.Errorf("unexpected body: %q", w.Body.String())
}
}
// delete field
@ -1000,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// delete field again
@ -1020,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// delete index again
@ -1030,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusNotFound {
t.Errorf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" {
t.Errorf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
})
@ -1042,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// create field
@ -1052,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"success":true}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
} else {
var resp struct {
Success bool `json:"success"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if !resp.Success {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// set some bits

9
server/server.go
View file

@ -274,6 +274,15 @@ func (m *Command) SetupServer() error {
grpcURI.SetPort(uint16(m.grpcLn.Addr().(*net.TCPAddr).Port))
}
if grpcURI.Scheme == "http" {
grpcURI.Scheme = "grpc"
}
// discover the address if not specified
if grpcURI.Host == "0.0.0.0" {
grpcURI.Host = outboundIP().String()
}
// Setup TLS
if uri.Scheme == "https" {
m.tlsConfig, err = GetTLSConfig(&m.Config.TLS, m.logger.Logger())

44
server/server_test.go
View file

@ -1092,6 +1092,50 @@ func TestClusterExhaustingConnections(t *testing.T) {
}
}
func TestQueryingWithQuotesAndStuff(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
t.Fatal(err)
}
// Execute Set() commands.
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}); err != nil {
t.Fatal(err)
}
if err := client.CreateFieldWithOptions(context.Background(), "i", "fld", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
}
// Test escaped single quote gets set properly
if res, err := m.Query(t, "i", "", `Set('bl\'ah', fld=ha)`); err != nil {
t.Fatal(err)
} else if !strings.Contains(res, "[true]") {
t.Errorf("setting escaped single quote result: %s", res)
}
if res, err := m.Query(t, "i", "", `Row(fld=ha)`); err != nil {
t.Fatal(err)
} else if !strings.Contains(res, `bl'ah`) {
t.Errorf("value with escaped single quote set improperly: %s", res)
}
// Test escaped double quote gets set properly
if res, err := m.Query(t, "i", "", `Set("d\"ah", fld=dq)`); err != nil {
t.Fatal(err)
} else if !strings.Contains(res, "[true]") {
t.Errorf("value with escaped double quote set improperly: %s", res)
}
if res, err := m.Query(t, "i", "", `Row(fld=dq)`); err != nil {
t.Fatal(err)
} else if !strings.Contains(res, `d\"ah`) {
// the backslash is there because JSON needs to escape the
// double quote since it uses double quotes
t.Errorf("value with escaped double quote set improperly: %s", res)
}
}
func TestClusterExhaustingConnectionsImport(t *testing.T) {
if !runStress {
t.Skip("stress")

14
test/pilosa.go
View file

@ -92,7 +92,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
opts = prependWithMemStore(opts)
opts = prependTestServerOpts(opts)
m := newCommand(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
@ -434,25 +434,25 @@ func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
return c
}
// prependOpts applies prependWithMemStore to each of the ops (one per
// prependOpts applies prependTestServerOpts to each of the ops (one per
// node, or one for the entire cluser).
func prependOpts(opts [][]server.CommandOption) [][]server.CommandOption {
if len(opts) == 0 {
opts = [][]server.CommandOption{
prependWithMemStore([]server.CommandOption{}),
prependTestServerOpts([]server.CommandOption{}),
}
} else {
for i := range opts {
opts[i] = prependWithMemStore(opts[i])
opts[i] = prependTestServerOpts(opts[i])
}
}
return opts
}
// prependWithMemStore prepends opts with the OpenInMemTranslateStore.
func prependWithMemStore(opts []server.CommandOption) []server.CommandOption {
// prependTestServerOpts prepends opts with the OpenInMemTranslateStore.
func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
defaultOpts := []server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)),
server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore), pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond)),
}
return append(defaultOpts, opts...)
}

125
tracker.go Normal file
View file

@ -0,0 +1,125 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"sort"
"sync"
"time"
)
type ActiveQueryStatus struct {
Query string `json:"query"`
Age time.Duration `json:"age"`
}
type activeQuery struct {
query string
started time.Time
}
type queryStatusUpdate struct {
q *activeQuery
end bool
}
type queryTracker struct {
updates chan<- queryStatusUpdate
checks chan<- chan<- []*activeQuery
wg sync.WaitGroup
stop chan struct{}
}
func newQueryTracker() *queryTracker {
done := make(chan struct{})
updates := make(chan queryStatusUpdate, 128)
checks := make(chan chan<- []*activeQuery)
tracker := &queryTracker{
updates: updates,
checks: checks,
stop: done,
}
tracker.wg.Add(1)
go func() {
defer tracker.wg.Done()
activeQueries := make(map[*activeQuery]struct{})
for {
select {
case update := <-updates:
if update.end {
delete(activeQueries, update.q)
} else {
activeQueries[update.q] = struct{}{}
}
case check := <-checks:
out := make([]*activeQuery, len(activeQueries))
i := 0
for q := range activeQueries {
out[i] = q
i++
}
check <- out
close(check)
case <-done:
return
}
}
}()
return tracker
}
func (t *queryTracker) Start(query string) *activeQuery {
now := time.Now()
q := &activeQuery{query, now}
t.updates <- queryStatusUpdate{q, false}
return q
}
func (t *queryTracker) Finish(q *activeQuery) {
t.updates <- queryStatusUpdate{q, true}
}
func (t *queryTracker) ActiveQueries() []ActiveQueryStatus {
ch := make(chan []*activeQuery, 1)
t.checks <- ch
queries := <-ch
sort.Slice(queries, func(i, j int) bool {
switch {
case queries[i].started.Before(queries[j].started):
return true
case queries[i].started.After(queries[j].started):
return false
case queries[i].query < queries[j].query:
return true
case queries[i].query > queries[j].query:
return false
default:
return false
}
})
now := time.Now()
out := make([]ActiveQueryStatus, len(queries))
for i, v := range queries {
out[i] = ActiveQueryStatus{v.query, now.Sub(v.started)}
}
return out
}
func (t *queryTracker) Stop() {
close(t.stop)
t.wg.Wait()
}

42
tracker_test.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import "testing"
func TestQueryTracker(t *testing.T) {
tracker := newQueryTracker()
defer tracker.Stop()
if queries := tracker.ActiveQueries(); len(queries) > 0 {
t.Fatalf("expected no active queries; found %v", queries)
}
qs := tracker.Start("test query")
var queries []ActiveQueryStatus
for len(queries) < 1 {
queries = tracker.ActiveQueries()
}
if len(queries) > 1 || queries[0].Query != "test query" {
t.Fatalf("unexpected queries: %v", queries)
}
tracker.Finish(qs)
for len(queries) > 0 {
queries = tracker.ActiveQueries()
}
}

2
transaction.md
View file

@ -30,7 +30,7 @@ The base transaction endpoints are `/transactions`, for listing or creating
transactions, and `/transaction/[id]`, for listing, creating, finishing, or
cancelling a transaction.
A POST to `/transactions` attempts to create a transaction, assigning it an
A POST to `/transaction` attempts to create a transaction, assigning it an
arbitrary ID that is not the ID of any existing transaction. A `GET` from
`/transactions` lists existing transactions.