diff --git a/.circleci/config.yml b/.circleci/config.yml index 9904cff3e..f2e70731e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,8 +6,12 @@ executors: version: type: string default: "1.14" + resource_class: + type: string + default: medium docker: - image: circleci/golang:<< parameters.version >> + resource_class: << parameters.resource_class >> working_directory: /go/src/github.com/pilosa/pilosa environment: GO111MODULE: "on" # TODO: Only needed for Go <1.13, remove when dropping support for 1.11/1.12. @@ -64,6 +68,9 @@ jobs: - run: make build GOOS=linux GOARCH=arm64 test: parameters: + resource_class: + type: string + default: medium golang_version: type: string default: "1.14" @@ -82,6 +89,7 @@ jobs: executor: name: golang version: << parameters.golang_version >> + resource_class: << parameters.resource_class >> steps: - attach_workspace: at: . @@ -191,6 +199,7 @@ workflows: - test: name: test-race test_make_target: test-race + resource_class: xlarge requires: - setup - test: diff --git a/api.go b/api.go index 77a3fc34e..5f5b3807f 100644 --- a/api.go +++ b/api.go @@ -45,6 +45,7 @@ type API struct { holder *Holder cluster *cluster server *Server + tracker *queryTracker importWorkersWG sync.WaitGroup importWorkerPoolSize int @@ -95,6 +96,8 @@ func NewAPI(opts ...apiOption) (*API, error) { }() } + api.tracker = newQueryTracker() + return api, nil } @@ -130,6 +133,7 @@ func (api *API) validate(f apiMethod) error { func (api *API) Close() error { close(api.importWork) api.importWorkersWG.Wait() + api.tracker.Stop() return nil } @@ -146,6 +150,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return QueryResponse{}, errors.Wrap(err, "parsing") } + defer api.tracker.Finish(api.tracker.Start(req.Query)) execOpts := &execOptions{ Remote: req.Remote, Profile: req.Profile, @@ -181,11 +186,18 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index if err != nil { return nil, errors.Wrap(err, "creating index") } + + createdAt := timestamp() + index.mu.Lock() + index.createdAt = createdAt + index.mu.Unlock() + // Send the create index message to all nodes. err = api.server.SendSync( &CreateIndexMessage{ - Index: indexName, - Meta: &options, + Index: indexName, + CreatedAt: createdAt, + Meta: &options, }) if err != nil { return nil, errors.Wrap(err, "sending CreateIndex message") @@ -269,14 +281,18 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str if err != nil { return nil, errors.Wrap(err, "creating field") } + createdAt := timestamp() + field.mu.Lock() + field.createdAt = createdAt + field.mu.Unlock() // Send the create field message to all nodes. - err = api.server.SendSync( - &CreateFieldMessage{ - Index: indexName, - Field: fieldName, - Meta: &fo, - }) + err = api.server.SendSync(&CreateFieldMessage{ + Index: indexName, + Field: fieldName, + CreatedAt: createdAt, + Meta: &fo, + }) if err != nil { api.server.logger.Printf("problem sending CreateField message: %s", err) return nil, errors.Wrap(err, "sending CreateField message") @@ -413,14 +429,17 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "validating api method") } - nodes := api.cluster.shardNodes(indexName, shard) - - field := api.holder.Field(indexName, fieldName) - if field == nil { + index, field, err := api.indexField(indexName, fieldName, shard) + if index == nil || field == nil { return newNotFoundError(ErrFieldNotFound) } - errCh := make(chan error, len(nodes)) + if err = req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return newPreconditionFailedError(err) + } + + nodes := api.cluster.shardNodes(indexName, shard) + errCh := make(chan error, len(nodes)) for _, node := range nodes { node := node if node.ID == api.server.nodeID { @@ -803,6 +822,18 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return errors.Wrap(err, "validating api method") } + // set CreatedAt for indexes and fields (if empty), and then apply schema. + for _, index := range s.Indexes { + if index.CreatedAt == 0 { + index.CreatedAt = timestamp() + } + for _, field := range index.Fields { + if field.CreatedAt == 0 { + field.CreatedAt = timestamp() + } + } + } + if !remote { nodes := api.cluster.Nodes() for i, node := range nodes { @@ -813,7 +844,7 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { } } - return api.holder.applySchema(s) + return errors.Wrap(api.holder.applySchema(s), "applying schema") } // Views returns the views in the given field. @@ -999,16 +1030,23 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp return errors.Wrap(err, "validating api method") } + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { + return errors.Wrap(err, "validating import value request") + } + // Set up import options. options, err := setUpImportOptions(opts...) if err != nil { return errors.Wrap(err, "setting up import options") } - - index, field, err := api.indexField(req.Index, req.Field, req.Shard) - if err != nil { - return errors.Wrap(err, "getting index and field") - } + span.LogKV( + "index", req.Index, + "field", req.Field) // Unless explicitly ignoring key validation (meaning keys have been // translated to ids in a previous step at the coordinator node), then @@ -1016,6 +1054,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp if !options.IgnoreKeyCheck { // Translate row keys. if field.Keys() { + span.LogKV("rowKeys", true) if len(req.RowIDs) != 0 { return errors.New("row ids cannot be used because field uses string keys") } @@ -1026,6 +1065,7 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp // Translate column keys. if index.Keys() { + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1110,7 +1150,12 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "validating api method") } - if err := req.Validate(); err != nil { + index, field, err := api.indexField(req.Index, req.Field, req.Shard) + if err != nil { + return errors.Wrap(err, "getting index and field") + } + + if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil { return errors.Wrap(err, "validating import value request") } @@ -1120,17 +1165,20 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . return errors.Wrap(err, "setting up import options") } - index, field, err := api.indexField(req.Index, req.Field, req.Shard) + index, field, err = api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting index and field") } - + span.LogKV( + "index", req.Index, + "field", req.Field) // Unless explicitly ignoring key validation (meaning keys have been // translate to ids in a previous step at the coordinator node), then // check to see if keys need translation. if !options.IgnoreKeyCheck { // Translate column keys. if index.Keys() { + span.LogKV("columnKeys", true) if len(req.ColumnIDs) != 0 { return errors.New("column ids cannot be used because index uses string keys") } @@ -1144,6 +1192,7 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts . // the field has a ForeignIndex with keys). if field.Keys() { // Perform translation. + span.LogKV("rowKeys", true) uints, err := api.cluster.translateIndexKeys(ctx, field.ForeignIndex(), req.StringValues) if err != nil { return err @@ -1257,6 +1306,12 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq return errors.Wrap(err, "validating shard ownership") } + if req.IndexCreatedAt != 0 { + if index.CreatedAt() != req.IndexCreatedAt { + return ErrPreconditionFailed + } + } + bulkAttrs := make(map[uint64]map[string]interface{}) for n := 0; n < len(req.ColumnIDs); n++ { bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} @@ -1594,14 +1649,41 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du if err := api.validate(apiStartTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.StartTransaction(ctx, id, timeout, exclusive, remote) + t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote) + if exclusive { + switch err { + case nil: + api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0) + case ErrTransactionExclusive: + api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0) + } + if t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + } else { + switch err { + case nil: + api.holder.Stats.Count(MetricTransactionStart, 1, 1.0) + case ErrTransactionExclusive: + api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0) + } + } + return t, err } func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error) { if err := api.validate(apiFinishTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.FinishTransaction(ctx, id, remote) + t, err := api.server.FinishTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive { + api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0) + } else { + api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0) + } + } + return t, err } func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error) { @@ -1615,7 +1697,20 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr if err := api.validate(apiGetTransaction); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.server.GetTransaction(ctx, id, remote) + t, err := api.server.GetTransaction(ctx, id, remote) + if err == nil { + if t.Exclusive && t.Active { + api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + } + } + return t, err +} + +func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) { + if err := api.validate(apiActiveQueries); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + return api.tracker.ActiveQueries(), nil } type serverInfo struct { @@ -1669,6 +1764,7 @@ const ( apiFinishTransaction apiTransactions apiGetTransaction + apiActiveQueries ) var methodsCommon = map[apiMethod]struct{}{ @@ -1708,4 +1804,5 @@ var methodsNormal = map[apiMethod]struct{}{ apiFinishTransaction: {}, apiTransactions: {}, apiGetTransaction: {}, + apiActiveQueries: {}, } diff --git a/api_test.go b/api_test.go index b7f0fc06f..02e8a182f 100644 --- a/api_test.go +++ b/api_test.go @@ -63,15 +63,15 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { m1 := c[1] t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() - index := "i" - field := "f" + indexName := "i" + fieldName := "f" attrKey := "k" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field) + _, err = m0.API.CreateField(ctx, indexName, fieldName) if err != nil { t.Fatalf("creating field: %v", err) } @@ -86,27 +86,28 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { columnIDs0 = append(columnIDs0, uint64(n)) val0 := attrFun(uint64(n)) attrVals0 = append(attrVals0, val0) - setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, field) - if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql0}); err != nil { + setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName) + if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil { t.Fatal(err) } columnIDs1 = append(columnIDs1, uint64(n+ShardWidth)) val1 := attrFun(uint64(n + ShardWidth)) attrVals1 = append(attrVals1, val1) - setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, field) - if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: setPql1}); err != nil { + setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName) + if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil { t.Fatal(err) } } // send shard0 to node1 req := &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs0, - AttrVals: attrVals0, - Shard: 0, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs0, + AttrVals: attrVals0, + Shard: 0, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { @@ -115,11 +116,12 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { // send shard1 to node0 req = &pilosa.ImportColumnAttrsRequest{ - AttrKey: attrKey, - ColumnIDs: columnIDs1, - AttrVals: attrVals1, - Shard: 1, - Index: index, + AttrKey: attrKey, + ColumnIDs: columnIDs1, + AttrVals: attrVals1, + Shard: 1, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), } if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { @@ -127,8 +129,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } // Query node0. - pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -143,8 +145,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } } // Query node1. - pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", field) - res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName) + res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}) if err != nil { t.Fatal(err) } @@ -186,17 +188,24 @@ func TestAPI_Import(t *testing.T) { t.Run("RowIDColumnKey", func(t *testing.T) { ctx := context.Background() - index := "rick" - field := "f" + indexName := "rick" + fieldName := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m0.API.CreateField(ctx, index, field, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if index.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field, err := m0.API.CreateField(ctx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } rowID := uint64(1) timestamp := int64(0) @@ -215,21 +224,23 @@ func TestAPI_Import(t *testing.T) { // Import data with keys to the coordinator (node0) and verify that it gets // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) req := &pilosa.ImportRequest{ - Index: index, - Field: field, - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, + Index: indexName, + IndexCreatedAt: index.CreatedAt(), + Field: fieldName, + FieldCreatedAt: field.CreatedAt(), + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, } if err := m0.API.Import(ctx, req); err != nil { t.Fatal(err) } - pql := fmt.Sprintf("Row(%s=%d)", field, rowID) + pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) // Query node0. - if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { t.Fatal(err) } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %#v", keys) @@ -237,7 +248,7 @@ func TestAPI_Import(t *testing.T) { // Query node1. if err := test.RetryUntil(5*time.Second, func() error { - if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { return err } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { return fmt.Errorf("unexpected column keys: %#v", keys) diff --git a/cluster.go b/cluster.go index 6fce679fb..eacad25f7 100644 --- a/cluster.go +++ b/cluster.go @@ -62,9 +62,8 @@ const ( resizeJobActionAdd = "ADD" resizeJobActionRemove = "REMOVE" - confirmDownRetries = 10 - confirmDownSleep = 1 - confirmDownTimeout = 2 + defaultConfirmDownRetries = 10 + defaultConfirmDownSleep = 1 * time.Second ) // Node represents a node in the cluster. @@ -239,6 +238,9 @@ type cluster struct { // nolint: maligned logger logger.Logger InternalClient InternalClient + + confirmDownRetries int + confirmDownSleep time.Duration } // newCluster returns a new instance of Cluster with defaults. @@ -258,6 +260,9 @@ func newCluster() *cluster { InternalClient: newNopInternalClient(), logger: logger.NopLogger, + + confirmDownRetries: defaultConfirmDownRetries, + confirmDownSleep: defaultConfirmDownSleep, } } @@ -610,6 +615,7 @@ func (c *cluster) unprotectedStatus() *ClusterStatus { ClusterID: c.id, State: c.state, Nodes: c.nodes, + Schema: &Schema{Indexes: c.holder.Schema()}, } } @@ -659,6 +665,7 @@ func (c *cluster) addNodeBasicSorted(node *Node) bool { n.State = node.State n.IsCoordinator = node.IsCoordinator n.URI = node.URI + n.GRPCURI = node.GRPCURI return true } return false @@ -1042,6 +1049,11 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { func (c *cluster) ownsPartition(nodeID string, partition int) bool { c.mu.RLock() defer c.mu.RUnlock() + return c.unprotectedOwnsPartition(nodeID, partition) +} + +// unprotectedOwnsPartition returns true if a host owns a partition. +func (c *cluster) unprotectedOwnsPartition(nodeID string, partition int) bool { return Nodes(c.partitionNodes(partition)).ContainsID(nodeID) } @@ -1271,9 +1283,7 @@ func (c *cluster) listenForJoins() { // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. // We use a bool `setNormal` to indicate when at least one node has joined. var setNormal bool - for { - // Handle all pending joins before changing state back to NORMAL. select { case nodeAction := <-c.joiningLeavingNodes: @@ -1745,8 +1755,9 @@ func (j *resizeJob) distributeResizeInstructions() error { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. node := &Node{ - ID: instr.Node.ID, - URI: instr.Node.URI, + ID: instr.Node.ID, + URI: instr.Node.URI, + GRPCURI: instr.Node.GRPCURI, } j.Logger.Printf("send resize instructions: %v", instr) if err := j.Broadcaster.SendTo(node, instr); err != nil { @@ -1917,7 +1928,7 @@ func (c *cluster) considerTopology() error { // band aid to protect against false nodeLeave events from memberlist // the test is the lightest weight endpoint of the node in question /version // TODO provide more robust solution to false nodeLeave events -func confirmNodeDown(uri URI, log logger.Logger) bool { +func (c *cluster) confirmNodeDown(uri URI) bool { u := url.URL{ Scheme: uri.Scheme, Host: uri.HostPort(), @@ -1925,11 +1936,11 @@ func confirmNodeDown(uri URI, log logger.Logger) bool { } req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - log.Printf("bad request:%s %s", u.String(), err) + c.logger.Printf("bad request:%s %s", u.String(), err) return false } - for i := 0; i < confirmDownRetries; i++ { - ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second) + for i := 0; i < c.confirmDownRetries; i++ { + ctx, cancel := context.WithTimeout(context.Background(), c.confirmDownSleep*2) defer cancel() resp, err := http.DefaultClient.Do(req.WithContext(ctx)) var bod []byte @@ -1940,8 +1951,8 @@ func confirmNodeDown(uri URI, log logger.Logger) bool { } } - log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod) - time.Sleep(confirmDownSleep * time.Second) + c.logger.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod) + time.Sleep(c.confirmDownSleep) } return true } @@ -1969,7 +1980,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) { // not already removed by a removeNode request. We treat this as the // host being temporarily unavailable, and expect it to come back // up. - if confirmNodeDown(e.Node.URI, c.logger) { + if c.confirmNodeDown(e.Node.URI) { if c.removeNodeBasicSorted(e.Node.ID) { c.Topology.nodeStates[e.Node.ID] = nodeStateDown // put the cluster into STARTING if we've lost a number of nodes @@ -2046,6 +2057,9 @@ func (c *cluster) nodeJoin(node *Node) error { c.logger.Printf("node: %v changed URI from %s to %s", cnode.ID, cnode.URI, node.URI) cnode.URI = node.URI } + if cnode.GRPCURI != node.GRPCURI { + cnode.GRPCURI = node.GRPCURI + } return c.unprotectedSetStateAndBroadcast(c.determineClusterState()) } @@ -2141,7 +2155,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { - is := &IndexStatus{Name: idx.Name} + is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() @@ -2150,6 +2164,7 @@ func (c *cluster) nodeStatus() *NodeStatus { } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, }) } @@ -2445,6 +2460,7 @@ type ClusterStatus struct { ClusterID string State string Nodes []*Node + Schema *Schema } // ResizeInstruction contains the instruction provided to a node @@ -2486,7 +2502,7 @@ type translationResizeNode struct { // Schema contains information about indexes and their configuration. type Schema struct { - Indexes []*IndexInfo + Indexes []*IndexInfo `json:"indexes"` } func encodeTopology(topology *Topology) *internal.Topology { @@ -2524,8 +2540,9 @@ type CreateShardMessage struct { // CreateIndexMessage is an internal message indicating index creation. type CreateIndexMessage struct { - Index string - Meta *IndexOptions + Index string + CreatedAt int64 + Meta *IndexOptions } // DeleteIndexMessage is an internal message indicating index deletion. @@ -2535,9 +2552,10 @@ type DeleteIndexMessage struct { // CreateFieldMessage is an internal message indicating field creation. type CreateFieldMessage struct { - Index string - Field string - Meta *FieldOptions + Index string + Field string + CreatedAt int64 + Meta *FieldOptions } // DeleteFieldMessage is an internal message indicating field deletion. @@ -2600,13 +2618,15 @@ type NodeStatus struct { // IndexStatus is an internal message representing the contents of an index. type IndexStatus struct { - Name string - Fields []*FieldStatus + Name string + CreatedAt int64 + Fields []*FieldStatus } // FieldStatus is an internal message representing the contents of a field. type FieldStatus struct { Name string + CreatedAt int64 AvailableShards *roaring.Bitmap } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index de87ad5ae..b9e714326 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -943,18 +943,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) { t.Error(err) } uri.Port = uint16(iport) - if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + c := newCluster() + c.logger = logger.NewVerboseLogger(os.Stdout) + if c.confirmNodeDown(uri) { t.Errorf("expected node to be up") } } func TestCluster_confirmNodeDownTimeout(t *testing.T) { + sleep := 50 * time.Millisecond + retries := 5 if testing.Short() { t.Skip() } r := mux.NewRouter() r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(confirmDownSleep * time.Second * confirmDownRetries) + time.Sleep(sleep * time.Duration(retries)) fmt.Fprintln(w, "ignored") })) server := httptest.NewServer(r) @@ -973,8 +977,11 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) { t.Error(err) } uri.Port = uint16(iport) - - if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + c := newCluster() + c.confirmDownSleep = sleep + c.confirmDownRetries = retries + c.logger = logger.NewVerboseLogger(os.Stdout) + if !c.confirmNodeDown(uri) { t.Errorf("expected node to be down") } } @@ -987,8 +994,12 @@ func TestCluster_confirmNodeDownDown(t *testing.T) { uri.Scheme = "http" uri.Host = "DoesntMatter" uri.Port = 6666 + c := newCluster() + c.confirmDownSleep = 50 * time.Millisecond + c.confirmDownRetries = 5 + c.logger = logger.NewVerboseLogger(os.Stdout) - if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) { + if !c.confirmNodeDown(uri) { t.Errorf("expected node to be down") } } diff --git a/docs/api-reference.md b/docs/api-reference.md index 68dae46b5..e8a2ac1a3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -24,21 +24,25 @@ curl -XGET localhost:10101/index/user ``` ``` response { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ + { + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } } + ], + "shardWidth": 1048576 } ``` @@ -57,7 +61,7 @@ The request payload is in JSON, and may contain the `options` field. The `option curl -XPOST localhost:10101/index/user -d '{"options":{"keys":true}}' ``` ``` response -{"success":true} +{"success":true,"name":"user","createdAt":1591179042178854000} ``` ### Remove index @@ -151,20 +155,34 @@ represents a particular bit to be set. Timestamps are optional, but if they exist must also contain the same number of items as rows and columns. The column IDs must all be in the shard specified in the request. +Some endpoints and data structures include a `CreatedAt` fields. +This is typically stored as a timestamp, but it's purpose is not to inform of the creation date of a particular index or field, +but to serve as a unique identifier for use in cache invalidation. + +The problem is that users of Pilosa (such as ingesters e.g. the [IDK](https://github.com/molecula/idk)) +can usually assume that translation keys for records and field values never change - they are only appended to, and can therefore be trivially cached. +This is true except in cases where an index or field gets deleted and then recreated, +or if Pilosa is restored from a backup. +So the ingesters must send their current `CreatedAt` value which will have changed if either of those two conditions has occured (or if Pilosa was just restarted), +and the ingester will know that it needs to drop its cache. + ``` message ImportRequest { - string Index = 1; - string Field = 2; - uint64 Shard = 3; - repeated uint64 RowIDs = 4; - repeated uint64 ColumnIDs = 5; - repeated string RowKeys = 7; - repeated string ColumnKeys = 8; - repeated int64 Timestamps = 6; + string Index = 1; + string Field = 2; + uint64 Shard = 3; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; + repeated int64 Timestamps = 6; + repeated string RowKeys = 7; + repeated string ColumnKeys = 8; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } ``` + ### Create field `POST /index//field/` @@ -200,7 +218,7 @@ curl localhost:10101/index/user/field/quantity \ -d '{"options": {"type": "int", "min": -1000, "max":2000}}' ``` ``` response -{"success":true} +{"success":true,"name":"quantity","createdAt":1591180110914425000} ``` Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. @@ -209,16 +227,16 @@ Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, curl localhost:10101/index/user/field/language -X POST ``` ``` response -{"success":true} +{"success":true,"name":"language","createdAt":1591180128294321000} ``` ``` request curl localhost:10101/index/repository/field/stats \ -X POST \ - -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"options":{"type": "int", "min": 0, "max": 1000000}}' ``` ``` response -{"success":true} +{"success":true,"name":"stats","createdAt":1591180737881627000} ``` ### Remove field @@ -245,34 +263,52 @@ curl -XGET localhost:10101/schema ``` ``` response { - "indexes": [ + "indexes": [ + { + "name": "user", + "createdAt": 1591178953061239000, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ { - "fields": [ - { - "name": "event", - "options": { - "keys": false, - "timeQuantum": "YMD", - "type": "time" - } - }, - { - "name": "language", - "options": { - "cacheSize": 50000, - "cacheType": "ranked", - "keys": false, - "type": "set" - } - } - ], - "name": "user", - "options": { - "keys": false, - "trackExistence": true - } + "name": "event", + "createdAt": 1591178962332452000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "language", + "createdAt": 1591180128294321000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "quantity", + "createdAt": 1591180110914425000, + "options": { + "type": "int", + "base": 0, + "bitDepth": 0, + "min": -1000, + "max": 2000, + "keys": false, + "foreignIndex": "" + } } - ] + ], + "shardWidth": 1048576 + } + ] } ``` @@ -304,7 +340,7 @@ Returns the version of the Pilosa server. curl -XGET localhost:10101/version ``` ``` response -{"version":"v0.6.0"} +{"version":"2.0.0-alpha.20-6-gb9d8d6b4"} ``` ### Get status @@ -318,19 +354,25 @@ curl -XGET localhost:10101/status ``` ```response { - "localID": "d3369125-29d8-4305-a351-b4474d14a542", - "nodes": [ - { - "id": "d3369125-29d8-4305-a351-b4474d14a542", - "isCoordinator": true, - "uri": { - "host": "localhost", - "port": 10101, - "scheme": "http" - } - } - ], - "state": "NORMAL" + "state": "NORMAL", + "nodes": [ + { + "id": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9", + "uri": { + "scheme": "http", + "host": "localhost", + "port": 10101 + }, + "grpc-uri": { + "scheme": "http", + "host": "localhost", + "port": 20101 + }, + "isCoordinator": true, + "state": "READY" + } + ], + "localID": "1b018ce0-5de5-4da9-9285-6c4c0d8106f9" } ``` diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c41d38c53..82a31ce05 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -403,27 +403,31 @@ func (s Serializer) encodeImportResponse(m *pilosa.ImportResponse) *internal.Imp func (s Serializer) encodeImportRequest(m *pilosa.ImportRequest) *internal.ImportRequest { return &internal.ImportRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - RowIDs: m.RowIDs, - ColumnIDs: m.ColumnIDs, - RowKeys: m.RowKeys, - ColumnKeys: m.ColumnKeys, - Timestamps: m.Timestamps, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + RowIDs: m.RowIDs, + ColumnIDs: m.ColumnIDs, + RowKeys: m.RowKeys, + ColumnKeys: m.ColumnKeys, + Timestamps: m.Timestamps, } } func (s Serializer) encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValueRequest { return &internal.ImportValueRequest{ - Index: m.Index, - Field: m.Field, - Shard: m.Shard, - ColumnIDs: m.ColumnIDs, - ColumnKeys: m.ColumnKeys, - Values: m.Values, - FloatValues: m.FloatValues, - StringValues: m.StringValues, + Index: m.Index, + Field: m.Field, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Shard: m.Shard, + ColumnIDs: m.ColumnIDs, + ColumnKeys: m.ColumnKeys, + Values: m.Values, + FloatValues: m.FloatValues, + StringValues: m.StringValues, } } @@ -438,20 +442,23 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) * i++ } return &internal.ImportRoaringRequest{ - Clear: m.Clear, - Action: m.Action, - Block: uint64(m.Block), - Views: views, + IndexCreatedAt: m.IndexCreatedAt, + FieldCreatedAt: m.FieldCreatedAt, + Clear: m.Clear, + Action: m.Action, + Block: uint64(m.Block), + Views: views, } } func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *internal.ImportColumnAttrsRequest { return &internal.ImportColumnAttrsRequest{ - Index: m.Index, - Shard: m.Shard, - AttrKey: m.AttrKey, - AttrVals: m.AttrVals, - ColumnIDs: m.ColumnIDs, + Index: m.Index, + IndexCreatedAt: m.IndexCreatedAt, + Shard: m.Shard, + AttrKey: m.AttrKey, + AttrVals: m.AttrVals, + ColumnIDs: m.ColumnIDs, } } @@ -593,9 +600,10 @@ func (s Serializer) encodeIndexInfos(idxs []*pilosa.IndexInfo) []*internal.Index func (s Serializer) encodeIndexInfo(idx *pilosa.IndexInfo) *internal.Index { return &internal.Index{ - Name: idx.Name, - Options: s.encodeIndexMeta(&idx.Options), - Fields: s.encodeFieldInfos(idx.Fields), + Name: idx.Name, + CreatedAt: idx.CreatedAt, + Options: s.encodeIndexMeta(&idx.Options), + Fields: s.encodeFieldInfos(idx.Fields), } } @@ -609,9 +617,10 @@ func (s Serializer) encodeFieldInfos(fs []*pilosa.FieldInfo) []*internal.Field { func (s Serializer) encodeFieldInfo(f *pilosa.FieldInfo) *internal.Field { ifield := &internal.Field{ - Name: f.Name, - Meta: s.encodeFieldOptions(&f.Options), - Views: make([]string, 0, len(f.Views)), + Name: f.Name, + CreatedAt: f.CreatedAt, + Meta: s.encodeFieldOptions(&f.Options), + Views: make([]string, 0, len(f.Views)), } for _, viewinfo := range f.Views { @@ -655,6 +664,7 @@ func (s Serializer) encodeNode(n *pilosa.Node) *internal.Node { URI: s.encodeURI(n.URI), IsCoordinator: n.IsCoordinator, State: n.State, + GRPCURI: s.encodeURI(n.GRPCURI), } } @@ -671,6 +681,7 @@ func (s Serializer) encodeClusterStatus(m *pilosa.ClusterStatus) *internal.Clust State: m.State, ClusterID: m.ClusterID, Nodes: s.encodeNodes(m.Nodes), + Schema: s.encodeSchema(m.Schema), } } @@ -684,8 +695,9 @@ func (s Serializer) encodeCreateShardMessage(m *pilosa.CreateShardMessage) *inte func (s Serializer) encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateIndexMessage { return &internal.CreateIndexMessage{ - Index: m.Index, - Meta: s.encodeIndexMeta(m.Meta), + Index: m.Index, + CreatedAt: m.CreatedAt, + Meta: s.encodeIndexMeta(m.Meta), } } @@ -704,9 +716,10 @@ func (s Serializer) encodeDeleteIndexMessage(m *pilosa.DeleteIndexMessage) *inte func (s Serializer) encodeCreateFieldMessage(m *pilosa.CreateFieldMessage) *internal.CreateFieldMessage { return &internal.CreateFieldMessage{ - Index: m.Index, - Field: m.Field, - Meta: s.encodeFieldOptions(m.Meta), + Index: m.Index, + Field: m.Field, + CreatedAt: m.CreatedAt, + Meta: s.encodeFieldOptions(m.Meta), } } @@ -785,8 +798,9 @@ func (s Serializer) encodeNodeStatus(m *pilosa.NodeStatus) *internal.NodeStatus func (s Serializer) encodeIndexStatus(m *pilosa.IndexStatus) *internal.IndexStatus { return &internal.IndexStatus{ - Name: m.Name, - Fields: s.encodeFieldStatuses(m.Fields), + Name: m.Name, + CreatedAt: m.CreatedAt, + Fields: s.encodeFieldStatuses(m.Fields), } } @@ -801,6 +815,7 @@ func (s Serializer) encodeIndexStatuses(a []*pilosa.IndexStatus) []*internal.Ind func (s Serializer) encodeFieldStatus(m *pilosa.FieldStatus) *internal.FieldStatus { return &internal.FieldStatus{ Name: m.Name, + CreatedAt: m.CreatedAt, AvailableShards: m.AvailableShards.Slice(), } } @@ -937,6 +952,7 @@ func (s Serializer) decodeIndexes(idxs []*internal.Index, m []*pilosa.IndexInfo) func (s Serializer) decodeIndex(idx *internal.Index, m *pilosa.IndexInfo) { m.Name = idx.Name + m.CreatedAt = idx.CreatedAt m.Options = pilosa.IndexOptions{} s.decodeIndexMeta(idx.Options, &m.Options) m.Fields = make([]*pilosa.FieldInfo, len(idx.Fields)) @@ -952,6 +968,7 @@ func (s Serializer) decodeFields(fs []*internal.Field, m []*pilosa.FieldInfo) { func (s Serializer) decodeField(f *internal.Field, m *pilosa.FieldInfo) { m.Name = f.Name + m.CreatedAt = f.CreatedAt m.Options = pilosa.FieldOptions{} s.decodeFieldOptions(f.Meta, &m.Options) m.Views = make([]*pilosa.ViewInfo, 0, len(f.Views)) @@ -991,11 +1008,14 @@ func (s Serializer) decodeClusterStatus(cs *internal.ClusterStatus, m *pilosa.Cl m.ClusterID = cs.ClusterID m.Nodes = make([]*pilosa.Node, len(cs.Nodes)) s.decodeNodes(cs.Nodes, m.Nodes) + m.Schema = &pilosa.Schema{} + s.decodeSchema(cs.Schema, m.Schema) } func (s Serializer) decodeNode(node *internal.Node, m *pilosa.Node) { m.ID = node.ID s.decodeURI(node.URI, &m.URI) + s.decodeURI(node.GRPCURI, &m.GRPCURI) m.IsCoordinator = node.IsCoordinator m.State = node.State } @@ -1014,6 +1034,7 @@ func (s Serializer) decodeCreateShardMessage(pb *internal.CreateShardMessage, m func (s Serializer) decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateIndexMessage) { m.Index = pb.Index + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.IndexOptions{} s.decodeIndexMeta(pb.Meta, m.Meta) } @@ -1032,6 +1053,7 @@ func (s Serializer) decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m func (s Serializer) decodeCreateFieldMessage(pb *internal.CreateFieldMessage, m *pilosa.CreateFieldMessage) { m.Index = pb.Index m.Field = pb.Field + m.CreatedAt = pb.CreatedAt m.Meta = &pilosa.FieldOptions{} s.decodeFieldOptions(pb.Meta, m.Meta) } @@ -1105,6 +1127,7 @@ func (s Serializer) decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.Ind func (s Serializer) decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name + m.CreatedAt = pb.CreatedAt m.Fields = s.decodeFieldStatuses(pb.Fields) } @@ -1119,6 +1142,7 @@ func (s Serializer) decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.Fie func (s Serializer) decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { m.Name = pb.Name + m.CreatedAt = pb.CreatedAt m.AvailableShards = roaring.NewBitmap(pb.AvailableShards...) } @@ -1147,6 +1171,8 @@ func (s Serializer) decodeImportRequest(pb *internal.ImportRequest, m *pilosa.Im m.RowKeys = pb.RowKeys m.ColumnKeys = pb.ColumnKeys m.Timestamps = pb.Timestamps + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportValueRequest) { @@ -1158,6 +1184,8 @@ func (s Serializer) decodeImportValueRequest(pb *internal.ImportValueRequest, m m.Values = pb.Values m.FloatValues = pb.FloatValues m.StringValues = pb.StringValues + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { @@ -1169,10 +1197,13 @@ func (s Serializer) decodeImportRoaringRequest(pb *internal.ImportRoaringRequest m.Action = pb.Action m.Block = int(pb.Block) m.Views = views + m.IndexCreatedAt = pb.IndexCreatedAt + m.FieldCreatedAt = pb.FieldCreatedAt } func (s Serializer) decodeImportColumnAttrsRequest(pb *internal.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) { m.Index = pb.Index + m.IndexCreatedAt = pb.IndexCreatedAt m.Shard = pb.Shard m.AttrKey = pb.AttrKey m.AttrVals = pb.AttrVals diff --git a/executor.go b/executor.go index 3aa677c44..f2dd49fcb 100644 --- a/executor.go +++ b/executor.go @@ -59,6 +59,8 @@ type executor struct { // Maximum number of Set() or Clear() commands per request. MaxWritesPerRequest int + shutdown bool + workMu sync.RWMutex workersWG sync.WaitGroup workerPoolSize int work chan job @@ -115,6 +117,9 @@ func newExecutor(opts ...executorOption) *executor { } func (e *executor) Close() error { + e.workMu.Lock() + defer e.workMu.Unlock() + e.shutdown = true close(e.work) e.workersWG.Wait() return nil @@ -155,6 +160,7 @@ func (e *executor) registerOps(ops []ext.BitmapOp) error { // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") + span.LogKV("pql", q.String()) defer span.Finish() resp := QueryResponse{} @@ -357,7 +363,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call row = r.Pos default: return fmt.Errorf("precomputed call %s returned unexpected non-Row data: %T", c.Name, v) - + } + if err := ctx.Err(); err != nil { + return err } c.Children = []*pql.Call{} c.Name = "Precomputed" @@ -376,6 +384,9 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call // handlePreCallChildren handles any pre-calls in the children of a given call. func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) error { for i := range c.Children { + if err := ctx.Err(); err != nil { + return err + } if err := e.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil { return err } @@ -383,6 +394,9 @@ func (e *executor) handlePreCallChildren(ctx context.Context, index string, c *p for _, val := range c.Args { // Handle Call() operations which exist inside named arguments, too. if call, ok := val.(*pql.Call); ok { + if err := ctx.Err(); err != nil { + return err + } if err := e.handlePreCalls(ctx, index, call, shards, opt); err != nil { return err } @@ -649,12 +663,12 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeIncludesColumnCallShard(ctx, index, c, shard, col) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(bool) return other || v.(bool) } @@ -704,12 +718,12 @@ func (e *executor) executeFieldValueCall(ctx context.Context, index string, c *p shard := colID / ShardWidth // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeFieldValueCallShard(ctx, field, colID, shard) } // Select single returned result at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) if other.Count == 1 { return other @@ -833,12 +847,15 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call // using the executor.mapReduce() method. func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeAllCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { + if err := ctx.Err(); err != nil { + return err + } other, _ := prev.(*Row) if other == nil { other = NewRow() @@ -888,12 +905,12 @@ func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeSumCountShard(ctx, index, c, nil, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.add(v.(ValCount)) } @@ -940,13 +957,16 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeGenericFieldShard(ctx, index, c, op, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(SignedRow) + if err := ctx.Err(); err != nil { + return err + } return other.union(v.(SignedRow)) } @@ -973,12 +993,12 @@ func (e *executor) executeMin(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMinShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.smaller(v.(ValCount)) } @@ -1009,12 +1029,12 @@ func (e *executor) executeMax(ctx context.Context, index string, c *pql.Call, sh } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMaxShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(ValCount) return other.larger(v.(ValCount)) } @@ -1041,12 +1061,12 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMinRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. if prev == nil { @@ -1080,12 +1100,12 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeMaxRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. if prev == nil { @@ -1124,6 +1144,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, index string, c * // executeBitmapCall executes a call that returns a bitmap. func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") + span.LogKV("pqlCallName", c.Name) defer span.Finish() indexTag := "index:" + index @@ -1136,16 +1157,19 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeBitmapCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(*Row) if other == nil { other = NewRow() } + if err := ctx.Err(); err != nil { + return err + } other.Merge(v.(*Row)) return other } @@ -1503,12 +1527,12 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C defer span.Finish() // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeTopNShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(*PairsField) vpf, _ := v.(*PairsField) if other == nil { @@ -1516,6 +1540,9 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C } else if vpf == nil { return other } + if err := ctx.Err(); err != nil { + return err + } other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs) return other } @@ -1743,10 +1770,16 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call return nil, err } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // perform necessary Rows queries (any that have limit or columns args) - // TODO, call async? would only help if multiple Rows queries had a column // or limit arg. // TODO support TopN in here would be really cool - and pretty easy I think. + bases := make(map[int]int64) childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { // Check "field" first for backwards compatibility, then set _field. @@ -1766,6 +1799,19 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if err != nil { return nil, errors.Wrap(err, "getting column") } + fieldName, ok := child.Args["_field"].(string) + if !ok { + return nil, errors.Errorf("%s call must have field with valid (string) field name. Got %v of type %[2]T", child.Name, child.Args["_field"]) + } + f := idx.Field(fieldName) + if f == nil { + return nil, ErrFieldNotFound + } + switch f.Type() { + case FieldTypeInt: + bases[i] = f.bsiGroup(f.name).Base + } + if hasLimit || hasCol { // we need to perform this query cluster-wide ahead of executeGroupByShard childRows[i], err = e.executeRows(ctx, index, child, shards, opt) if err != nil { @@ -1778,12 +1824,15 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, filter, shard, childRows) + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { + return e.executeGroupByShard(ctx, index, c, filter, shard, childRows, bases) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.([]GroupCount) + if err := ctx.Err(); err != nil { + return err + } return mergeGroupCounts(other, v.([]GroupCount), limit) } // Get full result set. @@ -2127,7 +2176,7 @@ func applyConditionToGroupCounts(gcs []GroupCount, subj string, cond *pql.Condit return gcs[:i] } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) { +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs, bases map[int]int64) (_ []GroupCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupByShard") defer span.Finish() @@ -2175,6 +2224,13 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } } + // Apply bases. + for i, base := range bases { + for _, r := range results { + *r.Group[i].Value += base + } + } + return results, nil } @@ -2197,7 +2253,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeRowsShard(ctx, index, fieldName, c, shard) } @@ -2210,8 +2266,11 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(RowIDs) + if err := ctx.Err(); err != nil { + return err + } return other.merge(v.(RowIDs), limit) } // Get full result set. @@ -2223,7 +2282,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(_ context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(ctx context.Context, index string, fieldName string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -2333,12 +2392,15 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s } for _, view := range views { + if err := ctx.Err(); err != nil { + return nil, err + } frag := e.Holder.fragment(index, fieldName, view, shard) if frag == nil { continue } - viewRows := frag.rows(start, filters...) + viewRows := frag.rows(ctx, start, filters...) rowIDs = rowIDs.merge(viewRows, limit) } @@ -2463,21 +2525,15 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return nil, ErrFieldNotFound } - // EQ null (not implemented: flip frag.NotNull with max ColumnID) + // EQ null _exists - frag.NotNull() // NEQ null frag.NotNull() // BETWEEN a,b(in) BETWEEN/frag.RowBetween() // BETWEEN a,b(out) BETWEEN/frag.NotNull() // EQ frag.RangeOp // NEQ frag.RangeOp - // Handle `!= null`. + // Handle `!= null` and `== null`. if cond.Op == pql.NEQ && cond.Value == nil { - // Find bsiGroup. - bsig := f.bsiGroup(fieldName) - if bsig == nil { - return nil, ErrBSIGroupNotFound - } - // Retrieve fragment. frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { @@ -2486,6 +2542,37 @@ func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c return frag.notNull() + } else if cond.Op == pql.EQ && cond.Value == nil { + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + var notNull *Row + var err error + + // Retrieve notNull from fragment if it exists. + if frag := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard); frag != nil { + if notNull, err = frag.notNull(); err != nil { + return nil, errors.Wrap(err, "getting fragment not null") + } + } else { + notNull = NewRow() + } + + return existenceRow.Difference(notNull), nil + } else if cond.Op == pql.BETWEEN || cond.Op == pql.BTWN_LT_LT || cond.Op == pql.BTWN_LTE_LT || cond.Op == pql.BTWN_LT_LTE { predicates, err := getCondIntSlice(f, cond) @@ -2787,7 +2874,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err @@ -2796,7 +2883,7 @@ func (e *executor) executeGenericCount(ctx context.Context, index string, c *pql } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(uint64) return other + v.(uint64) } @@ -2822,7 +2909,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err @@ -2831,7 +2918,7 @@ func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { other, _ := prev.(uint64) return other + v.(uint64) } @@ -2944,12 +3031,12 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeClearRowShard(ctx, index, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { val := v.(bool) if prev == nil { return val @@ -3032,12 +3119,12 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C } // Execute calls in bulk on each remote node and merge. - mapFn := func(shard uint64) (interface{}, error) { + mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { return e.executeSetRowShard(ctx, indexName, c, shard) } // Merge returned results at coordinating node. - reduceFn := func(prev, v interface{}) interface{} { + reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { val := v.(bool) if prev == nil { return val @@ -3046,7 +3133,15 @@ func (e *executor) executeSetRow(ctx context.Context, indexName string, c *pql.C } result, err := e.mapReduce(ctx, indexName, shards, c, opt, mapFn, reduceFn) - return result.(bool), err + if err != nil { + return false, err + } + + b, ok := result.(bool) + if !ok { + return false, errors.New("unsupported result type") + } + return b, nil } // executeSetRowShard executes a SetRow() call for a single shard. @@ -3566,7 +3661,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, cancel, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { return nil, errors.Wrap(err, "starting mapper") } @@ -3586,7 +3681,7 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { + if err := e.mapper(ctx, cancel, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); errors.Cause(err) == errShardUnavailable { return nil, resp.err } else if err != nil { return nil, errors.Wrap(err, "calling mapper") @@ -3595,7 +3690,11 @@ func (e *executor) mapReduce(ctx context.Context, index string, shards []uint64, } // Reduce value. - result = reduceFn(result, resp.result) + result = reduceFn(ctx, result, resp.result) + if err, ok := result.(error); ok { + cancel() + return nil, err + } // If all shards have been processed then return. shardN += len(resp.shards) @@ -3643,9 +3742,10 @@ func makeEmbeddedDataForShards(allRows []*Row, shards []uint64) []*Row { return newRows } -func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { +func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *execOptions, mapFn mapFunc, reduceFn reduceFunc) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") defer span.Finish() + done := ctx.Done() // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) @@ -3672,11 +3772,16 @@ func (e *executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod } resp.err = err } - // Return response to the channel. select { - case <-ctx.Done(): + case <-done: case ch <- resp: + // The cancel coming after the above send is intentional. + // We want to report the actual error that happened + // before we cause anything to return "context canceled". + if resp.err != nil { + cancel() + } } }(n, nodeShards) } @@ -3693,7 +3798,7 @@ type job struct { func worker(work chan job) { for j := range work { - result, err := j.mapFn(j.shard) + result, err := j.mapFn(j.ctx, j.shard) select { case <-j.ctx.Done(): @@ -3702,10 +3807,21 @@ func worker(work chan job) { } } +var errShutdown = errors.New("executor has shut down") + // mapperLocal performs map & reduce entirely on the local node. func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapperLocal") defer span.Finish() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + done := ctx.Done() + e.workMu.RLock() + defer e.workMu.RUnlock() + + if e.shutdown { + return nil, errShutdown + } ch := make(chan mapResponse, len(shards)) @@ -3723,13 +3839,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu var result interface{} for { select { - case <-ctx.Done(): + case <-done: return nil, ctx.Err() case resp := <-ch: if resp.err != nil { return nil, resp.err } - result = reduceFn(result, resp.result) + result = reduceFn(ctx, result, resp.result) + if err, ok := result.(error); ok { + cancel() + return nil, err + } maxShard++ } @@ -4241,9 +4361,9 @@ func validateQueryContext(ctx context.Context) error { // errShardUnavailable is a marker error if no nodes are available. var errShardUnavailable = errors.New("shard unavailable") -type mapFunc func(shard uint64) (interface{}, error) +type mapFunc func(ctx context.Context, shard uint64) (interface{}, error) -type reduceFunc func(prev, v interface{}) interface{} +type reduceFunc func(ctx context.Context, prev, v interface{}) interface{} type mapResponse struct { node *Node @@ -4743,18 +4863,21 @@ func newGroupByIterator(executor *executor, rowIDs []RowIDs, children []*pql.Cal // nextAtIdx is a recursive helper method for getting the next row for the field // at index i, and then updating the rows in the "higher" fields if it wraps. -func (gbi *groupByIterator) nextAtIdx(i int) { +func (gbi *groupByIterator) nextAtIdx(ctx context.Context, i int) (err error) { // loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed. for { + if err = ctx.Err(); err != nil { + return err + } nr, rowID, value, wrapped := gbi.rowIters[i].Next() if nr == nil { gbi.done = true - return + return nil } if wrapped && i != 0 { - gbi.nextAtIdx(i - 1) - if gbi.done { - return + err = gbi.nextAtIdx(ctx, i-1) + if gbi.done || err != nil { + return err } } if i == 0 && gbi.filter != nil { @@ -4771,6 +4894,7 @@ func (gbi *groupByIterator) nextAtIdx(i int) { break } } + return nil } // Next returns a GroupCount representing the next group by record. When there @@ -4778,6 +4902,9 @@ func (gbi *groupByIterator) nextAtIdx(i int) { func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool, err error) { // loop until we find a result with count > 0 for { + if err := ctx.Err(); err != nil { + return ret, false, err + } if gbi.done { return ret, true, nil } @@ -4805,7 +4932,10 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool } } if ret.Count == 0 { - gbi.nextAtIdx(len(gbi.rows) - 1) + err := gbi.nextAtIdx(ctx, len(gbi.rows)-1) + if err != nil { + return ret, false, err + } continue } break @@ -4820,9 +4950,9 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool } // set up for next call - gbi.nextAtIdx(len(gbi.rows) - 1) + err = gbi.nextAtIdx(ctx, len(gbi.rows)-1) - return ret, false, nil + return ret, false, err } // getCondIntSlice looks at the field, the cond op type (which is diff --git a/executor_test.go b/executor_test.go index e29dd6b99..6bf3f2a14 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2371,7 +2371,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{TrackExistence: true}) if err != nil { t.Fatal(err) } @@ -2414,6 +2414,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { } t.Run("EQ", func(t *testing.T) { + // EQ null + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{1, + 50, + ShardWidth, + ShardWidth + 1, + ShardWidth + 2, + (5 * ShardWidth) + 100, + }, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) + } + // EQ if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) { @@ -2983,6 +2996,37 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { test.CheckGroupBy(t, expected, results) } }) + + t.Run("groupBy on ints with offset regression", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, hint=1) + Set(1, hint=2) + Set(2, hint=3) + `}); err != nil { + t.Fatalf("querying remote: %v", err) + } + + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `GroupBy(Rows(hint))`, + }); err != nil { + t.Fatalf("GroupBy querying: %v", err) + } else { + var a, b, c int64 = 1, 2, 3 + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "hint", Value: &a}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "hint", Value: &b}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1}, + } + + results := res.Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + } + }) } // Ensure executor returns an error if too many writes are in a single request. @@ -3463,15 +3507,15 @@ func TestExecutor_Execute_All(t *testing.T) { t.Fatal(err) } - // Create an import request that sets a full shard, - // plus a couple bits set on either side of it, and - // a final bit set in a fourth shard. + // Create an import request that sets things on either end + // of a shard, plus a couple bits set on either side of it, + // and a final bit set in a fourth shard. // // shard0 shard1 shard2 shard3 // |----------|----------|----------|----------| - // | **|**********|** | * + // | **|** **|** | * // - bitCount := ShardWidth + 5 + bitCount := 100 + 5 req := &pilosa.ImportRequest{ Index: index.Name(), Field: fld.Name(), @@ -3479,10 +3523,14 @@ func TestExecutor_Execute_All(t *testing.T) { RowIDs: make([]uint64, bitCount), ColumnIDs: make([]uint64, bitCount), } - for i := 0; i < bitCount-1; i++ { + for i := 0; i < bitCount/2; i++ { req.RowIDs[i] = 10 req.ColumnIDs[i] = uint64(i + ShardWidth - 2) } + for i := bitCount / 2; i < bitCount-1; i++ { + req.RowIDs[i] = 10 + req.ColumnIDs[i] = uint64(i + (ShardWidth * 2) - bitCount + 5) + } req.RowIDs[bitCount-1] = 10 req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) @@ -3508,7 +3556,7 @@ func TestExecutor_Execute_All(t *testing.T) { {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2}, {qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2}, {qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1}, - {qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth}, + {qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)}, } for i, test := range tests { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { @@ -4001,6 +4049,25 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } }) + t.Run("Err_Store(Distinct)", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name()) + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil { + t.Fatalf("expected 'unsupported result type' error, got: %+v", res) + } + }) } func benchmarkExistence(nn bool, b *testing.B) { diff --git a/field.go b/field.go index d38aa2ffb..84bdbec73 100644 --- a/field.go +++ b/field.go @@ -86,10 +86,11 @@ var availableShardFileFlushDuration = &protected{ // Field represents a container for views. type Field struct { - mu sync.RWMutex - path string - index string - name string + mu sync.RWMutex + createdAt int64 + path string + index string + name string viewMap map[string]*view @@ -382,6 +383,14 @@ func newField(path, index, name string, opts FieldOption) (*Field, error) { // Name returns the name the field was initialized with. func (f *Field) Name() string { return f.name } +// CreatedAt is an timestamp for a specific version of field. +func (f *Field) CreatedAt() int64 { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.createdAt +} + // Index returns the index name the field was initialized with. func (f *Field) Index() string { return f.index } @@ -1971,9 +1980,10 @@ func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FieldInfo represents schema information for a field. type FieldInfo struct { - Name string `json:"name"` - Options FieldOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` } type fieldInfoSlice []*FieldInfo diff --git a/fragment.go b/fragment.go index 81f644e3c..5f26828ec 100644 --- a/fragment.go +++ b/fragment.go @@ -1159,14 +1159,24 @@ func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, } } +func absInt64(v int64) uint64 { + switch { + case v > 0: + return uint64(v) + case v == -9223372036854775808: + return 9223372036854775808 + default: + return uint64(-v) + } +} + func (f *fragment) rangeEQ(bitDepth uint, predicate int64) (*Row, error) { // Start with set of columns with values set. b := f.row(bsiExistsBit) // Filter to only positive/negative numbers. - upredicate := uint64(predicate) + upredicate := absInt64(predicate) if predicate < 0 { - upredicate = uint64(-predicate) b = b.Intersect(f.row(bsiSignBit)) // only negatives } else { b = b.Difference(f.row(bsiSignBit)) // only positives @@ -1204,27 +1214,42 @@ func (f *fragment) rangeNEQ(bitDepth uint, predicate int64) (*Row, error) { } func (f *fragment) rangeLT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + if predicate == 1 && !allowEquality { + predicate, allowEquality = 0, true + } + // Start with set of columns with values set. b := f.row(bsiExistsBit) - // Create predicate without sign bit. - upredicate := uint64(predicate) - if predicate < 0 { - upredicate = uint64(-predicate) - } + // Get the sign bit row. + sign := f.row(bsiSignBit) - // If predicate is positive, return all positives less than predicate and all negatives. - if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { - pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) + // Create predicate without sign bit. + upredicate := absInt64(predicate) + + switch { + case predicate == 0 && !allowEquality: + // Match all negative integers. + return b.Intersect(sign), nil + case predicate == 0 && allowEquality: + // Match all integers that are either negative or 0. + zeroes, err := f.rangeEQ(bitDepth, 0) if err != nil { return nil, err } - neg := f.row(bsiSignBit) - return neg.Union(pos), nil + return b.Intersect(sign).Union(zeroes), nil + case predicate < 0: + // Match all every negative number beyond the predicate. + return f.rangeGTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + default: + // Match positive numbers less than the predicate, and all negatives. + pos, err := f.rangeLTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + neg := b.Intersect(sign) + return pos.Union(neg), nil } - - // Otherwise if predicate is negative, return all negatives greater than upredicate. - return f.rangeGTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) } // msb gives the 1-indexed position (counting from lsb) of the most @@ -1237,110 +1262,118 @@ func msb(x uint64) uint { // rangeLTUnsigned returns all bits LT/LTE the predicate without considering the sign bit. func (f *fragment) rangeLTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - - // if the predicate is larger than all representable numbers given - // our bitDepth... then just return everything. - if msb(predicate) > bitDepth { + switch { + case msb(predicate) > bitDepth: + fallthrough + case predicate == (1<= 0; i-- { + // Compare intermediate bits. + matched := NewRow() + remaining := filter + for i := int(bitDepth - 1); i >= 0 && predicate > 0 && remaining.Any(); i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit := (predicate >> uint(i)) & 1 - - // Remove any columns with higher bits set. - if leadingZeros { - if bit == 0 { - filter = filter.Difference(row) - continue - } else { - leadingZeros = false - } - } - - // Handle last bit differently. - // If bit is zero then return only already kept columns. - // If bit is one then remove any one columns. - if i == 0 && !allowEquality { - if bit == 0 { - return keep, nil - } - return filter.Difference(row.Difference(keep)), nil - } - - // If bit is zero then remove all set columns not in excluded bitmap. - if bit == 0 { - filter = filter.Difference(row.Difference(keep)) - continue - } - - // If bit is set then add columns for set bits to exclude. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep = keep.Union(filter.Difference(row)) + zeroes := remaining.Difference(row) + switch (predicate >> uint(i)) & 1 { + case 1: + // Match everything with a zero bit here. + matched = matched.Union(zeroes) + predicate &^= 1 << uint(i) + case 0: + // Discard everything with a one bit here. + remaining = zeroes } } - return filter, nil + return matched, nil } func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (*Row, error) { + if predicate == -1 && !allowEquality { + predicate, allowEquality = 0, true + } + b := f.row(bsiExistsBit) // Create predicate without sign bit. - upredicate := uint64(predicate) - if predicate < 0 { - upredicate = uint64(-predicate) - } + upredicate := absInt64(predicate) - // If predicate is positive, return all positives greater than predicate. - if (predicate >= 0 && allowEquality) || (predicate >= -1 && !allowEquality) { - return f.rangeGTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) - } + sign := f.row(bsiSignBit) - // If predicate is negative, return all negatives less than than upredicate and all positives. - neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicate, allowEquality) - if err != nil { - return nil, err + switch { + case predicate == 0 && !allowEquality: + // Match all positive numbers except zero. + nonzero, err := f.rangeNEQ(bitDepth, 0) + if err != nil { + return nil, err + } + b = nonzero + fallthrough + case predicate == 0 && allowEquality: + // Match all positive numbers. + return b.Difference(sign), nil + case predicate >= 0: + // Match all positive numbers greater than the predicate. + return f.rangeGTUnsigned(b.Difference(sign), bitDepth, upredicate, allowEquality) + default: + // Match all positives and greater negatives. + neg, err := f.rangeLTUnsigned(b.Intersect(sign), bitDepth, upredicate, allowEquality) + if err != nil { + return nil, err + } + pos := b.Difference(sign) + return pos.Union(neg), nil } - pos := b.Difference(f.row(bsiSignBit)) - return pos.Union(neg), nil } func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - keep := NewRow() - // Filter any bits that don't match the current bit value. - for i := int(bitDepth - 1); i >= 0; i-- { + switch { + case predicate == 0 && allowEquality: + // This query matches all possible values. + return filter, nil + case predicate == 0 && !allowEquality: + // This query matches everything that is not 0. + matches := NewRow() + for i := uint(0); i < bitDepth; i++ { + row := f.row(uint64(bsiOffsetBit + i)) + matches = matches.Union(filter.Intersect(row)) + } + return matches, nil + case allowEquality: + predicate-- + } + + // Compare intermediate bits. + matched := NewRow() + remaining := filter + predicate |= (^uint64(0)) << bitDepth + for i := int(bitDepth - 1); i >= 0 && predicate < ^uint64(0) && remaining.Any(); i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit := (predicate >> uint(i)) & 1 - - // Handle last bit differently. - // If bit is one then return only already kept columns. - // If bit is zero then remove any unset columns. - if i == 0 && !allowEquality { - if bit == 1 { - return keep, nil - } - return filter.Difference(filter.Difference(row, keep)), nil - } - - // If bit is set then remove all unset columns not already kept. - if bit == 1 { - filter = filter.Difference(filter.Difference(row, keep)) - continue - } - - // If bit is unset then add columns with set bit to keep. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep = keep.Union(filter.Intersect(row)) + ones := remaining.Intersect(row) + switch (predicate >> uint(i)) & 1 { + case 1: + // Discard everything with a zero bit here. + remaining = ones + case 0: + // Match everything with a one bit here. + matched = matched.Union(ones) + predicate |= 1 << uint(i) } } - return filter, nil + return matched, nil } // notNull returns the exists row. @@ -1353,73 +1386,65 @@ func (f *fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax int64) b := f.row(bsiExistsBit) // Convert predicates to unsigned values. - upredicateMin, upredicateMax := uint64(predicateMin), uint64(predicateMax) - if predicateMin < 0 { - upredicateMin = uint64(-predicateMin) - } - if predicateMax < 0 { - upredicateMax = uint64(-predicateMax) - } + upredicateMin, upredicateMax := absInt64(predicateMin), absInt64(predicateMax) - // Handle positive-only values. - if predicateMin >= 0 { + switch { + case predicateMin == predicateMax: + return f.rangeEQ(bitDepth, predicateMin) + case predicateMin >= 0: + // Handle positive-only values. return f.rangeBetweenUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMin, upredicateMax) - } - - // Handle negative-only values. Swap unsigned min/max predicates. - if predicateMax < 0 { + case predicateMax < 0: + // Handle negative-only values. Swap unsigned min/max predicates. return f.rangeBetweenUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMax, upredicateMin) + default: + // If predicate crosses positive/negative boundary then handle separately and union. + pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) + if err != nil { + return nil, err + } + neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) + if err != nil { + return nil, err + } + return pos.Union(neg), nil } - - // If predicate crosses positive/negative boundary then handle separately and union. - pos, err := f.rangeLTUnsigned(b.Difference(f.row(bsiSignBit)), bitDepth, upredicateMax, true) - if err != nil { - return nil, err - } - neg, err := f.rangeLTUnsigned(b.Intersect(f.row(bsiSignBit)), bitDepth, upredicateMin, true) - if err != nil { - return nil, err - } - return pos.Union(neg), nil } // rangeBetweenUnsigned returns BSI columns for a range of values. Disregards the sign bit. func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - keep1 := NewRow() // GTE - keep2 := NewRow() // LTE + switch { + case predicateMax > (1<= 0; i-- { + // Compare any upper bits which are equal. + firstDiff := int(msb(predicateMax^predicateMin)) - 1 + remaining := filter + for i := int(bitDepth - 1); i > firstDiff; i-- { row := f.row(uint64(bsiOffsetBit + i)) - bit1 := (predicateMin >> uint(i)) & 1 - bit2 := (predicateMax >> uint(i)) & 1 - - // GTE predicateMin - // If bit is set then remove all unset columns not already kept. - if bit1 == 1 { - filter = filter.Difference(filter.Difference(row, keep1)) - } else { - // If bit is unset then add columns with set bit to keep. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep1 = keep1.Union(filter.Intersect(row)) - } - } - - // LTE predicateMax - // If bit is zero then remove all set bits not in excluded bitmap. - if bit2 == 0 { - filter = filter.Difference(row.Difference(keep2)) - } else { - // If bit is set then add columns for set bits to exclude. - // Don't bother to compute this on the final iteration. - if i > 0 { - keep2 = keep2.Union(filter.Difference(row)) - } + switch (predicateMin >> uint(i)) & 1 { + case 1: + remaining = remaining.Intersect(row) + case 0: + remaining = remaining.Difference(row) } } - return filter, nil + var err error + remaining, err = f.rangeGTUnsigned(remaining, uint(firstDiff+1), predicateMin, true) + if err != nil { + return nil, err + } + remaining, err = f.rangeLTUnsigned(remaining, uint(firstDiff+1), predicateMax, true) + if err != nil { + return nil, err + } + return remaining, nil } // pos translates the row ID and column ID into a position in the storage bitmap. @@ -2607,14 +2632,14 @@ func filterWithRows(rows []uint64) rowFilter { // returning done == true will cause processing to stop after all filters for // this container have been processed. The rows accumulated up to this point // (including this row if all filters passed) will be returned. -func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) rows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { f.mu.RLock() defer f.mu.RUnlock() - return f.unprotectedRows(start, filters...) + return f.unprotectedRows(ctx, start, filters...) } // unprotectedRows calls rows without grabbing the mutex. -func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64 { +func (f *fragment) unprotectedRows(ctx context.Context, start uint64, filters ...rowFilter) []uint64 { startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) @@ -2622,6 +2647,10 @@ func (f *fragment) unprotectedRows(start uint64, filters ...rowFilter) []uint64 // Loop over the existing containers. for i.Next() { + // caller doesn't need a result anymore. + if ctx.Err() != nil { + return nil + } key, c := i.Value() // virtual row for the current container @@ -2861,7 +2890,7 @@ type setRowIterator struct { func (f *fragment) setRowIterator(wrap bool, filters ...rowFilter) rowIterator { return &setRowIterator{ f: f, - rowIDs: f.rows(0, filters...), // TODO: this may be memory intensive in high cardinality cases + rowIDs: f.rows(context.Background(), 0, filters...), // TODO: this may be memory intensive in high cardinality cases wrap: wrap, } } @@ -3282,7 +3311,7 @@ func newRowsVector(f *fragment) *rowsVector { // otherwise it returns false. Ensure that you already // have the mutex before calling this. func (v *rowsVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(0, filterColumn(colID)) + rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { @@ -3316,7 +3345,7 @@ func newBoolVector(f *fragment) *boolVector { // otherwise it returns false. Ensure that you already // have the fragment mutex before calling this. func (v *boolVector) Get(colID uint64) (uint64, bool, error) { - rows := v.f.unprotectedRows(0, filterColumn(colID)) + rows := v.f.unprotectedRows(context.Background(), 0, filterColumn(colID)) if len(rows) > 1 { return 0, false, errors.New("found multiple row values for column") } else if len(rows) == 1 { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 3394825f9..5912eb132 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -111,6 +111,11 @@ func TestFragment_ClearBit(t *testing.T) { func TestFragment_RowcacheMap(t *testing.T) { var done int64 f := mustOpenFragment("i", "f", viewStandard, 0, "") + // Under -race, this test turns out to take a fairly long time + // to run with larger OpN, because we write 50,000 bits to + // the bitmap, and everything is being race-detected, and we don't + // actually need that many to get the result we care about. + f.MaxOpN = 2000 defer f.Clean(t) ch := make(chan struct{}) @@ -619,6 +624,23 @@ func TestFragment_Range(t *testing.T) { } }) + t.Run("LTMaxRegression", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + if _, err := f.setValue(1, 2, 3); err != nil { + t.Fatal(err) + } else if _, err := f.setValue(2, 2, 0); err != nil { + t.Fatal(err) + } + + if b, err := f.rangeLTUnsigned(NewRow(1, 2), 2, 3, false); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { + t.Fatalf("unepxected coulmns: %+v", b.Columns()) + } + }) + t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) @@ -667,6 +689,23 @@ func TestFragment_Range(t *testing.T) { } }) + t.Run("GTMinRegression", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + if _, err := f.setValue(1, 2, 0); err != nil { + t.Fatal(err) + } else if _, err := f.setValue(2, 2, 1); err != nil { + t.Fatal(err) + } + + if b, err := f.rangeGTUnsigned(NewRow(1, 2), 2, 0, false); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2}) { + t.Fatalf("unepxected coulmns: %+v", b.Columns()) + } + }) + t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") defer f.Clean(t) @@ -2412,8 +2451,9 @@ func TestGetZipfRowsSliceRoaring(t *testing.T) { if err != nil { t.Fatalf("importing roaring: %v", err) } - if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { - t.Fatalf("unexpected rows: %v", f.rows(0)) + rows := f.rows(context.Background(), 0) + if !reflect.DeepEqual(rows, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { + t.Fatalf("unexpected rows: %v", rows) } for i := uint64(1); i < 10; i++ { if f.row(i).Count() >= f.row(i-1).Count() { @@ -2714,12 +2754,12 @@ func TestFragment_RowsIteration(t *testing.T) { } } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expectedAll, ids) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids = f.rows(0, filterColumn(1)) + ids = f.rows(context.Background(), 0, filterColumn(1)) if !reflect.DeepEqual(expectedOdd, ids) { t.Fatalf("Do not match %v %v", expectedOdd, ids) } @@ -2738,12 +2778,12 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatal(err) } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } - ids = f.rows(0, filterColumn(66000)) + ids = f.rows(context.Background(), 0, filterColumn(66000)) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } @@ -2754,18 +2794,18 @@ func TestFragment_RowsIteration(t *testing.T) { defer f.Clean(t) expectedRows := make([]uint64, 0) - for r := uint64(1); r < uint64(10000); r += 100 { + for r := uint64(1); r < uint64(10000); r += 250 { expectedRows = append(expectedRows, r) - for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 { + for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) { if _, err := f.setBit(r, c); err != nil { t.Fatal(err) } - ids := f.rows(0) + ids := f.rows(context.Background(), 0) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids = f.rows(0, filterColumn(c)) + ids = f.rows(context.Background(), 0, filterColumn(c)) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } diff --git a/generator/slice.go b/generator/slice.go new file mode 100644 index 000000000..595f5a132 --- /dev/null +++ b/generator/slice.go @@ -0,0 +1,51 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generator + +import ( + "math/rand" + "sort" +) + +// Uint64Slice generates between [0, n) random uint64 numbers between min and max. +func Uint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 { + a := make([]uint64, rand.Intn(n)) + for i := range a { + a[i] = min + uint64(rand.Int63n(int64(max-min))) + } + + if sorted { + sort.Sort(uint64Slice(a)) + } + + return a +} + +// Uint64SetSlice returns the values in a uint64 set. +func Uint64SetSlice(m map[uint64]struct{}) []uint64 { + a := make([]uint64, 0, len(m)) + for v := range m { + a = append(a, v) + } + sort.Sort(uint64Slice(a)) + return a +} + +// uint64Slice represents a sortable slice of uint64 numbers. +type uint64Slice []uint64 + +func (u uint64Slice) Swap(i, j int) { u[i], u[j] = u[j], u[i] } +func (u uint64Slice) Len() int { return len(u) } +func (u uint64Slice) Less(i, j int) bool { return u[i] < u[j] } diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index cb526ca4d..b251c39a4 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -15,6 +15,7 @@ package gopsutil import ( + "math" "runtime" "strings" @@ -114,6 +115,13 @@ func (s *systemInfo) collectPlatformInfo() error { } s.cpuModel = infos[0].ModelName s.cpuMHz = computeMHz(s.cpuModel) + if s.cpuMHz < 0 { + s.cpuMHz = int(math.Round(infos[0].Mhz)) + } + if s.cpuMHz < 0 { + // This is supposed to be unsigned. + s.cpuMHz = 0 + } // gopsutil reports core and clock speed info inconsistently // by OS diff --git a/gossip/gossip.go b/gossip/gossip.go index 84f914975..975c57a19 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -324,16 +324,20 @@ func (g *memberSet) LocalState(join bool) []byte { Schema: &pilosa.Schema{Indexes: g.papi.Schema(context.Background())}, } for _, idx := range m.Schema.Indexes { - is := &pilosa.IndexStatus{Name: idx.Name} + is := &pilosa.IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} + for _, f := range idx.Fields { availableShards := roaring.NewBitmap() if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() } - is.Fields = append(is.Fields, &pilosa.FieldStatus{ + + fs := &pilosa.FieldStatus{ Name: f.Name, + CreatedAt: f.CreatedAt, AvailableShards: availableShards, - }) + } + is.Fields = append(is.Fields, fs) } m.Indexes = append(m.Indexes, is) } diff --git a/handler.go b/handler.go index 3c40d20ad..17156245b 100644 --- a/handler.go +++ b/handler.go @@ -114,8 +114,10 @@ var NopHandler Handler = nopHandler{} // ImportValueRequest describes the import request structure // for a value (BSI) import. type ImportValueRequest struct { - Index string - Field string + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 @@ -141,6 +143,11 @@ func (ivr *ImportValueRequest) Swap(i, j int) { // Validate ensures that the payload of the request is valid. func (ivr *ImportValueRequest) Validate() error { + return ivr.ValidateWithTimestamp(ivr.IndexCreatedAt, ivr.FieldCreatedAt) +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if ivr.Index == "" || ivr.Field == "" { return errors.Errorf("index and field required, but got '%s' and '%s'", ivr.Index, ivr.Field) } @@ -160,30 +167,48 @@ func (ivr *ImportValueRequest) Validate() error { if valueSetCount > 1 { return errors.Errorf("must pass ints, floats, or strings but not multiple") } + if ivr.IndexCreatedAt != 0 && ivr.FieldCreatedAt != 0 { + if ivr.IndexCreatedAt != indexCreatedAt || ivr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } return nil } // ImportColumnAttrsRequest describes the import request structure // for a ColumnAttr import type ImportColumnAttrsRequest struct { - AttrKey string - ColumnIDs []uint64 - AttrVals []string - Shard int64 - Index string + AttrKey string + ColumnIDs []uint64 + AttrVals []string + Shard int64 + Index string + IndexCreatedAt int64 } // ImportRequest describes the import request structure // for an import. type ImportRequest struct { - Index string - Field string - Shard uint64 - RowIDs []uint64 - ColumnIDs []uint64 - RowKeys []string - ColumnKeys []string - Timestamps []int64 + Index string + IndexCreatedAt int64 + Field string + FieldCreatedAt int64 + Shard uint64 + RowIDs []uint64 + ColumnIDs []uint64 + RowKeys []string + ColumnKeys []string + Timestamps []int64 +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if ir.IndexCreatedAt != 0 && ir.FieldCreatedAt != 0 { + if ir.IndexCreatedAt != indexCreatedAt || ir.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } const ( @@ -195,10 +220,22 @@ const ( // ImportRoaringRequest describes the import request structure // for an import containing roaring-encoded data. type ImportRoaringRequest struct { - Clear bool - Action string // [set, clear, overwrite] - Block int - Views map[string][]byte + IndexCreatedAt int64 + FieldCreatedAt int64 + Clear bool + Action string // [set, clear, overwrite] + Block int + Views map[string][]byte +} + +// ValidateWithTimestamp ensures that the payload of the request is valid. +func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { + if irr.IndexCreatedAt != 0 && irr.FieldCreatedAt != 0 { + if irr.IndexCreatedAt != indexCreatedAt || irr.FieldCreatedAt != fieldCreatedAt { + return ErrPreconditionFailed + } + } + return nil } // ImportResponse is the structured response of an import. diff --git a/holder.go b/holder.go index ad52ee082..de3cd0701 100644 --- a/holder.go +++ b/holder.go @@ -234,7 +234,13 @@ func (h *Holder) Open() error { return errors.Wrap(err, "opening index") } - if err := index.Open(); err != nil { + if h.isCoordinator() { + index.createdAt = timestamp() + err = index.OpenWithTimestamp() + } else { + err = index.Open() + } + if err != nil { if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue @@ -380,11 +386,16 @@ func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{ - Name: index.Name(), - Options: index.Options(), + Name: index.Name(), + CreatedAt: index.CreatedAt(), + Options: index.Options(), } for _, field := range index.Fields() { - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), + } for _, view := range field.views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } @@ -404,6 +415,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{ Name: index.Name(), + CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, } @@ -411,7 +423,11 @@ func (h *Holder) limitedSchema() []*IndexInfo { if strings.HasPrefix(field.name, "_") { continue } - fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + fi := &FieldInfo{ + Name: field.Name(), + CreatedAt: field.CreatedAt(), + Options: field.Options(), + } di.Fields = append(di.Fields, fi) } sort.Sort(fieldInfoSlice(di.Fields)) @@ -424,20 +440,32 @@ func (h *Holder) limitedSchema() []*IndexInfo { // applySchema applies an internal Schema to Holder. func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. - for _, index := range schema.Indexes { - idx, err := h.CreateIndexIfNotExists(index.Name, index.Options) + for _, i := range schema.Indexes { + idx, err := h.CreateIndexIfNotExists(i.Name, i.Options) if err != nil { return errors.Wrap(err, "creating index") } + if i.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = i.CreatedAt + idx.mu.Unlock() + } + // Create fields that don't exist. - for _, f := range index.Fields { - field, err := idx.createFieldIfNotExists(f.Name, &f.Options) + for _, f := range i.Fields { + fld, err := idx.createFieldIfNotExists(f.Name, &f.Options) if err != nil { return errors.Wrap(err, "creating field") } + if f.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = f.CreatedAt + fld.mu.Unlock() + } + // Create views that don't exist. for _, v := range f.Views { - _, err := field.createViewIfNotExists(v.Name) + _, err := fld.createViewIfNotExists(v.Name) if err != nil { return errors.Wrap(err, "creating view") } @@ -447,6 +475,32 @@ func (h *Holder) applySchema(schema *Schema) error { return nil } +func (h *Holder) applyCreatedAt(indexes []*IndexInfo) { + for _, ii := range indexes { + idx := h.Index(ii.Name) + if idx == nil { + continue + } + if ii.CreatedAt != 0 { + idx.mu.Lock() + idx.createdAt = ii.CreatedAt + idx.mu.Unlock() + } + + for _, fi := range ii.Fields { + fld := idx.Field(fi.Name) + if fld == nil { + continue + } + if fi.CreatedAt != 0 { + fld.mu.Lock() + fld.createdAt = fi.CreatedAt + fld.mu.Unlock() + } + } + } +} + // IndexPath returns the path where a given index is stored. func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } @@ -652,6 +706,13 @@ func (h *Holder) recalculateCaches() { } } +func (h *Holder) isCoordinator() bool { + if s, ok := h.broadcaster.(*Server); ok { + return s.isCoordinator + } + return false +} + // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { oldLimit := &syscall.Rlimit{} @@ -1059,7 +1120,8 @@ func (s *holderSyncer) stopTranslationSync() error { // writing new translation keys. Index stores are writable if the node owns the // partition. Field stores are writable if the node is the coordinator. func (s *holderSyncer) setTranslateReadOnlyFlags() { - isCoordinator := s.Cluster.isCoordinator() + s.Cluster.mu.RLock() + isCoordinator := s.Cluster.unprotectedIsCoordinator() for _, index := range s.Holder.Indexes() { // There is a race condition here: @@ -1079,7 +1141,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() { // done using it. index.mu.RLock() for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ { - ownsPartition := s.Cluster.ownsPartition(s.Node.ID, partitionID) + ownsPartition := s.Cluster.unprotectedOwnsPartition(s.Node.ID, partitionID) if ts := index.TranslateStore(partitionID); ts != nil { ts.SetReadOnly(!ownsPartition) } @@ -1090,6 +1152,7 @@ func (s *holderSyncer) setTranslateReadOnlyFlags() { field.TranslateStore().SetReadOnly(!isCoordinator) } } + s.Cluster.mu.RUnlock() } // initializeIndexTranslateReplication connects to each node that is the diff --git a/http/handler.go b/http/handler.go index 5b9eaf405..f6e0ac001 100644 --- a/http/handler.go +++ b/http/handler.go @@ -15,14 +15,15 @@ package http import ( + "bytes" "context" "crypto/tls" "encoding/json" "expvar" "fmt" "io" - "io/ioutil" "math" + "mime" "net" "net/http" _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. @@ -358,6 +359,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/transaction/{id}", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/{id}/finish", handler.handlePostFinishTransaction).Methods("POST").Name("PostFinishTransaction") router.HandleFunc("/transactions", handler.handleGetTransactions).Methods("GET").Name("GetTransactions") + router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") // /internal endpoints are for internal use only; they may change at any time. @@ -401,9 +403,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // successResponse is a general success/error struct for http responses. type successResponse struct { - h *Handler - Success bool `json:"success"` - Error *Error `json:"error,omitempty"` + h *Handler + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + Error *Error `json:"error,omitempty"` } // check determines success or failure based on the error. @@ -473,9 +477,33 @@ func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) { // headers are present, but none of them are "application/json" // (or any matching wildcard). Otherwise returns true. func validHeaderAcceptJSON(header http.Header) bool { + return validHeaderAcceptType(header, "application", "json") +} + +func validHeaderAcceptType(header http.Header, typ, subtyp string) bool { if v, found := header["Accept"]; found { for _, v := range v { - if v == "application/json" || v == "*/*" || v == "*/json" || v == "application/*" { + t, _, err := mime.ParseMediaType(v) + if err != nil { + switch err { + case mime.ErrInvalidMediaParameter: + // This is an optional feature, so we can keep going anyway. + default: + continue + } + } + spl := strings.SplitN(t, "/", 2) + if len(spl) < 2 { + continue + } + switch { + case spl[0] == typ && spl[1] == subtyp: + return true + case spl[0] == "*" && spl[1] == subtyp: + return true + case spl[0] == typ && spl[1] == "*": + return true + case spl[0] == "*" && spl[1] == "*": return true } } @@ -504,7 +532,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") schema := h.api.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(map[string]interface{}{"indexes": schema}); err != nil { // TODO: use pilosa.Schema instead of map[string]interface{} here? + if err := json.NewEncoder(w).Encode(pilosa.Schema{Indexes: schema}); err != nil { h.logger.Printf("write schema response error: %s", err) } } @@ -773,7 +801,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: indexName} // Decode request. req := postIndexRequest{ @@ -787,8 +815,15 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - _, err = h.api.CreateIndex(r.Context(), indexName, req.Options) + index, err := h.api.CreateIndex(r.Context(), indexName, req.Options) + if index != nil { + resp.CreatedAt = index.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + if index, _ = h.api.Index(r.Context(), indexName); index != nil { + resp.CreatedAt = index.CreatedAt() + } + } resp.write(w, err) } @@ -826,6 +861,52 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request } } +func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) { + var rtype string + switch { + case validHeaderAcceptType(r.Header, "text", "plain"): + rtype = "text/plain" + case validHeaderAcceptJSON(r.Header): + rtype = "application/json" + default: + http.Error(w, "no acceptable response type selected", http.StatusNotAcceptable) + return + } + queries, err := h.api.ActiveQueries(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", rtype) + switch rtype { + case "text/plain": + durations := make([]string, len(queries)) + for i, q := range queries { + durations[i] = q.Age.String() + } + var maxlen int + for _, l := range durations { + if len(l) > maxlen { + maxlen = len(l) + } + } + for i, q := range queries { + _, err := fmt.Fprintf(w, "%*s%q\n", -(maxlen + 2), durations[i], q.Query) + if err != nil { + h.logger.Printf("sending GetActiveQueries response: %s", err) + return + } + } + if _, err := w.Write([]byte{'\n'}); err != nil { + h.logger.Printf("sending GetActiveQueries response: %s", err) + } + case "application/json": + if err := json.NewEncoder(w).Encode(queries); err != nil { + h.logger.Printf("encoding GetActiveQueries response: %s", err) + } + } +} + type postIndexAttrDiffRequest struct { Blocks []pilosa.AttrBlock `json:"blocks"` } @@ -853,7 +934,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { return } - resp := successResponse{h: h} + resp := successResponse{h: h, Name: fieldName} // Decode request. var req postFieldRequest @@ -925,11 +1006,18 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldForeignIndex(*req.Options.ForeignIndex)) } - _, err = h.api.CreateField(r.Context(), indexName, fieldName, fos...) + field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) if _, ok := err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } + if field != nil { + resp.CreatedAt = field.CreatedAt() + } else if _, ok = errors.Cause(err).(pilosa.ConflictError); ok { + if field, _ = h.api.Field(r.Context(), indexName, fieldName); field != nil { + resp.CreatedAt = field.CreatedAt() + } + } resp.write(w, err) } @@ -1240,7 +1328,7 @@ func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error // readProtobufQueryRequest parses query parameters in protobuf from r. func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { // Slurp the body. - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1258,7 +1346,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er q := r.URL.Query() // Parse query string. - buf, err := ioutil.ReadAll(r.Body) + buf, err := readBody(r) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1319,95 +1407,14 @@ func (h *Handler) writeJSONQueryResponse(w io.Writer, resp *pilosa.QueryResponse return json.NewEncoder(w).Encode(resp) } -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. +func validateProtobufHeader(r *http.Request) (error string, code int) { if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return + return "Unsupported media type", http.StatusUnsupportedMediaType } - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - - // If the clear flag is true, treat the import as clear bits. - q := r.URL.Query() - doClear := q.Get("clear") == "true" - doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" - - opts := []pilosa.ImportOption{ - pilosa.OptImportOptionsClear(doClear), - pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), - } - - // Get index and field type to determine how to handle the - // import data. - field, err := h.api.Field(r.Context(), indexName, fieldName) - if err != nil { - switch errors.Cause(err) { - case pilosa.ErrIndexNotFound: - fallthrough - case pilosa.ErrFieldNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal request based on field type. - if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { - // Field type: Int - // Marshal into request object. - req := &pilosa.ImportValueRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } else { - // Field type: set, time, mutex - // Marshal into request object. - req := &pilosa.ImportRequest{} - if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.api.Import(r.Context(), req, opts...); err != nil { - switch errors.Cause(err) { - case pilosa.ErrClusterDoesNotOwnShard: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - } - - // Write response. - _, err = w.Write(importOk) - if err != nil { - h.logger.Printf("writing import response: %v", err) + if r.Header.Get("Accept") != "application/x-protobuf" { + return "Not acceptable", http.StatusNotAcceptable } + return } // handleGetExport handles /export requests. @@ -1885,6 +1892,95 @@ func GetHTTPClient(t *tls.Config) *http.Client { return &http.Client{Transport: transport} } +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + // Get index and field type to determine how to handle the + // import data. + indexName := mux.Vars(r)["index"] + index, err := h.api.Index(r.Context(), indexName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + fieldName := mux.Vars(r)["field"] + field := index.Field(fieldName) + if field == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // If the clear flag is true, treat the import as clear bits. + q := r.URL.Query() + doClear := q.Get("clear") == "true" + doIgnoreKeyCheck := q.Get("ignoreKeyCheck") == "true" + + opts := []pilosa.ImportOption{ + pilosa.OptImportOptionsClear(doClear), + pilosa.OptImportOptionsIgnoreKeyCheck(doIgnoreKeyCheck), + } + + // Read entire body. + body, err := readBody(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Unmarshal request based on field type. + if field.Type() == pilosa.FieldTypeInt || field.Type() == pilosa.FieldTypeDecimal { + // Field type: Int + // Marshal into request object. + req := &pilosa.ImportValueRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.ImportValue(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } else { + // Field type: set, time, mutex + // Marshal into request object. + req := &pilosa.ImportRequest{} + if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.api.Import(r.Context(), req, opts...); err != nil { + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + } + + // Write response. + _, err = w.Write(importOk) + if err != nil { + h.logger.Printf("writing import response: %v", err) + } +} + // handlePostImportColumnAttrs func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. @@ -1898,7 +1994,7 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req opts := []pilosa.ImportOption{} - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1911,7 +2007,12 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + switch errors.Cause(err) { + case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1922,16 +2023,16 @@ func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Req } } -// handlPostRoaringImport +// handlePostImportRoaring func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) return } + + // Get index and field type to determine how to handle the + // import data. indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] @@ -1946,7 +2047,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request // Read entire body. span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") - body, err := ioutil.ReadAll(r.Body) + body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() if err != nil { @@ -1976,6 +2077,10 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { w.WriteHeader(http.StatusBadRequest) + } else if _, ok := err.(pilosa.NotFoundError); ok { + w.WriteHeader(http.StatusNotFound) + } else if _, ok := err.(pilosa.PreconditionFailedError); ok { + w.WriteHeader(http.StatusPreconditionFailed) } else { w.WriteHeader(http.StatusInternalServerError) } @@ -2040,3 +2145,18 @@ func (h *Handler) handlePostTranslateIDs(w http.ResponseWriter, r *http.Request) h.logger.Printf("writing translate keys response: %v", err) } } + +// Read entire request body. +func readBody(r *http.Request) ([]byte, error) { + var contentLength int64 = bytes.MinRead + if r.ContentLength > 0 { + contentLength = r.ContentLength + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+contentLength)) + if _, err := buf.ReadFrom(r.Body); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/index.go b/index.go index a5f143bfc..481d22cf4 100644 --- a/index.go +++ b/index.go @@ -36,10 +36,11 @@ import ( // Index represents a container for fields. type Index struct { - mu sync.RWMutex - path string - name string - keys bool // use string keys + mu sync.RWMutex + createdAt int64 + path string + name string + keys bool // use string keys // Existence tracking. trackExistence bool @@ -103,6 +104,13 @@ func NewIndex(path, name string, partitionN int) (*Index, error) { }, nil } +// CreatedAt is an timestamp for a specific version of an index. +func (i *Index) CreatedAt() int64 { + i.mu.RLock() + defer i.mu.RUnlock() + return i.createdAt +} + // Name returns name of the index. func (i *Index) Name() string { return i.name } @@ -140,7 +148,12 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open() (err error) { +func (i *Index) Open() error { return i.open(false) } + +// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. +func (i *Index) OpenWithTimestamp() error { return i.open(true) } + +func (i *Index) open(withTimestamp bool) (err error) { // Ensure the path exists. i.logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -154,7 +167,7 @@ func (i *Index) Open() (err error) { } i.logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(); err != nil { + if err := i.openFields(withTimestamp); err != nil { return errors.Wrap(err, "opening fields") } @@ -197,7 +210,7 @@ func (i *Index) Open() (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields() error { +func (i *Index) openFields(withTimestamp bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -229,6 +242,9 @@ fileLoop: i.logger.Debugf("open field: %s", fi.Name()) mu.Lock() fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if withTimestamp { + fld.createdAt = timestamp() + } mu.Unlock() if err != nil { return errors.Wrapf(ErrName, "'%s'", fi.Name()) @@ -559,6 +575,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` diff --git a/internal/private.pb.go b/internal/private.pb.go index a1d0870bd..6dd3eb9bc 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -616,6 +616,7 @@ func (m *DeleteIndexMessage) GetIndex() string { type CreateIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -668,10 +669,18 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } +func (m *CreateIndexMessage) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type CreateFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta,proto3" json:"Meta,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -731,6 +740,13 @@ func (m *CreateFieldMessage) GetMeta() *FieldOptions { return nil } +func (m *CreateFieldMessage) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -853,6 +869,7 @@ type Field struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta,proto3" json:"Meta,omitempty"` Views []string `protobuf:"bytes,3,rep,name=Views,proto3" json:"Views,omitempty"` + CreatedAt int64 `protobuf:"varint,4,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -912,6 +929,13 @@ func (m *Field) GetViews() []string { return nil } +func (m *Field) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type Schema struct { Indexes []*Index `protobuf:"bytes,1,rep,name=Indexes,proto3" json:"Indexes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -961,6 +985,7 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` Options *IndexMeta `protobuf:"bytes,5,opt,name=Options,proto3" json:"Options,omitempty"` Fields []*Field `protobuf:"bytes,4,rep,name=Fields,proto3" json:"Fields,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1008,6 +1033,13 @@ func (m *Index) GetName() string { return "" } +func (m *Index) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + func (m *Index) GetOptions() *IndexMeta { if m != nil { return m.Options @@ -1090,6 +1122,7 @@ type Node struct { URI *URI `protobuf:"bytes,2,opt,name=URI,proto3" json:"URI,omitempty"` IsCoordinator bool `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"` State string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` + GRPCURI *URI `protobuf:"bytes,5,opt,name=GRPCURI,proto3" json:"GRPCURI,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1156,6 +1189,13 @@ func (m *Node) GetState() string { return "" } +func (m *Node) GetGRPCURI() *URI { + if m != nil { + return m.GRPCURI + } + return nil +} + type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -1332,6 +1372,7 @@ func (m *NodeStatus) GetIndexes() []*IndexStatus { type IndexStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Fields []*FieldStatus `protobuf:"bytes,2,rep,name=Fields,proto3" json:"Fields,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1384,9 +1425,17 @@ func (m *IndexStatus) GetFields() []*FieldStatus { return nil } +func (m *IndexStatus) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type FieldStatus struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` AvailableShards []uint64 `protobuf:"varint,2,rep,packed,name=AvailableShards,proto3" json:"AvailableShards,omitempty"` + CreatedAt int64 `protobuf:"varint,3,opt,name=CreatedAt,proto3" json:"CreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1439,10 +1488,18 @@ func (m *FieldStatus) GetAvailableShards() []uint64 { return nil } +func (m *FieldStatus) GetCreatedAt() int64 { + if m != nil { + return m.CreatedAt + } + return 0 +} + type ClusterStatus struct { ClusterID string `protobuf:"bytes,1,opt,name=ClusterID,proto3" json:"ClusterID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Nodes []*Node `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Schema *Schema `protobuf:"bytes,4,opt,name=Schema,proto3" json:"Schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1502,6 +1559,13 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } +func (m *ClusterStatus) GetSchema() *Schema { + if m != nil { + return m.Schema + } + return nil +} + type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` @@ -2413,95 +2477,99 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1395 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x17, 0xcd, 0x72, 0xdb, 0x44, - 0x18, 0x59, 0x8e, 0x63, 0x7f, 0x8e, 0x53, 0x67, 0xdb, 0xa6, 0x6a, 0x60, 0x82, 0x59, 0x3a, 0xd4, - 0x74, 0x86, 0xd0, 0x69, 0x99, 0xe1, 0xb7, 0x33, 0x6d, 0xe2, 0xb4, 0x98, 0x92, 0xb4, 0x5d, 0xa7, - 0xbd, 0x71, 0xd8, 0xc8, 0x3b, 0x8d, 0x26, 0xb2, 0x64, 0xa4, 0x55, 0xea, 0xf4, 0xc0, 0x15, 0x66, - 0x78, 0x01, 0x8e, 0xbc, 0x07, 0x2f, 0xc0, 0x91, 0x47, 0x60, 0xca, 0x53, 0x70, 0x63, 0xf6, 0xdb, - 0x5d, 0x49, 0x76, 0x1c, 0x52, 0x52, 0x6e, 0xfb, 0xfd, 0xff, 0x7f, 0x9f, 0x04, 0xad, 0x71, 0x12, - 0x1c, 0x71, 0x29, 0x36, 0xc6, 0x49, 0x2c, 0x63, 0x52, 0x0f, 0x22, 0x29, 0x92, 0x88, 0x87, 0x6b, - 0x4b, 0xe3, 0x6c, 0x3f, 0x0c, 0x7c, 0x8d, 0xa7, 0x0f, 0xa0, 0xd1, 0x8f, 0x86, 0x62, 0xb2, 0x23, - 0x24, 0x27, 0x04, 0xaa, 0x0f, 0xc5, 0x71, 0xea, 0xb9, 0x1d, 0xa7, 0x5b, 0x67, 0xf8, 0x26, 0x1f, - 0xc0, 0xf2, 0x5e, 0xc2, 0xfd, 0xc3, 0xed, 0x49, 0x90, 0x4a, 0x11, 0xf9, 0xc2, 0xab, 0x22, 0x75, - 0x06, 0x4b, 0x7f, 0x75, 0x61, 0xe9, 0x7e, 0x20, 0xc2, 0xe1, 0xa3, 0xb1, 0x0c, 0xe2, 0x28, 0x55, - 0xca, 0xf6, 0x8e, 0xc7, 0xc2, 0xab, 0x77, 0x9c, 0x6e, 0x83, 0xe1, 0x9b, 0xbc, 0x03, 0x8d, 0x2d, - 0xee, 0x1f, 0x08, 0x24, 0xb8, 0x48, 0x28, 0x10, 0x39, 0x75, 0x10, 0xbc, 0xd4, 0x56, 0x5a, 0xac, - 0x40, 0x90, 0x0e, 0x34, 0xf7, 0x82, 0x91, 0x78, 0x92, 0xf1, 0x48, 0x66, 0x23, 0x6f, 0x01, 0xa5, - 0xcb, 0x28, 0xb2, 0x0a, 0xb5, 0x47, 0xe1, 0x70, 0x27, 0x88, 0xbc, 0x46, 0xc7, 0xe9, 0xba, 0xcc, - 0x40, 0x16, 0xcf, 0x27, 0x1e, 0x14, 0x78, 0x3e, 0xc9, 0xc3, 0x6d, 0x4e, 0x87, 0xbb, 0x1b, 0x0f, - 0x24, 0x8f, 0x86, 0x3c, 0x19, 0x3e, 0x0b, 0xc4, 0x0b, 0x6f, 0x49, 0x87, 0x3b, 0x8d, 0x55, 0xb2, - 0x9b, 0x3c, 0x15, 0x5e, 0x0b, 0x35, 0xe2, 0x9b, 0xac, 0x41, 0x7d, 0x33, 0x90, 0x3d, 0x31, 0x96, - 0x07, 0xde, 0x72, 0xc7, 0xe9, 0x56, 0x59, 0x0e, 0x93, 0x4b, 0xb0, 0x30, 0xf0, 0x79, 0x28, 0xbc, - 0x0b, 0x28, 0xa0, 0x01, 0x42, 0x61, 0xe9, 0x7e, 0x9c, 0x88, 0xe0, 0x79, 0x84, 0x45, 0xf0, 0xda, - 0x18, 0xd4, 0x14, 0x8e, 0xbc, 0x0f, 0xae, 0x0a, 0x69, 0xa5, 0xe3, 0x74, 0x9b, 0xb7, 0x56, 0x36, - 0x6c, 0x1d, 0x37, 0x7a, 0xc2, 0x0f, 0x46, 0x3c, 0x64, 0x8a, 0x8a, 0x4c, 0x7c, 0xe2, 0x91, 0xd3, - 0x99, 0xf8, 0x84, 0x52, 0x58, 0xee, 0x8f, 0xc6, 0x71, 0x22, 0x99, 0x48, 0xc7, 0x71, 0x94, 0x0a, - 0xd2, 0x06, 0x77, 0x3b, 0x49, 0x3c, 0x07, 0xcd, 0xaa, 0x27, 0xfd, 0x01, 0xda, 0x9b, 0x61, 0xec, - 0x1f, 0xf6, 0xb8, 0xe4, 0x4c, 0x7c, 0x9f, 0x89, 0x54, 0x2a, 0xdf, 0xb5, 0x7b, 0x9a, 0x4f, 0x03, - 0x0a, 0x8b, 0xf5, 0xf6, 0x2a, 0x1a, 0x8b, 0x80, 0xca, 0x0b, 0x66, 0x4d, 0x97, 0x07, 0xdf, 0x18, - 0xfb, 0x01, 0x4f, 0x86, 0x58, 0xd3, 0x2a, 0xd3, 0x80, 0xc2, 0xa2, 0x25, 0xec, 0x83, 0x2a, 0xd3, - 0x00, 0xed, 0xc3, 0x4a, 0xc9, 0xbe, 0x71, 0x73, 0x15, 0x6a, 0x2c, 0x7e, 0xd1, 0xef, 0xa5, 0x9e, - 0xd3, 0x71, 0xbb, 0x55, 0x66, 0x20, 0x6c, 0x98, 0x38, 0xcc, 0x46, 0x91, 0x22, 0x55, 0x90, 0x54, - 0x20, 0xe8, 0x55, 0x58, 0xc0, 0xee, 0x51, 0x51, 0x16, 0xb2, 0xea, 0x49, 0x7f, 0x74, 0xa0, 0xb1, - 0xc3, 0x27, 0xe8, 0x48, 0x4a, 0xee, 0x40, 0xdd, 0xd6, 0x16, 0x99, 0x9a, 0xb7, 0xde, 0x2b, 0x32, - 0x98, 0xb3, 0x6d, 0x58, 0x9e, 0xed, 0x48, 0x26, 0xc7, 0x2c, 0x17, 0x59, 0xfb, 0x12, 0x5a, 0x53, - 0x24, 0x65, 0xef, 0x50, 0x1c, 0xdb, 0xac, 0x1e, 0x8a, 0x63, 0x15, 0xeb, 0x11, 0x0f, 0x33, 0x81, - 0xb9, 0xaa, 0x32, 0x0d, 0x7c, 0x51, 0xf9, 0xcc, 0xa1, 0xcf, 0x80, 0x6c, 0x25, 0x82, 0x4b, 0x81, - 0x46, 0x76, 0x44, 0x9a, 0xf2, 0xe7, 0xe2, 0xac, 0x8c, 0xbb, 0xe5, 0x8c, 0xe7, 0xd9, 0xad, 0x94, - 0xb2, 0x4b, 0x6f, 0x00, 0xe9, 0x89, 0x50, 0x48, 0x61, 0xa6, 0xfb, 0x5f, 0xf4, 0xd2, 0x81, 0xf5, - 0xe1, 0x6c, 0x5e, 0x72, 0x1d, 0xaa, 0x6a, 0x55, 0xa0, 0xb1, 0xe6, 0xad, 0x8b, 0x45, 0x9e, 0xf2, - 0x2d, 0xc2, 0x90, 0x81, 0x86, 0x56, 0x29, 0x7a, 0xf9, 0x9a, 0x81, 0x4d, 0xb5, 0xd2, 0x0d, 0x63, - 0xca, 0x45, 0x53, 0xab, 0x85, 0xa9, 0xf2, 0x9a, 0x31, 0xd6, 0xee, 0xda, 0x70, 0xcf, 0x6b, 0x8d, - 0xfa, 0xf0, 0xb6, 0xd6, 0x70, 0xef, 0x88, 0x07, 0x21, 0xdf, 0x0f, 0xff, 0x53, 0x45, 0xa6, 0x1c, - 0xf7, 0x60, 0x11, 0x65, 0xfb, 0x3d, 0xd3, 0xdb, 0x16, 0xa4, 0xdf, 0x41, 0x31, 0x26, 0xbb, 0x7c, - 0x24, 0x8c, 0x36, 0x7c, 0xe7, 0xf1, 0x56, 0xce, 0x8e, 0x57, 0x19, 0x56, 0xa3, 0xa5, 0x56, 0xb5, - 0xab, 0x0c, 0x23, 0x40, 0x6f, 0x43, 0x6d, 0xe0, 0x1f, 0x88, 0x11, 0x27, 0x1f, 0xc2, 0x22, 0x7a, - 0x28, 0x52, 0xd3, 0xd1, 0x17, 0x66, 0x2a, 0xc5, 0x2c, 0x9d, 0xa6, 0x26, 0xb2, 0xb9, 0x3e, 0x7d, - 0x04, 0x8b, 0xc6, 0x30, 0x4e, 0xf4, 0x29, 0x15, 0xb7, 0x3c, 0xe4, 0x3a, 0xd4, 0xd0, 0xd9, 0xd4, - 0xab, 0xce, 0x5a, 0x45, 0x3c, 0x33, 0x64, 0xba, 0x0d, 0xee, 0x53, 0xd6, 0x57, 0x83, 0x8d, 0x0e, - 0x5b, 0xa3, 0x06, 0x52, 0xae, 0x7c, 0x1d, 0xa7, 0xd2, 0xa4, 0x15, 0xdf, 0x0a, 0xf7, 0x38, 0x4e, - 0x24, 0xa6, 0xb4, 0xc5, 0xf0, 0x4d, 0x53, 0xa8, 0xee, 0xc6, 0x43, 0x41, 0x96, 0xa1, 0xd2, 0xef, - 0x19, 0x1d, 0x95, 0x7e, 0x8f, 0xbc, 0x8b, 0xea, 0x4d, 0x26, 0x5b, 0x85, 0x13, 0x4f, 0x59, 0x9f, - 0xa1, 0xe1, 0x6b, 0xd0, 0xea, 0xa7, 0x5b, 0x71, 0x9c, 0x0c, 0x83, 0x88, 0xcb, 0x38, 0x31, 0x27, - 0x6f, 0x1a, 0x89, 0xa3, 0x25, 0xb9, 0xd4, 0xc7, 0xa8, 0xc1, 0x34, 0x40, 0xef, 0x42, 0x5b, 0x19, - 0x45, 0xc0, 0xb6, 0xc7, 0x2a, 0xd4, 0x14, 0x2e, 0x77, 0xc2, 0x40, 0x85, 0x86, 0x4a, 0x59, 0xc3, - 0xb7, 0x5a, 0xc3, 0xf6, 0x91, 0x88, 0x64, 0xa9, 0xc1, 0x10, 0x46, 0x05, 0x2d, 0xa6, 0x01, 0x42, - 0x75, 0x80, 0x26, 0x92, 0xe5, 0x22, 0x12, 0x85, 0x65, 0x48, 0xa3, 0x3f, 0x3b, 0x00, 0xd6, 0xa1, - 0x2c, 0xcd, 0x45, 0x9c, 0xd3, 0x45, 0x48, 0xd7, 0x36, 0x8a, 0x19, 0xae, 0x76, 0xc1, 0xa5, 0xf1, - 0xcc, 0x36, 0xd2, 0xc7, 0x45, 0x23, 0xe9, 0x92, 0x5e, 0x9e, 0x69, 0x00, 0x6d, 0xb5, 0x68, 0xa7, - 0xc7, 0xd0, 0x2c, 0xe1, 0x4f, 0x69, 0x2a, 0xdb, 0x25, 0x95, 0x59, 0x95, 0x88, 0x37, 0x2a, 0x6d, - 0xaf, 0x3c, 0x84, 0x66, 0x09, 0x3d, 0x57, 0x63, 0x17, 0x2e, 0x4c, 0x8f, 0xad, 0x3d, 0x07, 0xb3, - 0x68, 0x1a, 0x40, 0x6b, 0x2b, 0xcc, 0x52, 0x29, 0x12, 0xa3, 0x4e, 0xdd, 0x10, 0x8d, 0xc8, 0x8b, - 0x57, 0x20, 0xe6, 0xd7, 0x8f, 0x5c, 0x83, 0x05, 0x95, 0x46, 0x3d, 0x7d, 0x27, 0x73, 0xac, 0x89, - 0xf4, 0x19, 0xd4, 0x37, 0x07, 0xfd, 0x07, 0x49, 0x9c, 0x8d, 0xe7, 0x3a, 0x6d, 0x3f, 0x90, 0x2a, - 0xa5, 0x0f, 0xa4, 0xb6, 0x3e, 0xf6, 0x2e, 0x7e, 0x24, 0xe0, 0x65, 0x6f, 0xeb, 0xcb, 0x5e, 0x35, - 0x18, 0xae, 0xd6, 0xf5, 0x8a, 0xde, 0xac, 0x6a, 0xe8, 0xcf, 0xb3, 0x9f, 0xec, 0x8d, 0x76, 0x8b, - 0x1b, 0xad, 0x94, 0xea, 0xf5, 0xf7, 0x7f, 0x2a, 0xfd, 0xbb, 0x02, 0x2b, 0x4c, 0xa4, 0xc1, 0x4b, - 0xd1, 0x8f, 0x52, 0x99, 0x64, 0xbe, 0xda, 0x12, 0x4a, 0xfe, 0x9b, 0x78, 0xdf, 0x64, 0xdb, 0x65, - 0x1a, 0x78, 0x9d, 0x4e, 0x27, 0x37, 0xa1, 0x39, 0x3b, 0xb3, 0x27, 0x59, 0xcb, 0x2c, 0xe4, 0x26, - 0x2c, 0x0e, 0xe2, 0x2c, 0xf1, 0xf3, 0xf6, 0x2d, 0xad, 0x55, 0xed, 0x99, 0x26, 0x33, 0xcb, 0x46, - 0x9e, 0x00, 0xd9, 0x4b, 0x78, 0x94, 0x86, 0x5c, 0x39, 0x6b, 0x85, 0xeb, 0xb3, 0x9f, 0x05, 0x25, - 0x9e, 0x29, 0x3d, 0x73, 0x84, 0xc9, 0x27, 0xe5, 0xf9, 0xf4, 0x16, 0xd1, 0xeb, 0x4b, 0xd3, 0x5e, - 0x9b, 0x96, 0x2f, 0xcf, 0xf1, 0x9d, 0x99, 0x4e, 0xf5, 0x6a, 0x28, 0x78, 0xa5, 0x10, 0x9c, 0x22, - 0xb3, 0x69, 0x6e, 0xfa, 0x93, 0x03, 0x4b, 0x65, 0xcf, 0x5e, 0x6b, 0x2f, 0xe4, 0x05, 0xaf, 0x9c, - 0xfd, 0xdd, 0x61, 0x0b, 0x5e, 0x9d, 0xf7, 0xa5, 0xb7, 0x50, 0xfe, 0x16, 0xc9, 0xe0, 0xca, 0x29, - 0xe9, 0x7a, 0x03, 0xa7, 0x3a, 0xd0, 0x7c, 0xcc, 0x13, 0x19, 0x28, 0x95, 0xe6, 0xd0, 0x2e, 0xb0, - 0x32, 0x8a, 0x1e, 0xc2, 0xd5, 0x13, 0xcd, 0xb7, 0x15, 0x8f, 0xc6, 0xaa, 0xcb, 0xdf, 0xa0, 0x09, - 0xd5, 0xa2, 0x4e, 0x12, 0xd3, 0x7e, 0x0d, 0xa6, 0x01, 0xfa, 0x39, 0x5c, 0x1e, 0x08, 0x59, 0x6a, - 0x3d, 0x3b, 0x43, 0x1d, 0x70, 0x77, 0xc5, 0x8b, 0x53, 0x02, 0x54, 0x24, 0xfa, 0x15, 0x78, 0x4f, - 0xc7, 0x43, 0x2e, 0xc5, 0xb9, 0xa4, 0x37, 0xa1, 0xbe, 0x17, 0x8f, 0xe3, 0x30, 0x7e, 0x7e, 0x7c, - 0xc6, 0x2e, 0xf3, 0x60, 0x51, 0x5f, 0x25, 0xbd, 0x1c, 0x1b, 0xcc, 0x82, 0xf4, 0xa2, 0x1a, 0x53, - 0x9f, 0x87, 0x7e, 0x16, 0x2a, 0x37, 0xd4, 0x47, 0x73, 0x4a, 0x85, 0x19, 0x04, 0x8e, 0x89, 0x2b, - 0x1d, 0xba, 0x7b, 0x88, 0xb0, 0x87, 0x4e, 0x43, 0xe4, 0x53, 0x68, 0x96, 0xb8, 0x4d, 0x02, 0x2f, - 0xcf, 0xcc, 0x8b, 0x26, 0xb2, 0x32, 0x27, 0xfd, 0xcd, 0x99, 0x92, 0x3c, 0x71, 0xca, 0x8d, 0xc1, - 0x23, 0x5d, 0x94, 0x3a, 0x33, 0x90, 0x8a, 0x75, 0x7b, 0xe2, 0x87, 0x59, 0xaa, 0x48, 0xfa, 0x7a, - 0x17, 0x08, 0x15, 0xab, 0xfa, 0x33, 0x8c, 0x33, 0x69, 0x36, 0xa7, 0x05, 0xd5, 0x4f, 0x5a, 0x4f, - 0xf0, 0x61, 0x18, 0x44, 0x02, 0xbb, 0xd4, 0x65, 0x39, 0x4c, 0x6e, 0xea, 0x6d, 0x6f, 0x47, 0x6d, - 0x6d, 0xae, 0xfb, 0xc8, 0xa1, 0x2f, 0x41, 0x4a, 0x09, 0xb4, 0x67, 0x49, 0x9b, 0xed, 0xdf, 0x5f, - 0xad, 0x3b, 0x7f, 0xbc, 0x5a, 0x77, 0xfe, 0x7c, 0xb5, 0xee, 0xfc, 0xf2, 0xd7, 0xfa, 0x5b, 0xfb, - 0x35, 0xfc, 0xd7, 0xbe, 0xfd, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x76, 0xf5, 0x59, 0x94, - 0x0f, 0x00, 0x00, + // 1458 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcb, 0x72, 0x1b, 0x45, + 0x17, 0xfe, 0x47, 0x23, 0xd9, 0xd2, 0x91, 0xe5, 0xc8, 0x9d, 0xc4, 0x99, 0xf8, 0xff, 0xcb, 0xbf, + 0x68, 0x52, 0x44, 0xa4, 0x2a, 0x26, 0x95, 0x50, 0xc5, 0x35, 0x55, 0x89, 0x2d, 0x27, 0x08, 0xb0, + 0x93, 0xb4, 0x9c, 0xec, 0xdb, 0xa3, 0xae, 0x78, 0xca, 0xa3, 0x19, 0x65, 0x2e, 0x8e, 0x1c, 0xaa, + 0xd8, 0x42, 0xc1, 0x8a, 0x62, 0xc3, 0x82, 0x05, 0xef, 0xc1, 0x0b, 0xb0, 0xe4, 0x11, 0xa8, 0xf0, + 0x14, 0xec, 0xa8, 0x3e, 0xdd, 0x3d, 0x17, 0x59, 0x8e, 0x4c, 0xc2, 0x6e, 0xce, 0xfd, 0x3b, 0x97, + 0x3e, 0xdd, 0x12, 0xb4, 0xc6, 0x91, 0x77, 0xc4, 0x13, 0xb1, 0x31, 0x8e, 0xc2, 0x24, 0x24, 0x75, + 0x2f, 0x48, 0x44, 0x14, 0x70, 0x7f, 0x6d, 0x69, 0x9c, 0xee, 0xfb, 0x9e, 0xab, 0xf8, 0xf4, 0x3e, + 0x34, 0xfa, 0xc1, 0x50, 0x4c, 0x76, 0x44, 0xc2, 0x09, 0x81, 0xea, 0x17, 0xe2, 0x38, 0x76, 0xec, + 0x8e, 0xd5, 0xad, 0x33, 0xfc, 0x26, 0xef, 0xc0, 0xf2, 0x5e, 0xc4, 0xdd, 0xc3, 0xed, 0x89, 0x17, + 0x27, 0x22, 0x70, 0x85, 0x53, 0x45, 0xe9, 0x14, 0x97, 0xfe, 0x62, 0xc3, 0xd2, 0x3d, 0x4f, 0xf8, + 0xc3, 0x07, 0xe3, 0xc4, 0x0b, 0x83, 0x58, 0x3a, 0xdb, 0x3b, 0x1e, 0x0b, 0xa7, 0xde, 0xb1, 0xba, + 0x0d, 0x86, 0xdf, 0xe4, 0x7f, 0xd0, 0xd8, 0xe2, 0xee, 0x81, 0x40, 0x81, 0x8d, 0x82, 0x9c, 0x91, + 0x49, 0x07, 0xde, 0x0b, 0x15, 0xa5, 0xc5, 0x72, 0x06, 0xe9, 0x40, 0x73, 0xcf, 0x1b, 0x89, 0x47, + 0x29, 0x0f, 0x92, 0x74, 0xe4, 0xd4, 0xd0, 0xba, 0xc8, 0x22, 0xab, 0xb0, 0xf0, 0xc0, 0x1f, 0xee, + 0x78, 0x81, 0xd3, 0xe8, 0x58, 0x5d, 0x9b, 0x69, 0xca, 0xf0, 0xf9, 0xc4, 0x81, 0x9c, 0xcf, 0x27, + 0x59, 0xba, 0xcd, 0x72, 0xba, 0xbb, 0xe1, 0x20, 0xe1, 0xc1, 0x90, 0x47, 0xc3, 0x27, 0x9e, 0x78, + 0xee, 0x2c, 0xa9, 0x74, 0xcb, 0x5c, 0x69, 0xbb, 0xc9, 0x63, 0xe1, 0xb4, 0xd0, 0x23, 0x7e, 0x93, + 0x35, 0xa8, 0x6f, 0x7a, 0x49, 0x4f, 0x8c, 0x93, 0x03, 0x67, 0xb9, 0x63, 0x75, 0xab, 0x2c, 0xa3, + 0xc9, 0x05, 0xa8, 0x0d, 0x5c, 0xee, 0x0b, 0xe7, 0x1c, 0x1a, 0x28, 0x82, 0x50, 0x58, 0xba, 0x17, + 0x46, 0xc2, 0x7b, 0x1a, 0x60, 0x13, 0x9c, 0x36, 0x26, 0x55, 0xe2, 0x91, 0xb7, 0xc1, 0x96, 0x29, + 0xad, 0x74, 0xac, 0x6e, 0xf3, 0xe6, 0xca, 0x86, 0xe9, 0xe3, 0x46, 0x4f, 0xb8, 0xde, 0x88, 0xfb, + 0x4c, 0x4a, 0x51, 0x89, 0x4f, 0x1c, 0x72, 0xba, 0x12, 0x9f, 0x50, 0x0a, 0xcb, 0xfd, 0xd1, 0x38, + 0x8c, 0x12, 0x26, 0xe2, 0x71, 0x18, 0xc4, 0x82, 0xb4, 0xc1, 0xde, 0x8e, 0x22, 0xc7, 0xc2, 0xb0, + 0xf2, 0x93, 0x7e, 0x0d, 0xed, 0x4d, 0x3f, 0x74, 0x0f, 0x7b, 0x3c, 0xe1, 0x4c, 0x3c, 0x4b, 0x45, + 0x9c, 0x48, 0xec, 0x0a, 0x9e, 0xd2, 0x53, 0x84, 0xe4, 0x62, 0xbf, 0x9d, 0x8a, 0xe2, 0x22, 0x21, + 0xeb, 0x82, 0x55, 0x53, 0xed, 0xc1, 0x6f, 0xcc, 0xfd, 0x80, 0x47, 0x43, 0xec, 0x69, 0x95, 0x29, + 0x42, 0x72, 0x31, 0x12, 0xce, 0x41, 0x95, 0x29, 0x82, 0xf6, 0x61, 0xa5, 0x10, 0x5f, 0xc3, 0x5c, + 0x85, 0x05, 0x16, 0x3e, 0xef, 0xf7, 0x62, 0xc7, 0xea, 0xd8, 0xdd, 0x2a, 0xd3, 0x14, 0x0e, 0x4c, + 0xe8, 0xa7, 0xa3, 0x40, 0x8a, 0x2a, 0x28, 0xca, 0x19, 0xf4, 0x32, 0xd4, 0x70, 0x7a, 0x64, 0x96, + 0xb9, 0xad, 0xfc, 0xa4, 0xdf, 0x58, 0xd0, 0xd8, 0xe1, 0x13, 0x04, 0x12, 0x93, 0xdb, 0x50, 0x37, + 0xbd, 0x45, 0xa5, 0xe6, 0xcd, 0xb7, 0xf2, 0x0a, 0x66, 0x6a, 0x1b, 0x46, 0x67, 0x3b, 0x48, 0xa2, + 0x63, 0x96, 0x99, 0xac, 0x7d, 0x02, 0xad, 0x92, 0x48, 0xc6, 0x3b, 0x14, 0xc7, 0xa6, 0xaa, 0x87, + 0xe2, 0x58, 0xe6, 0x7a, 0xc4, 0xfd, 0x54, 0x60, 0xad, 0xaa, 0x4c, 0x11, 0x1f, 0x57, 0x3e, 0xb4, + 0xe8, 0x13, 0x20, 0x5b, 0x91, 0xe0, 0x89, 0xc0, 0x20, 0x3b, 0x22, 0x8e, 0xf9, 0x53, 0x31, 0xaf, + 0xe2, 0x76, 0xb1, 0xe2, 0x59, 0x75, 0x2b, 0x85, 0xea, 0xd2, 0x6b, 0x40, 0x7a, 0xc2, 0x17, 0x89, + 0xd0, 0xa7, 0xfb, 0x15, 0x7e, 0xe9, 0x33, 0x83, 0x61, 0xbe, 0x2e, 0xb9, 0x0a, 0x55, 0xb9, 0x2a, + 0x30, 0x58, 0xf3, 0xe6, 0xf9, 0xbc, 0x4e, 0xd9, 0x16, 0x61, 0xa8, 0x80, 0xbd, 0x41, 0xa7, 0xc3, + 0xbb, 0x09, 0x02, 0xb6, 0x59, 0xce, 0xa0, 0xdf, 0x59, 0x26, 0x26, 0x26, 0x71, 0xc6, 0xbc, 0x4b, + 0x93, 0x76, 0x4d, 0x23, 0xb1, 0x11, 0xc9, 0x6a, 0x8e, 0xa4, 0xb8, 0x85, 0x66, 0x81, 0xa9, 0x4e, + 0x83, 0xb9, 0x63, 0x6a, 0xf5, 0xba, 0x58, 0xa8, 0x0b, 0xff, 0x55, 0x1e, 0xee, 0x1e, 0x71, 0xcf, + 0xe7, 0xfb, 0xfe, 0x3f, 0x6a, 0x67, 0x29, 0x2d, 0x07, 0x16, 0xd1, 0xb6, 0xdf, 0xd3, 0x07, 0xc3, + 0x90, 0xf4, 0x2b, 0xc8, 0xcf, 0xd8, 0x2e, 0x1f, 0x09, 0xed, 0x0d, 0xbf, 0xb3, 0x6a, 0x54, 0xce, + 0x50, 0x8d, 0x0b, 0x50, 0x93, 0xe7, 0x52, 0xee, 0x79, 0x5b, 0x06, 0x46, 0x62, 0x4e, 0x8d, 0x6e, + 0xc1, 0xc2, 0xc0, 0x3d, 0x10, 0x23, 0x4e, 0xde, 0x85, 0x45, 0xc4, 0x2f, 0x62, 0x7d, 0x58, 0xce, + 0x4d, 0x0d, 0x01, 0x33, 0x72, 0xfa, 0x83, 0xa5, 0x13, 0x9f, 0x09, 0xb9, 0x14, 0xb0, 0x32, 0x15, + 0x90, 0x5c, 0x87, 0x45, 0x8d, 0x1a, 0x77, 0xc9, 0x29, 0xb3, 0x66, 0x74, 0xc8, 0x55, 0x58, 0xc0, + 0x4c, 0x63, 0xa7, 0x3a, 0x0d, 0x0a, 0xf9, 0x4c, 0x8b, 0xe9, 0x36, 0xd8, 0x8f, 0x59, 0x5f, 0xae, + 0x14, 0xcc, 0xc7, 0x40, 0xd2, 0x94, 0x04, 0xfa, 0x59, 0x18, 0x27, 0xba, 0x27, 0xf8, 0x2d, 0x79, + 0x0f, 0xc3, 0x48, 0x4d, 0x71, 0x8b, 0xe1, 0x37, 0xfd, 0xd9, 0x82, 0xea, 0x6e, 0x38, 0x14, 0x64, + 0x19, 0x2a, 0xfd, 0x9e, 0x76, 0x52, 0xe9, 0xf7, 0xc8, 0xff, 0xd1, 0xbf, 0xee, 0x43, 0x2b, 0x47, + 0xf1, 0x98, 0xf5, 0x19, 0x46, 0xbe, 0x02, 0xad, 0x7e, 0xbc, 0x15, 0x86, 0xd1, 0xd0, 0x0b, 0x78, + 0x12, 0x46, 0xfa, 0xb6, 0x2d, 0x33, 0xf1, 0x54, 0x27, 0x3c, 0x51, 0xf7, 0x60, 0x83, 0x29, 0x82, + 0x5c, 0x85, 0xc5, 0xfb, 0xec, 0xe1, 0x96, 0x0c, 0x50, 0x9b, 0x15, 0xc0, 0x48, 0xe9, 0x1d, 0x68, + 0x4b, 0x74, 0x68, 0x65, 0xa6, 0x70, 0x15, 0x16, 0x24, 0x2f, 0x43, 0xab, 0xa9, 0x3c, 0x54, 0xa5, + 0x10, 0x8a, 0x7e, 0xa9, 0x3c, 0x6c, 0x1f, 0x89, 0x20, 0x29, 0xcc, 0x31, 0xd2, 0xe8, 0xa0, 0xc5, + 0x14, 0x41, 0xa8, 0xaa, 0x84, 0x4e, 0x79, 0x39, 0x47, 0x24, 0xb9, 0x0c, 0x65, 0xf4, 0x7b, 0x0b, + 0xc0, 0x00, 0x4a, 0xe3, 0xcc, 0xc4, 0x3a, 0xdd, 0x84, 0x74, 0xcd, 0xc4, 0xe9, 0x13, 0xde, 0xce, + 0xb5, 0x14, 0x9f, 0x99, 0x89, 0x7c, 0x2f, 0x9f, 0x48, 0xd5, 0xfc, 0x8b, 0x53, 0xa3, 0xa2, 0xa2, + 0xe6, 0x73, 0x19, 0x40, 0xb3, 0xc0, 0x9f, 0x39, 0x9c, 0xd7, 0xb3, 0x79, 0xaa, 0x4c, 0xbb, 0x44, + 0xbe, 0x76, 0xa9, 0x95, 0xe6, 0x6c, 0x3b, 0x0f, 0x9a, 0x05, 0xa3, 0x99, 0xf1, 0xba, 0x70, 0xae, + 0xbc, 0x3b, 0xcc, 0x85, 0x36, 0xcd, 0x9e, 0x13, 0xea, 0x47, 0x0b, 0x5a, 0x5b, 0x7e, 0x1a, 0x27, + 0x22, 0xd2, 0xd1, 0xa4, 0xbe, 0x62, 0x64, 0x9d, 0xcf, 0x19, 0xb3, 0x9b, 0x4f, 0xae, 0x40, 0x4d, + 0xf6, 0x40, 0x6d, 0x88, 0x93, 0x0d, 0x52, 0xc2, 0x42, 0x87, 0xaa, 0xaf, 0xee, 0x10, 0x7d, 0x02, + 0xf5, 0xcd, 0x41, 0xff, 0x7e, 0x14, 0xa6, 0xe3, 0x99, 0xd9, 0x9b, 0xb7, 0x62, 0xa5, 0xf0, 0x56, + 0x6c, 0xab, 0x77, 0x8f, 0xca, 0x10, 0x1f, 0x39, 0x6d, 0xf5, 0xc8, 0xa9, 0x6a, 0x0e, 0x9f, 0xd0, + 0x01, 0xac, 0xa8, 0xd4, 0xe5, 0x0a, 0x7b, 0x9d, 0x6d, 0x6b, 0x9e, 0x2b, 0x76, 0xfe, 0x5c, 0x91, + 0x4e, 0xd5, 0x32, 0xff, 0x37, 0x9d, 0xfe, 0x55, 0x81, 0x15, 0x26, 0x62, 0xef, 0x85, 0xe8, 0x07, + 0x71, 0x12, 0xa5, 0xae, 0x5c, 0x5b, 0xd2, 0xfe, 0xf3, 0x70, 0x5f, 0xf7, 0xc5, 0x66, 0x8a, 0x38, + 0xcb, 0x81, 0x22, 0x37, 0xa0, 0x39, 0xbd, 0x43, 0x4e, 0xaa, 0x16, 0x55, 0xc8, 0x0d, 0x58, 0x1c, + 0x84, 0x69, 0xe4, 0x66, 0xa7, 0xa4, 0x70, 0x49, 0x28, 0x64, 0x4a, 0xcc, 0x8c, 0x1a, 0x79, 0x04, + 0x64, 0x2f, 0xe2, 0x41, 0xec, 0x73, 0x09, 0xd6, 0x18, 0xd7, 0xa7, 0x5f, 0x48, 0x05, 0x9d, 0x92, + 0x9f, 0x19, 0xc6, 0xe4, 0xfd, 0xe2, 0x1a, 0x70, 0x16, 0x11, 0xf5, 0x85, 0x32, 0x6a, 0x7d, 0xb2, + 0x8a, 0xeb, 0xe2, 0xf6, 0xd4, 0x4c, 0x3b, 0x0b, 0x68, 0x78, 0x29, 0x37, 0x2c, 0x89, 0x59, 0x59, + 0x9b, 0x7e, 0x6b, 0xc1, 0x52, 0x11, 0xd9, 0x99, 0xd6, 0x4f, 0xd6, 0xf0, 0xca, 0xfc, 0x27, 0x98, + 0x69, 0x78, 0x75, 0xd6, 0xa3, 0xb7, 0x56, 0x7c, 0x96, 0xa5, 0x70, 0xe9, 0x94, 0x72, 0xbd, 0x01, + 0xa8, 0x0e, 0x34, 0x1f, 0xf2, 0x28, 0xf1, 0xa4, 0x4b, 0xfd, 0x6c, 0xa8, 0xb1, 0x22, 0x8b, 0x1e, + 0xc2, 0xe5, 0x13, 0xc3, 0xb7, 0x15, 0x8e, 0xc6, 0x72, 0xca, 0xdf, 0x60, 0x08, 0xe5, 0x7d, 0x10, + 0x45, 0x7a, 0xfc, 0x1a, 0x4c, 0x11, 0xf4, 0x23, 0xb8, 0x38, 0x10, 0x49, 0x61, 0xf4, 0xcc, 0x19, + 0xea, 0x80, 0xbd, 0x2b, 0x9e, 0x9f, 0x92, 0xa0, 0x14, 0xd1, 0x4f, 0xc1, 0x79, 0x3c, 0x1e, 0xf2, + 0x44, 0xbc, 0x96, 0xf5, 0x26, 0xd4, 0xf7, 0xc2, 0x71, 0xe8, 0x87, 0x4f, 0x8f, 0xe7, 0x6c, 0x3d, + 0x07, 0x16, 0xd5, 0xe5, 0xa7, 0xb6, 0x6c, 0x83, 0x19, 0x92, 0x9e, 0x97, 0xc7, 0xd4, 0xe5, 0xbe, + 0x9b, 0xfa, 0x12, 0x86, 0xfc, 0xfd, 0x10, 0x53, 0xa1, 0x0f, 0x02, 0xc7, 0xc2, 0x15, 0xee, 0xd3, + 0xbb, 0xc8, 0x30, 0xf7, 0xa9, 0xa2, 0xc8, 0x07, 0xd0, 0x2c, 0x68, 0xeb, 0x02, 0x5e, 0x9c, 0x3a, + 0x2f, 0x4a, 0xc8, 0x8a, 0x9a, 0xf4, 0x57, 0xab, 0x64, 0x79, 0xe2, 0x69, 0xa1, 0x03, 0x1e, 0xa9, + 0xa6, 0xd4, 0x99, 0xa6, 0x64, 0xae, 0xdb, 0x13, 0xd7, 0x4f, 0x63, 0x29, 0x52, 0xaf, 0x89, 0x9c, + 0x21, 0x73, 0x95, 0x3f, 0x92, 0xc3, 0xd4, 0xbc, 0xea, 0x0c, 0x29, 0x7f, 0xaf, 0xf6, 0x04, 0x1f, + 0xfa, 0x5e, 0x20, 0x70, 0x4a, 0x6d, 0x96, 0xd1, 0xe4, 0x86, 0xba, 0x17, 0xcc, 0x51, 0x5b, 0x9b, + 0x09, 0x1f, 0x35, 0xd4, 0x9d, 0x11, 0x53, 0x02, 0xed, 0x69, 0xd1, 0x66, 0xfb, 0xb7, 0x97, 0xeb, + 0xd6, 0xef, 0x2f, 0xd7, 0xad, 0x3f, 0x5e, 0xae, 0x5b, 0x3f, 0xfd, 0xb9, 0xfe, 0x9f, 0xfd, 0x05, + 0xfc, 0xdb, 0xe1, 0xd6, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x31, 0xb0, 0x31, 0x3c, 0x9f, 0x10, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -3034,6 +3102,11 @@ func (m *CreateIndexMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3080,6 +3153,11 @@ func (m *CreateFieldMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x20 + } if m.Meta != nil { { size, err := m.Meta.MarshalToSizedBuffer(dAtA[:i]) @@ -3220,6 +3298,11 @@ func (m *Field) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x20 + } if len(m.Views) > 0 { for iNdEx := len(m.Views) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Views[iNdEx]) @@ -3342,6 +3425,11 @@ func (m *Index) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x10 + } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) @@ -3422,6 +3510,18 @@ func (m *Node) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.GRPCURI != nil { + { + size, err := m.GRPCURI.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if len(m.State) > 0 { i -= len(m.State) copy(dAtA[i:], m.State) @@ -3635,6 +3735,11 @@ func (m *IndexStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if len(m.Fields) > 0 { for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { { @@ -3683,21 +3788,26 @@ func (m *FieldStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.CreatedAt != 0 { + i = encodeVarintPrivate(dAtA, i, uint64(m.CreatedAt)) + i-- + dAtA[i] = 0x18 + } if len(m.AvailableShards) > 0 { - dAtA18 := make([]byte, len(m.AvailableShards)*10) - var j17 int + dAtA19 := make([]byte, len(m.AvailableShards)*10) + var j18 int for _, num := range m.AvailableShards { for num >= 1<<7 { - dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j17++ + j18++ } - dAtA18[j17] = uint8(num) - j17++ + dAtA19[j18] = uint8(num) + j18++ } - i -= j17 - copy(dAtA[i:], dAtA18[:j17]) - i = encodeVarintPrivate(dAtA, i, uint64(j17)) + i -= j18 + copy(dAtA[i:], dAtA19[:j18]) + i = encodeVarintPrivate(dAtA, i, uint64(j18)) i-- dAtA[i] = 0x12 } @@ -3735,6 +3845,18 @@ func (m *ClusterStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.Schema != nil { + { + size, err := m.Schema.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if len(m.Nodes) > 0 { for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { { @@ -4738,6 +4860,9 @@ func (m *CreateIndexMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4762,6 +4887,9 @@ func (m *CreateFieldMessage) Size() (n int) { l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4831,6 +4959,9 @@ func (m *Field) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4865,6 +4996,9 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -4925,6 +5059,10 @@ func (m *Node) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.GRPCURI != nil { + l = m.GRPCURI.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5012,6 +5150,9 @@ func (m *IndexStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5035,6 +5176,9 @@ func (m *FieldStatus) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } + if m.CreatedAt != 0 { + n += 1 + sovPrivate(uint64(m.CreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5061,6 +5205,10 @@ func (m *ClusterStatus) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } + if m.Schema != nil { + l = m.Schema.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6996,6 +7144,25 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7150,6 +7317,25 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7559,6 +7745,25 @@ func (m *Field) Unmarshal(dAtA []byte) error { } m.Views = append(m.Views, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7733,6 +7938,25 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -8114,6 +8338,42 @@ func (m *Node) Unmarshal(dAtA []byte) error { } m.State = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GRPCURI", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.GRPCURI == nil { + m.GRPCURI = &URI{} + } + if err := m.GRPCURI.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8621,6 +8881,25 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8783,6 +9062,25 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field AvailableShards", wireType) } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedAt", wireType) + } + m.CreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8935,6 +9233,42 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Schema == nil { + m.Schema = &Schema{} + } + if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/internal/private.proto b/internal/private.proto index 804c3f2cd..d83a8d2c0 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -64,12 +64,14 @@ message DeleteIndexMessage { message CreateIndexMessage { string Index = 1; IndexMeta Meta = 2; + int64 CreatedAt = 3; } message CreateFieldMessage { string Index = 1; string Field = 2; FieldOptions Meta = 3; + int64 CreatedAt = 4; } message DeleteFieldMessage { @@ -87,6 +89,7 @@ message Field { string Name = 1; FieldOptions Meta = 2; repeated string Views = 3; + int64 CreatedAt = 4; } message Schema { @@ -95,6 +98,7 @@ message Schema { message Index { string Name = 1; + int64 CreatedAt = 2; IndexMeta Options = 5; repeated Field Fields = 4; } @@ -110,6 +114,7 @@ message Node { URI URI = 2; bool IsCoordinator = 3; string State = 4; + URI GRPCURI = 5; } message NodeStateMessage { @@ -131,17 +136,20 @@ message NodeStatus { message IndexStatus { string Name = 1; repeated FieldStatus Fields = 2; + int64 CreatedAt = 3; } message FieldStatus { string Name = 1; repeated uint64 AvailableShards = 2; + int64 CreatedAt = 3; } message ClusterStatus { string ClusterID = 1; string State = 2; repeated Node Nodes = 3; + Schema Schema = 4; } message BSIGroup { diff --git a/internal/public.pb.go b/internal/public.pb.go index bf04b2d90..9f71b0368 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1183,6 +1183,8 @@ type ImportRequest struct { RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys,proto3" json:"RowKeys,omitempty"` ColumnKeys []string `protobuf:"bytes,8,rep,name=ColumnKeys,proto3" json:"ColumnKeys,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps,proto3" json:"Timestamps,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,9,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,10,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1277,6 +1279,20 @@ func (m *ImportRequest) GetTimestamps() []int64 { return nil } +func (m *ImportRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1286,6 +1302,8 @@ type ImportValueRequest struct { Values []int64 `protobuf:"varint,6,rep,packed,name=Values,proto3" json:"Values,omitempty"` FloatValues []float64 `protobuf:"fixed64,8,rep,packed,name=FloatValues,proto3" json:"FloatValues,omitempty"` StringValues []string `protobuf:"bytes,9,rep,name=StringValues,proto3" json:"StringValues,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,10,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,11,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1380,6 +1398,20 @@ func (m *ImportValueRequest) GetStringValues() []string { return nil } +func (m *ImportValueRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportValueRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type TranslateKeysRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1660,6 +1692,8 @@ type ImportRoaringRequest struct { Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views,proto3" json:"views,omitempty"` Action string `protobuf:"bytes,3,opt,name=Action,proto3" json:"Action,omitempty"` Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,5,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` + FieldCreatedAt int64 `protobuf:"varint,6,opt,name=FieldCreatedAt,proto3" json:"FieldCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1726,12 +1760,27 @@ func (m *ImportRoaringRequest) GetBlock() uint64 { return 0 } +func (m *ImportRoaringRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + +func (m *ImportRoaringRequest) GetFieldCreatedAt() int64 { + if m != nil { + return m.FieldCreatedAt + } + return 0 +} + type ImportColumnAttrsRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Shard int64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` AttrKey string `protobuf:"bytes,3,opt,name=AttrKey,proto3" json:"AttrKey,omitempty"` AttrVals []string `protobuf:"bytes,4,rep,name=AttrVals,proto3" json:"AttrVals,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs,proto3" json:"ColumnIDs,omitempty"` + IndexCreatedAt int64 `protobuf:"varint,6,opt,name=IndexCreatedAt,proto3" json:"IndexCreatedAt,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1805,6 +1854,13 @@ func (m *ImportColumnAttrsRequest) GetColumnIDs() []uint64 { return nil } +func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { + if m != nil { + return m.IndexCreatedAt + } + return 0 +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") @@ -1837,83 +1893,86 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1207 bytes of a gzipped FileDescriptorProto + // 1258 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0xa6, 0x3d, 0xe3, 0xb5, 0x5d, 0xf6, 0x6e, 0x42, 0xc7, 0x09, 0x23, 0x14, 0x36, 0x56, 0x2b, - 0x20, 0xc3, 0x61, 0xa3, 0x0d, 0x21, 0xca, 0x09, 0xc8, 0xc6, 0x1b, 0xb0, 0xa2, 0xac, 0x42, 0x3b, - 0x32, 0x37, 0xa4, 0x59, 0xbb, 0xd9, 0x8c, 0x18, 0xcf, 0x98, 0xf9, 0xc1, 0xd9, 0x23, 0xcf, 0x00, - 0x07, 0xc4, 0x13, 0xf0, 0x28, 0x1c, 0x79, 0x04, 0x58, 0xee, 0x1c, 0xb8, 0x72, 0x41, 0x55, 0x3d, - 0xed, 0x6e, 0x7b, 0xbd, 0x4b, 0x14, 0x71, 0xeb, 0xaf, 0xaa, 0xa6, 0xba, 0xbe, 0xea, 0xea, 0xaa, - 0x1e, 0xe8, 0xcc, 0xcb, 0xe3, 0x38, 0x9a, 0xec, 0xcd, 0xb3, 0xb4, 0x48, 0x79, 0x33, 0x4a, 0x0a, - 0x95, 0x25, 0x61, 0x2c, 0x72, 0xf0, 0x64, 0xba, 0xe0, 0x01, 0x34, 0x1e, 0xa5, 0x71, 0x39, 0x4b, - 0xf2, 0x80, 0xf5, 0xbc, 0xbe, 0x2f, 0x0d, 0xe4, 0x1c, 0xfc, 0x27, 0xea, 0x34, 0x0f, 0xbc, 0x9e, - 0xd7, 0x6f, 0x49, 0x5a, 0xf3, 0xdb, 0x50, 0x7f, 0x58, 0x14, 0x59, 0x1e, 0xd4, 0x7a, 0x5e, 0xbf, - 0x7d, 0x77, 0x67, 0xcf, 0xb8, 0xdb, 0x43, 0xb1, 0xd4, 0x4a, 0xf4, 0x29, 0xd3, 0x30, 0x8b, 0x92, - 0x93, 0xc0, 0xef, 0xb1, 0x7e, 0x47, 0x1a, 0x28, 0x9e, 0x42, 0x6b, 0x14, 0x9d, 0x24, 0x6a, 0x8a, - 0x5b, 0xdf, 0x02, 0xef, 0x59, 0x8a, 0xdb, 0xb2, 0x7e, 0xfb, 0xee, 0xb6, 0x75, 0x25, 0xd3, 0x85, - 0x44, 0x0d, 0x1a, 0x1c, 0xa9, 0x93, 0xa0, 0xb6, 0xd1, 0xe0, 0x48, 0x9d, 0x88, 0x07, 0xb0, 0x23, - 0xd3, 0xc5, 0x70, 0xaa, 0x92, 0x22, 0xfa, 0x3a, 0x52, 0x19, 0x05, 0x2d, 0xd3, 0x85, 0xe1, 0x42, - 0xeb, 0x25, 0x91, 0x9a, 0x25, 0x22, 0x3e, 0x06, 0xff, 0x59, 0x18, 0x65, 0x7c, 0x07, 0x6a, 0xc3, - 0x01, 0x85, 0xe0, 0xcb, 0xda, 0x70, 0xc0, 0xaf, 0x82, 0xf7, 0x44, 0x9d, 0x06, 0x5e, 0x8f, 0xf5, - 0x5b, 0x12, 0x97, 0xbc, 0x0b, 0xf5, 0x47, 0x69, 0x99, 0x14, 0x14, 0x86, 0x2f, 0x35, 0x10, 0x87, - 0xd0, 0xc2, 0xef, 0x1f, 0x47, 0x2a, 0x9e, 0x72, 0xa1, 0x9d, 0x55, 0x4c, 0x9c, 0xa4, 0xa0, 0x54, - 0xea, 0x8d, 0xba, 0x50, 0x27, 0x63, 0x72, 0xd3, 0x92, 0x1a, 0x88, 0xcf, 0x01, 0x50, 0x9b, 0x6b, - 0x3f, 0xb7, 0xa1, 0x4e, 0x88, 0xa2, 0x3f, 0xef, 0x48, 0x2b, 0x2f, 0xf0, 0xf4, 0x0e, 0xd4, 0x87, - 0x49, 0x71, 0xff, 0x1e, 0xaa, 0xc7, 0x61, 0x5c, 0x2a, 0x8a, 0xc6, 0x93, 0x1a, 0x88, 0x12, 0x9a, - 0x64, 0x87, 0x79, 0x5f, 0x3a, 0x60, 0x8e, 0x03, 0x94, 0x62, 0x2e, 0x07, 0x86, 0x27, 0x01, 0x7e, - 0x03, 0xb6, 0x64, 0xba, 0xb0, 0x29, 0xa9, 0x10, 0x7f, 0xd7, 0xec, 0xe2, 0x13, 0xe7, 0x2b, 0x36, - 0x54, 0x8a, 0xc2, 0x6c, 0xfb, 0x15, 0xc0, 0x67, 0x59, 0x5a, 0xce, 0x29, 0x69, 0xbc, 0x0f, 0x75, - 0x42, 0x15, 0x3f, 0x6e, 0x3f, 0x32, 0xb1, 0x49, 0x6d, 0xb0, 0x39, 0xe9, 0x78, 0x38, 0xa3, 0x72, - 0x46, 0x91, 0x78, 0x12, 0x97, 0xe2, 0x7b, 0x06, 0xcd, 0x71, 0x18, 0x2f, 0xd5, 0xe3, 0x30, 0xae, - 0x78, 0xe3, 0x72, 0xd5, 0x8d, 0x67, 0xdc, 0xbc, 0x0d, 0xcd, 0xc7, 0x71, 0x1a, 0x16, 0x68, 0x8c, - 0xbe, 0x98, 0x5c, 0x62, 0xbe, 0x0f, 0x30, 0x50, 0x93, 0x68, 0x16, 0xc6, 0xa8, 0xd5, 0xe4, 0xde, - 0xb4, 0x71, 0x56, 0x3a, 0xe9, 0x18, 0x89, 0x8f, 0xa0, 0x51, 0xa1, 0xcd, 0xb9, 0x47, 0xe9, 0x68, - 0x12, 0xc6, 0xca, 0x44, 0x41, 0x40, 0x7c, 0x09, 0xdb, 0xfa, 0xa6, 0xe1, 0x9d, 0x19, 0xa9, 0xe2, - 0x15, 0x4a, 0xf1, 0x95, 0x6e, 0x9f, 0xf8, 0x85, 0x81, 0x8f, 0x2b, 0xe3, 0x80, 0x59, 0x07, 0x1c, - 0xfc, 0xe7, 0xa7, 0x73, 0x55, 0x65, 0x95, 0xd6, 0xbc, 0x07, 0xed, 0x51, 0x81, 0x97, 0x53, 0x47, - 0xae, 0xb7, 0x73, 0x45, 0x98, 0xaf, 0x61, 0x52, 0xd8, 0xe3, 0xf6, 0xe4, 0x12, 0xf3, 0x9b, 0xd0, - 0x3a, 0x48, 0xd3, 0x58, 0x2b, 0xeb, 0x3d, 0xd6, 0x6f, 0x4a, 0x2b, 0xe0, 0xbb, 0x00, 0x26, 0xb3, - 0xa5, 0x0a, 0xb6, 0x28, 0xd7, 0x8e, 0x44, 0xdc, 0x81, 0x06, 0x46, 0xfa, 0x34, 0x9c, 0x5b, 0x6e, - 0xec, 0x32, 0x6e, 0xff, 0x30, 0xe8, 0x7c, 0x51, 0xaa, 0xec, 0x54, 0xaa, 0x6f, 0x4b, 0x95, 0x17, - 0x98, 0x5b, 0xc2, 0xa6, 0x96, 0x09, 0x60, 0xd5, 0x8e, 0x5e, 0x84, 0xd9, 0x54, 0x67, 0xca, 0x97, - 0x15, 0x42, 0xae, 0x36, 0xe7, 0x39, 0x71, 0x6d, 0x4a, 0x57, 0x44, 0xf5, 0xae, 0x66, 0x69, 0x61, - 0xc8, 0x54, 0x88, 0xf7, 0xe1, 0xca, 0xe1, 0xcb, 0x49, 0x5c, 0x4e, 0x95, 0x4c, 0x17, 0xfa, 0xeb, - 0x2d, 0x32, 0x58, 0x17, 0xf3, 0xf7, 0x60, 0xa7, 0x12, 0x99, 0xbe, 0xda, 0x20, 0xc3, 0x35, 0x29, - 0xdf, 0x87, 0xce, 0xe1, 0xec, 0x58, 0x4d, 0xa7, 0x6a, 0x3a, 0x08, 0x8b, 0x30, 0x68, 0x12, 0xef, - 0xb5, 0x2e, 0xb7, 0x62, 0x22, 0x7e, 0x60, 0xb0, 0x5d, 0xb1, 0xcf, 0xe7, 0x69, 0x92, 0x2b, 0x3c, - 0xe2, 0xc3, 0x2c, 0x33, 0x47, 0x7c, 0x98, 0x65, 0xfc, 0x0e, 0x34, 0xa4, 0xca, 0xcb, 0xb8, 0x30, - 0x55, 0x72, 0xdd, 0x7a, 0x34, 0xdf, 0x96, 0x71, 0x21, 0x8d, 0x15, 0xff, 0x04, 0x76, 0x56, 0xea, - 0x50, 0x37, 0xfc, 0xf6, 0xdd, 0xb7, 0xec, 0x77, 0x2b, 0x7a, 0xb9, 0x66, 0x2e, 0xfe, 0xf2, 0xa0, - 0xed, 0x78, 0x5e, 0x16, 0x19, 0xe6, 0x67, 0xbb, 0x2a, 0xb2, 0x5b, 0x34, 0x6c, 0x2e, 0x68, 0xf5, - 0xd8, 0x93, 0x3a, 0xc0, 0x8e, 0xaa, 0xb2, 0x64, 0x47, 0xb6, 0x11, 0x7a, 0x97, 0x35, 0x42, 0x1c, - 0x5d, 0x2f, 0xc2, 0xe4, 0x44, 0x4d, 0xa9, 0x2c, 0x9b, 0xd2, 0x40, 0xbe, 0x67, 0xbb, 0x02, 0x9d, - 0xe3, 0x4a, 0xaf, 0x31, 0x1a, 0x69, 0x3b, 0x87, 0xee, 0x72, 0xc3, 0x01, 0x9e, 0x15, 0xd5, 0x8b, - 0x46, 0xfc, 0x3e, 0xb4, 0x6d, 0xfb, 0xca, 0xab, 0x23, 0xea, 0x5a, 0x57, 0x56, 0x29, 0x5d, 0x43, - 0xfe, 0xe9, 0xfa, 0x5c, 0x0a, 0x5a, 0x14, 0x45, 0xb0, 0xc2, 0xdc, 0xd1, 0xcb, 0xf5, 0x39, 0xb6, - 0xef, 0x0c, 0xca, 0x00, 0xe8, 0xe3, 0x6b, 0xf6, 0xe3, 0xa5, 0x4a, 0x3a, 0xe3, 0xf4, 0x9e, 0x3b, - 0x4b, 0x82, 0x36, 0x7d, 0xd3, 0x5d, 0xcd, 0x9c, 0xd6, 0x49, 0x77, 0xe6, 0xec, 0x3b, 0x83, 0x2c, - 0xe8, 0xac, 0x6f, 0xb4, 0x54, 0x49, 0x6b, 0x25, 0xfe, 0x60, 0xb0, 0x3d, 0x9c, 0xcd, 0xd3, 0xac, - 0x70, 0x6e, 0xe1, 0x30, 0x99, 0xaa, 0x97, 0xe6, 0x16, 0x12, 0xd8, 0x3c, 0xa8, 0xa8, 0x1b, 0xe2, - 0x6d, 0xa4, 0xdb, 0xe7, 0x4b, 0x0d, 0x9c, 0x13, 0xf0, 0x57, 0x4e, 0xe0, 0x26, 0xb4, 0x74, 0xb9, - 0xa1, 0xaa, 0x4e, 0x2a, 0x2b, 0xd0, 0x0f, 0x8d, 0x05, 0x0d, 0xf7, 0x06, 0x0d, 0x77, 0x03, 0xb1, - 0xf3, 0x68, 0x33, 0x52, 0x36, 0x49, 0xe9, 0x48, 0x50, 0xff, 0x3c, 0x9a, 0xa9, 0xbc, 0x08, 0x67, - 0x73, 0xbc, 0xca, 0x5e, 0xdf, 0x93, 0x8e, 0x44, 0xfc, 0xcd, 0x80, 0x6b, 0x8e, 0xd4, 0xa9, 0xfe, - 0x3f, 0xa2, 0x97, 0x13, 0x5a, 0x0d, 0xbb, 0x71, 0x2e, 0xec, 0x1b, 0xb0, 0x45, 0xf1, 0x98, 0x90, - 0x2b, 0x84, 0x8d, 0xcd, 0xb6, 0x55, 0xcd, 0x97, 0x49, 0x57, 0xc4, 0x05, 0x74, 0x9c, 0x9e, 0x8e, - 0x05, 0x89, 0xbe, 0x57, 0x64, 0x62, 0x0c, 0xdd, 0xe7, 0x59, 0x98, 0xe4, 0x71, 0x58, 0x28, 0xdc, - 0xee, 0x75, 0x58, 0x6f, 0x78, 0x35, 0x8a, 0xf7, 0xe1, 0xfa, 0x9a, 0x5f, 0xdb, 0xbe, 0x30, 0x0d, - 0x1e, 0xa5, 0x01, 0x97, 0x62, 0x04, 0xd7, 0x96, 0xa6, 0xc3, 0xc1, 0x6b, 0x45, 0x70, 0xde, 0xe9, - 0x07, 0x0e, 0x2f, 0x72, 0x5a, 0x6d, 0xbf, 0x29, 0xd6, 0x03, 0x08, 0xaa, 0xda, 0xd6, 0x4f, 0xd6, - 0x2a, 0x82, 0x71, 0xa4, 0x16, 0x68, 0x7f, 0x14, 0xce, 0x54, 0x15, 0x04, 0xad, 0x51, 0x46, 0xed, - 0xbb, 0x46, 0x0f, 0x5d, 0x5a, 0x8b, 0x1f, 0x19, 0x74, 0x37, 0x39, 0xa1, 0xf7, 0x48, 0xac, 0x42, - 0xdd, 0xb0, 0x9b, 0x52, 0x03, 0xfe, 0x00, 0xea, 0xdf, 0x45, 0x6a, 0x61, 0x1a, 0xb6, 0x70, 0xde, - 0x52, 0x17, 0x44, 0x22, 0xf5, 0x07, 0x58, 0x0e, 0x0f, 0x27, 0x45, 0x94, 0x26, 0xe6, 0x75, 0xa6, - 0x11, 0xee, 0x73, 0x10, 0xa7, 0x93, 0x6f, 0xa8, 0x2f, 0xfa, 0x52, 0x03, 0xf1, 0x33, 0x33, 0xdc, - 0x9c, 0x89, 0xf7, 0x9f, 0x19, 0xd6, 0x35, 0x6c, 0x9e, 0x2e, 0x54, 0xc3, 0x81, 0x1e, 0xdb, 0xf6, - 0x75, 0x62, 0x20, 0x3e, 0x15, 0x70, 0x39, 0x0e, 0x63, 0x7d, 0x91, 0x5b, 0x72, 0x89, 0x2f, 0xaf, - 0xfc, 0x83, 0xab, 0xbf, 0x9e, 0xed, 0xb2, 0xdf, 0xce, 0x76, 0xd9, 0xef, 0x67, 0xbb, 0xec, 0xa7, - 0x3f, 0x77, 0xdf, 0x38, 0xde, 0xa2, 0x3f, 0x96, 0x0f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xfc, - 0x97, 0x67, 0xec, 0xc1, 0x0c, 0x00, 0x00, + 0x10, 0x66, 0x3c, 0xe3, 0xbf, 0xb2, 0x77, 0x13, 0x3a, 0x4e, 0x18, 0xa1, 0xb0, 0xb1, 0x46, 0x01, + 0x19, 0x0e, 0x1b, 0x6d, 0x08, 0x51, 0x4e, 0x40, 0x36, 0xde, 0x80, 0x15, 0x65, 0x15, 0xca, 0x2b, + 0x73, 0x43, 0x9a, 0xb5, 0x9b, 0xcd, 0x88, 0xf1, 0x8c, 0x99, 0x1f, 0x9c, 0x3d, 0xf2, 0x0c, 0x5c, + 0x78, 0x04, 0xae, 0xbc, 0x02, 0x27, 0x8e, 0x3c, 0x02, 0x5a, 0x38, 0xf3, 0x02, 0x5c, 0x50, 0x55, + 0x4f, 0xbb, 0xc7, 0xde, 0xd9, 0xcd, 0x2a, 0xe2, 0xd6, 0x5f, 0x55, 0x4d, 0x75, 0xd5, 0xd7, 0xd5, + 0x55, 0x3d, 0xd0, 0x5d, 0xe4, 0xc7, 0x61, 0x30, 0xdd, 0x5d, 0x24, 0x71, 0x16, 0x8b, 0x56, 0x10, + 0x65, 0x32, 0x89, 0xfc, 0xd0, 0x4b, 0xc1, 0xc6, 0x78, 0x29, 0x5c, 0x68, 0x3e, 0x89, 0xc3, 0x7c, + 0x1e, 0xa5, 0xae, 0xd5, 0xb7, 0x07, 0x0e, 0x6a, 0x28, 0x04, 0x38, 0xcf, 0xe4, 0x69, 0xea, 0xda, + 0x7d, 0x7b, 0xd0, 0x46, 0x5e, 0x8b, 0xbb, 0x50, 0x7f, 0x9c, 0x65, 0x49, 0xea, 0xd6, 0xfa, 0xf6, + 0xa0, 0x73, 0x7f, 0x7b, 0x57, 0xbb, 0xdb, 0x25, 0x31, 0x2a, 0x25, 0xf9, 0xc4, 0xd8, 0x4f, 0x82, + 0xe8, 0xc4, 0x75, 0xfa, 0xd6, 0xa0, 0x8b, 0x1a, 0x7a, 0xcf, 0xa1, 0x3d, 0x0e, 0x4e, 0x22, 0x39, + 0xa3, 0xad, 0xef, 0x80, 0xfd, 0x22, 0xa6, 0x6d, 0xad, 0x41, 0xe7, 0xfe, 0x96, 0x71, 0x85, 0xf1, + 0x12, 0x49, 0x43, 0x06, 0x87, 0xf2, 0xc4, 0xad, 0x55, 0x1a, 0x1c, 0xca, 0x13, 0xef, 0x11, 0x6c, + 0x63, 0xbc, 0x1c, 0xcd, 0x64, 0x94, 0x05, 0xdf, 0x06, 0x32, 0xe1, 0xa0, 0x31, 0x5e, 0xea, 0x5c, + 0x78, 0xbd, 0x4a, 0xa4, 0x66, 0x12, 0xf1, 0x3e, 0x05, 0xe7, 0x85, 0x1f, 0x24, 0x62, 0x1b, 0x6a, + 0xa3, 0x21, 0x87, 0xe0, 0x60, 0x6d, 0x34, 0x14, 0xd7, 0xc1, 0x7e, 0x26, 0x4f, 0x5d, 0xbb, 0x6f, + 0x0d, 0xda, 0x48, 0x4b, 0xd1, 0x83, 0xfa, 0x93, 0x38, 0x8f, 0x32, 0x0e, 0xc3, 0x41, 0x05, 0xbc, + 0x03, 0x68, 0xd3, 0xf7, 0x4f, 0x03, 0x19, 0xce, 0x84, 0xa7, 0x9c, 0x15, 0x99, 0x94, 0x48, 0x21, + 0x29, 0xaa, 0x8d, 0x7a, 0x50, 0x67, 0x63, 0x76, 0xd3, 0x46, 0x05, 0xbc, 0x2f, 0x01, 0x48, 0x9b, + 0x2a, 0x3f, 0x77, 0xa1, 0xce, 0x88, 0xa3, 0x3f, 0xef, 0x48, 0x29, 0x2f, 0xf0, 0xf4, 0x1e, 0xd4, + 0x47, 0x51, 0xf6, 0xf0, 0x01, 0xa9, 0x27, 0x7e, 0x98, 0x4b, 0x8e, 0xc6, 0x46, 0x05, 0xbc, 0x1c, + 0x5a, 0x6c, 0x47, 0xbc, 0xaf, 0x1c, 0x58, 0x25, 0x07, 0x24, 0x25, 0x2e, 0x87, 0x3a, 0x4f, 0x06, + 0xe2, 0x16, 0x34, 0x30, 0x5e, 0x1a, 0x4a, 0x0a, 0x24, 0xde, 0xd7, 0xbb, 0x38, 0x9c, 0xf3, 0x35, + 0x13, 0x2a, 0x47, 0xa1, 0xb7, 0xfd, 0x06, 0xe0, 0x8b, 0x24, 0xce, 0x17, 0x4c, 0x9a, 0x18, 0x40, + 0x9d, 0x51, 0x91, 0x9f, 0x30, 0x1f, 0xe9, 0xd8, 0x50, 0x19, 0x54, 0x93, 0x4e, 0x87, 0x33, 0xce, + 0xe7, 0x1c, 0x89, 0x8d, 0xb4, 0xf4, 0x7e, 0xb4, 0xa0, 0x35, 0xf1, 0xc3, 0x95, 0x7a, 0xe2, 0x87, + 0x45, 0xde, 0xb4, 0x5c, 0x77, 0x63, 0x6b, 0x37, 0xef, 0x42, 0xeb, 0x69, 0x18, 0xfb, 0x19, 0x19, + 0x93, 0x2f, 0x0b, 0x57, 0x58, 0xec, 0x01, 0x0c, 0xe5, 0x34, 0x98, 0xfb, 0x21, 0x69, 0x55, 0x72, + 0x6f, 0x9b, 0x38, 0x0b, 0x1d, 0x96, 0x8c, 0xbc, 0x4f, 0xa0, 0x59, 0xa0, 0x6a, 0xee, 0x49, 0x3a, + 0x9e, 0xfa, 0xa1, 0xd4, 0x51, 0x30, 0xf0, 0xbe, 0x86, 0x2d, 0x75, 0xd3, 0xe8, 0xce, 0x8c, 0x65, + 0x76, 0x85, 0x52, 0xbc, 0xd2, 0xed, 0xf3, 0x7e, 0xb1, 0xc0, 0xa1, 0x95, 0x76, 0x60, 0x19, 0x07, + 0x02, 0x9c, 0xa3, 0xd3, 0x85, 0x2c, 0x58, 0xe5, 0xb5, 0xe8, 0x43, 0x67, 0x9c, 0xd1, 0xe5, 0x54, + 0x91, 0xab, 0xed, 0xca, 0x22, 0xe2, 0x6b, 0x14, 0x65, 0xe6, 0xb8, 0x6d, 0x5c, 0x61, 0x71, 0x1b, + 0xda, 0xfb, 0x71, 0x1c, 0x2a, 0x65, 0xbd, 0x6f, 0x0d, 0x5a, 0x68, 0x04, 0x62, 0x07, 0x40, 0x33, + 0x9b, 0x4b, 0xb7, 0xc1, 0x5c, 0x97, 0x24, 0xde, 0x3d, 0x68, 0x52, 0xa4, 0xcf, 0xfd, 0x85, 0xc9, + 0xcd, 0xba, 0x2c, 0xb7, 0x7f, 0x2d, 0xe8, 0x7e, 0x95, 0xcb, 0xe4, 0x14, 0xe5, 0xf7, 0xb9, 0x4c, + 0x33, 0xe2, 0x96, 0xb1, 0xae, 0x65, 0x06, 0x54, 0xb5, 0xe3, 0x97, 0x7e, 0x32, 0x53, 0x4c, 0x39, + 0x58, 0x20, 0xca, 0xd5, 0x70, 0x9e, 0x72, 0xae, 0x2d, 0x2c, 0x8b, 0xb8, 0xde, 0xe5, 0x3c, 0xce, + 0x74, 0x32, 0x05, 0x12, 0x03, 0xb8, 0x76, 0xf0, 0x6a, 0x1a, 0xe6, 0x33, 0x89, 0xf1, 0x52, 0x7d, + 0xdd, 0x60, 0x83, 0x4d, 0xb1, 0xf8, 0x00, 0xb6, 0x0b, 0x91, 0xee, 0xab, 0x4d, 0x36, 0xdc, 0x90, + 0x8a, 0x3d, 0xe8, 0x1e, 0xcc, 0x8f, 0xe5, 0x6c, 0x26, 0x67, 0x43, 0x3f, 0xf3, 0xdd, 0x16, 0xe7, + 0xbd, 0xd1, 0xe5, 0xd6, 0x4c, 0xbc, 0x9f, 0x2c, 0xd8, 0x2a, 0xb2, 0x4f, 0x17, 0x71, 0x94, 0x4a, + 0x3a, 0xe2, 0x83, 0x24, 0xd1, 0x47, 0x7c, 0x90, 0x24, 0xe2, 0x1e, 0x34, 0x51, 0xa6, 0x79, 0x98, + 0xe9, 0x2a, 0xb9, 0x69, 0x3c, 0xea, 0x6f, 0xf3, 0x30, 0x43, 0x6d, 0x25, 0x3e, 0x83, 0xed, 0xb5, + 0x3a, 0x54, 0x0d, 0xbf, 0x73, 0xff, 0x1d, 0xf3, 0xdd, 0x9a, 0x1e, 0x37, 0xcc, 0xbd, 0x7f, 0x6c, + 0xe8, 0x94, 0x3c, 0xaf, 0x8a, 0x8c, 0xf8, 0xd9, 0x2a, 0x8a, 0xec, 0x0e, 0x0f, 0x9b, 0x0b, 0x5a, + 0x3d, 0xf5, 0xa4, 0x2e, 0x58, 0x87, 0x45, 0x59, 0x5a, 0x87, 0xa6, 0x11, 0xda, 0x97, 0x35, 0x42, + 0x1a, 0x5d, 0x2f, 0xfd, 0xe8, 0x44, 0xce, 0xb8, 0x2c, 0x5b, 0xa8, 0xa1, 0xd8, 0x35, 0x5d, 0x81, + 0xcf, 0x71, 0xad, 0xd7, 0x68, 0x0d, 0x9a, 0xce, 0xa1, 0xba, 0xdc, 0x68, 0x48, 0x67, 0xc5, 0xf5, + 0xa2, 0x90, 0x78, 0x08, 0x1d, 0xd3, 0xbe, 0xd2, 0xe2, 0x88, 0x7a, 0xc6, 0x95, 0x51, 0x62, 0xd9, + 0x50, 0x7c, 0xbe, 0x39, 0x97, 0xdc, 0x36, 0x47, 0xe1, 0xae, 0x65, 0x5e, 0xd2, 0xe3, 0xe6, 0x1c, + 0xdb, 0x2b, 0x0d, 0x4a, 0x17, 0xf8, 0xe3, 0x1b, 0xe6, 0xe3, 0x95, 0x0a, 0x4b, 0xe3, 0xf4, 0x41, + 0x79, 0x96, 0xb8, 0x1d, 0xfe, 0xa6, 0xb7, 0xce, 0x9c, 0xd2, 0x61, 0x79, 0xe6, 0xec, 0x95, 0x06, + 0x99, 0xdb, 0xdd, 0xdc, 0x68, 0xa5, 0x42, 0x63, 0xe5, 0xfd, 0x5a, 0x83, 0xad, 0xd1, 0x7c, 0x11, + 0x27, 0x59, 0xe9, 0x16, 0x8e, 0xa2, 0x99, 0x7c, 0xa5, 0x6f, 0x21, 0x83, 0xea, 0x41, 0xc5, 0xdd, + 0x90, 0x6e, 0x23, 0xdf, 0x3e, 0x07, 0x15, 0x28, 0x9d, 0x80, 0xb3, 0x76, 0x02, 0xb7, 0xa1, 0xad, + 0xca, 0x8d, 0x54, 0x75, 0x56, 0x19, 0x81, 0x7a, 0x68, 0x2c, 0x79, 0xb8, 0x37, 0x79, 0xb8, 0x6b, + 0x48, 0x9d, 0x47, 0x99, 0xb1, 0xb2, 0xc5, 0xca, 0x92, 0x84, 0xf4, 0x47, 0xc1, 0x5c, 0xa6, 0x99, + 0x3f, 0x5f, 0xd0, 0x55, 0xb6, 0x07, 0x36, 0x96, 0x24, 0x74, 0x8b, 0x39, 0x89, 0x27, 0x89, 0xf4, + 0x33, 0x39, 0x7b, 0x9c, 0xf1, 0x09, 0xda, 0xb8, 0x21, 0x25, 0x3b, 0x4e, 0xcb, 0xd8, 0x81, 0xb2, + 0x5b, 0x97, 0x7a, 0xbf, 0xd5, 0x40, 0x28, 0xce, 0xb8, 0xf3, 0xfd, 0x7f, 0xc4, 0x5d, 0x4e, 0xd0, + 0x3a, 0x0d, 0xcd, 0x73, 0x34, 0xdc, 0x82, 0x06, 0xc7, 0xa3, 0x29, 0x28, 0x10, 0x35, 0x4a, 0xd3, + 0xa6, 0x15, 0x7f, 0x16, 0x96, 0x45, 0xc2, 0x83, 0x6e, 0x69, 0x46, 0x50, 0x81, 0x93, 0xef, 0x35, + 0x59, 0x05, 0x89, 0x70, 0x45, 0x12, 0x3b, 0x95, 0x24, 0x4e, 0xa0, 0x77, 0x94, 0xf8, 0x51, 0x1a, + 0xfa, 0x99, 0xa4, 0xf0, 0xdf, 0x84, 0xc5, 0x8a, 0x57, 0xad, 0xf7, 0x21, 0xdc, 0xdc, 0xf0, 0x6b, + 0xda, 0x2b, 0xd1, 0x6a, 0x33, 0xad, 0xb4, 0xf4, 0xc6, 0x70, 0x63, 0x65, 0x3a, 0x1a, 0xbe, 0x51, + 0x04, 0xe7, 0x9d, 0x7e, 0x54, 0xca, 0x8b, 0x9d, 0x16, 0xdb, 0x57, 0xc5, 0xba, 0x0f, 0x6e, 0x71, + 0xf7, 0xd4, 0x93, 0xba, 0x88, 0x60, 0x12, 0xc8, 0x25, 0xd9, 0x1f, 0xfa, 0x73, 0x59, 0x04, 0xc1, + 0x6b, 0x92, 0xf1, 0x78, 0xa9, 0xf1, 0x43, 0x9c, 0xd7, 0xde, 0xdf, 0x16, 0xf4, 0xaa, 0x9c, 0xf0, + 0x7b, 0x29, 0x94, 0xbe, 0x1a, 0x28, 0x2d, 0x54, 0x40, 0x3c, 0x82, 0xfa, 0x0f, 0x81, 0x5c, 0xea, + 0x81, 0xe2, 0x95, 0xde, 0x7a, 0x17, 0x44, 0x82, 0xea, 0x03, 0x2a, 0xaf, 0xc7, 0xd3, 0x2c, 0x88, + 0x23, 0xfd, 0x7a, 0x54, 0x88, 0xf6, 0xd9, 0x0f, 0xe3, 0xe9, 0x77, 0xdc, 0xb7, 0x1d, 0x54, 0xa0, + 0xa2, 0x5c, 0xea, 0x57, 0x2c, 0x97, 0x46, 0xf5, 0x9d, 0xb3, 0x34, 0x57, 0xa5, 0x09, 0xff, 0xda, + 0x13, 0x53, 0x77, 0x4c, 0x3f, 0xd5, 0xf8, 0x8e, 0xb9, 0xea, 0x99, 0x62, 0x5e, 0x63, 0x1a, 0xd2, + 0xd3, 0x88, 0x96, 0x13, 0x3f, 0x54, 0x8d, 0xab, 0x8d, 0x2b, 0xfc, 0x9a, 0x9b, 0x79, 0x3e, 0xd9, + 0x46, 0x55, 0xb2, 0xfb, 0xd7, 0x7f, 0x3f, 0xdb, 0xb1, 0xfe, 0x38, 0xdb, 0xb1, 0xfe, 0x3c, 0xdb, + 0xb1, 0x7e, 0xfe, 0x6b, 0xe7, 0xad, 0xe3, 0x06, 0xff, 0xc9, 0x7d, 0xfc, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xb8, 0x93, 0x5b, 0x24, 0xd9, 0x0d, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -2985,6 +3044,16 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x50 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x48 + } if len(m.ColumnKeys) > 0 { for iNdEx := len(m.ColumnKeys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ColumnKeys[iNdEx]) @@ -3104,6 +3173,16 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x58 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x50 + } if len(m.StringValues) > 0 { for iNdEx := len(m.StringValues) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.StringValues[iNdEx]) @@ -3446,6 +3525,16 @@ func (m *ImportRoaringRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FieldCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.FieldCreatedAt)) + i-- + dAtA[i] = 0x30 + } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x28 + } if m.Block != 0 { i = encodeVarintPublic(dAtA, i, uint64(m.Block)) i-- @@ -3509,6 +3598,11 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.IndexCreatedAt != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.IndexCreatedAt)) + i-- + dAtA[i] = 0x30 + } if len(m.ColumnIDs) > 0 { dAtA36 := make([]byte, len(m.ColumnIDs)*10) var j35 int @@ -4080,6 +4174,12 @@ func (m *ImportRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4132,6 +4232,12 @@ func (m *ImportValueRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4270,6 +4376,12 @@ func (m *ImportRoaringRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPublic(uint64(m.Block)) } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } + if m.FieldCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.FieldCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -4306,6 +4418,9 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } + if m.IndexCreatedAt != 0 { + n += 1 + sovPublic(uint64(m.IndexCreatedAt)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7525,6 +7640,44 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } m.ColumnKeys = append(m.ColumnKeys, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -7932,6 +8085,44 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.StringValues = append(m.StringValues, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -8771,6 +8962,44 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldCreatedAt", wireType) + } + m.FieldCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -9016,6 +9245,25 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IndexCreatedAt", wireType) + } + m.IndexCreatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.IndexCreatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 81220fbfe..698581215 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -116,6 +116,7 @@ message QueryResult { message ImportRequest { string Index = 1; + string Field = 2; uint64 Shard = 3; repeated uint64 RowIDs = 4; @@ -123,6 +124,8 @@ message ImportRequest { repeated string RowKeys = 7; repeated string ColumnKeys = 8; repeated int64 Timestamps = 6; + int64 IndexCreatedAt = 9; + int64 FieldCreatedAt = 10; } message ImportValueRequest { @@ -134,6 +137,8 @@ message ImportValueRequest { repeated int64 Values = 6; repeated double FloatValues = 8; repeated string StringValues = 9; + int64 IndexCreatedAt = 10; + int64 FieldCreatedAt = 11; } message TranslateKeysRequest { @@ -166,6 +171,9 @@ message ImportRoaringRequest { repeated ImportRoaringRequestView views = 2; string Action = 3; uint64 Block = 4; + int64 IndexCreatedAt = 5; + int64 FieldCreatedAt = 6; + } message ImportColumnAttrsRequest { @@ -174,4 +182,5 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; + int64 IndexCreatedAt = 6; } diff --git a/metrics.go b/metrics.go index c1888e0f6..8d58f8a73 100644 --- a/metrics.go +++ b/metrics.go @@ -58,4 +58,11 @@ const ( MetricStackInuse = "stack_inuse" MetricMallocs = "mallocs" MetricFrees = "frees" + MetricTransactionStart = "transaction_start" + MetricTransactionEnd = "trasaction_end" + MetricTransactionBlocked = "transaction_blocked" + MetricExclusiveTransactionRequest = "transaction_exclusive_request" + MetricExclusiveTransactionActive = "transaction_exclusive_active" + MetricExclusiveTransactionEnd = "trasaction_exclusive_end" + MetricExclusiveTransactionBlocked = "transaction_exclusive_blocked" ) diff --git a/pilosa.go b/pilosa.go index d7eb2f3c8..53733d9ce 100644 --- a/pilosa.go +++ b/pilosa.go @@ -17,6 +17,7 @@ package pilosa import ( "encoding/json" "regexp" + "time" "github.com/pkg/errors" ) @@ -65,6 +66,9 @@ var ( // we won't need this error at all by 2.0 though. ErrClusterDoesNotOwnShard = errors.New("node does not own shard") + // ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match + ErrPreconditionFailed = errors.New("precondition failed") + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") @@ -124,6 +128,15 @@ func newNotFoundError(err error) NotFoundError { return NotFoundError{err} } +type PreconditionFailedError struct { + error +} + +// newPreconditionFailedError returns err wrapped in a PreconditionFailedError. +func newPreconditionFailedError(err error) PreconditionFailedError { + return PreconditionFailedError{err} +} + // Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`) @@ -191,6 +204,10 @@ func stringSlicesAreEqual(a, b []string) bool { return true } +func timestamp() int64 { + return time.Now().UnixNano() +} + // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/pql/ast.go b/pql/ast.go index e4163a66e..e93a2aaf5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -134,6 +134,13 @@ func (q *Query) validateArgField(elem *callStackElem) { } func (q *Query) addVal(val interface{}) { + if vs, ok := val.(string); ok { + vsu, err := Unquote(vs) + if err != nil { + panic(err) + } + val = vsu + } elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) diff --git a/pql/parser.go b/pql/parser.go index b3ad984f9..8a5907dbf 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -18,7 +18,9 @@ import ( "fmt" "io" "io/ioutil" + "strconv" "strings" + "unicode/utf8" "github.com/pkg/errors" ) @@ -95,3 +97,79 @@ func (p *parser) Parse() (*Query, error) { return &p.Query, nil } + +// Unquote interprets s as a single-quoted, double-quoted, or +// backquoted Go string literal, returning the string value that s +// quotes. It is a copy of stdlib's strconv.Unquote, but modified so +// that if s is single-quoted, it can still be a string rather than +// only character literal. This version of Unquote also accepts +// unquoted strings and passes them back unchanged. +func Unquote(s string) (string, error) { + n := len(s) + if n < 2 { + return s, nil + } + quote := s[0] + if quote != '"' && quote != '\'' && quote != '`' { + return s, nil + } + if quote != s[n-1] { + return "", strconv.ErrSyntax + } + s = s[1 : n-1] + + if quote == '`' { + if contains(s, '`') { + return "", strconv.ErrSyntax + } + if contains(s, '\r') { + // -1 because we know there is at least one \r to remove. + buf := make([]byte, 0, len(s)-1) + for i := 0; i < len(s); i++ { + if s[i] != '\r' { + buf = append(buf, s[i]) + } + } + return string(buf), nil + } + return s, nil + } + if quote != '"' && quote != '\'' { + return "", strconv.ErrSyntax + } + if contains(s, '\n') { + return "", strconv.ErrSyntax + } + + // Is it trivial? Avoid allocation. + if !contains(s, '\\') && !contains(s, quote) { + switch quote { + case '"', '\'': + if utf8.ValidString(s) { + return s, nil + } + } + } + + var runeTmp [utf8.UTFMax]byte + buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. + for len(s) > 0 { + c, multibyte, ss, err := strconv.UnquoteChar(s, quote) + if err != nil { + return "", err + } + s = ss + if c < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(c)) + } else { + n := utf8.EncodeRune(runeTmp[:], c) + buf = append(buf, runeTmp[:n]...) + } + } + return string(buf), nil +} + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + return strings.ContainsRune(s, rune(c)) +} diff --git a/pql/parser_test.go b/pql/parser_test.go index 29c358865..8e5cf8cc4 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -16,6 +16,7 @@ package pql_test import ( "reflect" + "strings" "testing" "github.com/pilosa/pilosa/v2/pql" @@ -173,7 +174,7 @@ func TestParser_Parse(t *testing.T) { // Parse with condition arguments. t.Run("WithCondition", func(t *testing.T) { - q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) + q, err := pql.ParseString(`Row(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null, n == null)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -185,10 +186,77 @@ func TestParser_Parse(t *testing.T) { "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, "m": &pql.Condition{Op: pql.NEQ, Value: nil}, + "n": &pql.Condition{Op: pql.EQ, Value: nil}, }, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) + +} + +func TestUnquote(t *testing.T) { + tests := []struct { + name string + value string + exp string + expErr string + }{ + { + name: "simple double", + value: `"hello"`, + exp: "hello", + }, + { + name: "simple single", + value: `'hello'`, + exp: "hello", + }, + { + name: "double with esc", + value: `"he\"llo"`, + exp: "he\"llo", + }, + { + name: "single with esc", + value: `'he\'llo'`, + exp: "he'llo", + }, + { + name: "single with backslash and esc", + value: `'he\\\'llo'`, + exp: `he\'llo`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := pql.Unquote(test.value) + if testErr(t, test.expErr, err) { + return + } + if got != test.exp { + t.Errorf("exp: '%s'\ngot: '%s'", test.exp, got) + } + }) + } + +} + +func testErr(t *testing.T, exp string, actual error) (done bool) { + t.Helper() + if exp == "" && actual == nil { + return false + } + if exp == "" && actual != nil { + t.Fatalf("unexpected error: %v", actual) + } + if exp != "" && actual == nil { + t.Fatalf("expected error like '%s'", exp) + } + if !strings.Contains(actual.Error(), exp) { + t.Fatalf("unmatched errs exp/got\n%s\n%v", exp, actual) + } + return true } diff --git a/pql/pql.peg b/pql/pql.peg index ecb435549..7e6c039bd 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -64,8 +64,8 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } ) itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) } - / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } + / < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) } + / < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) } ) float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } @@ -74,8 +74,8 @@ decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) } ) -doublequotedstring <- ( '\\"' / '\\\\' / [^"] )* -singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )* +doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* +singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(buffer[begin:end]) } @@ -83,12 +83,12 @@ reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' col <- ( {p.addPosNum("_col", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_col", buffer[begin:end])} - / '"' '"' {p.addPosStr("_col", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])} ) row <- ( {p.addPosNum("_row", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_row", buffer[begin:end])} - / '"' '"' {p.addPosStr("_row", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])} ) open <- '(' sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index c1e2db0c7..3cba1b702 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -536,8 +536,7 @@ func (p *PQL) Execute() { case ruleAction46: p.addVal(buffer[begin:end]) case ruleAction47: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(buffer[begin:end]) case ruleAction48: p.addVal(buffer[begin:end]) case ruleAction49: @@ -836,42 +835,42 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l19 l20: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('\'') { - goto l23 - } - position++ { position24 := position + if buffer[position] != rune('\'') { + goto l23 + } + position++ if !_rules[rulesinglequotedstring]() { goto l23 } + if buffer[position] != rune('\'') { + goto l23 + } + position++ add(rulePegText, position24) } - if buffer[position] != rune('\'') { - goto l23 - } - position++ { add(ruleAction59, position) } goto l19 l23: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('"') { - goto l16 - } - position++ { position26 := position + if buffer[position] != rune('"') { + goto l16 + } + position++ if !_rules[ruledoublequotedstring]() { goto l16 } + if buffer[position] != rune('"') { + goto l16 + } + position++ add(rulePegText, position26) } - if buffer[position] != rune('"') { - goto l16 - } - position++ { add(ruleAction60, position) } @@ -2433,7 +2432,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position, tokenIndex = position209, tokenIndex209 return false }, - /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / ('\'' '\'' Action48))> */ + /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / (<('\'' singlequotedstring '\'')> Action48))> */ func() bool { position228, tokenIndex228 := position, tokenIndex { @@ -2599,21 +2598,21 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l230 l254: position, tokenIndex = position230, tokenIndex230 - if buffer[position] != rune('\'') { - goto l228 - } - position++ { position257 := position + if buffer[position] != rune('\'') { + goto l228 + } + position++ if !_rules[rulesinglequotedstring]() { goto l228 } + if buffer[position] != rune('\'') { + goto l228 + } + position++ add(rulePegText, position257) } - if buffer[position] != rune('\'') { - goto l228 - } - position++ { add(ruleAction48, position) } @@ -2630,7 +2629,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 21 decimal <- <((<('-'? [0-9]+ ('.' [0-9]*)?)> Action51) / (<('-'? '.' [0-9]+)> Action52))> */ nil, - /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ + /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('"' / '\\') .))*> */ func() bool { { position262 := position @@ -2660,16 +2659,49 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position++ goto l265 l267: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l268 + } + position++ + if buffer[position] != rune('n') { + goto l268 + } + position++ + goto l265 + l268: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l269 + } + position++ + if buffer[position] != rune('t') { + goto l269 + } + position++ + goto l265 + l269: position, tokenIndex = position265, tokenIndex265 { - position268, tokenIndex268 := position, tokenIndex - if buffer[position] != rune('"') { - goto l268 + position270, tokenIndex270 := position, tokenIndex + { + position271, tokenIndex271 := position, tokenIndex + if buffer[position] != rune('"') { + goto l272 + } + position++ + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if buffer[position] != rune('\\') { + goto l270 + } + position++ } - position++ + l271: goto l264 - l268: - position, tokenIndex = position268, tokenIndex268 + l270: + position, tokenIndex = position270, tokenIndex270 } if !matchDot() { goto l264 @@ -2684,795 +2716,828 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } return true }, - /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ + /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('\'' / '\\') .))*> */ func() bool { { - position270 := position - l271: + position274 := position + l275: { - position272, tokenIndex272 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex { - position273, tokenIndex273 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex if buffer[position] != rune('\\') { - goto l274 + goto l278 } position++ if buffer[position] != rune('\'') { - goto l274 + goto l278 } position++ - goto l273 - l274: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ - goto l273 - l275: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l280 + } + position++ + if buffer[position] != rune('n') { + goto l280 + } + position++ + goto l277 + l280: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l281 + } + position++ + if buffer[position] != rune('t') { + goto l281 + } + position++ + goto l277 + l281: + position, tokenIndex = position277, tokenIndex277 { - position276, tokenIndex276 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l276 + position282, tokenIndex282 := position, tokenIndex + { + position283, tokenIndex283 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l284 + } + position++ + goto l283 + l284: + position, tokenIndex = position283, tokenIndex283 + if buffer[position] != rune('\\') { + goto l282 + } + position++ } - position++ - goto l272 - l276: - position, tokenIndex = position276, tokenIndex276 + l283: + goto l276 + l282: + position, tokenIndex = position282, tokenIndex282 } if !matchDot() { - goto l272 + goto l276 } } - l273: - goto l271 - l272: - position, tokenIndex = position272, tokenIndex272 + l277: + goto l275 + l276: + position, tokenIndex = position276, tokenIndex276 } - add(rulesinglequotedstring, position270) + add(rulesinglequotedstring, position274) } return true }, /* 24 fieldExpr <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex { - position278 := position + position286 := position { - position279, tokenIndex279 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l280 + goto l288 } position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l288: + position, tokenIndex = position287, tokenIndex287 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l281 + goto l289 } position++ - goto l279 - l281: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l289: + position, tokenIndex = position287, tokenIndex287 if buffer[position] != rune('_') { - goto l277 + goto l285 } position++ } - l279: - l282: + l287: + l290: { - position283, tokenIndex283 := position, tokenIndex + position291, tokenIndex291 := position, tokenIndex { - position284, tokenIndex284 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l285 + goto l293 } position++ - goto l284 - l285: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l293: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l286 + goto l294 } position++ - goto l284 - l286: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l294: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l287 + goto l295 } position++ - goto l284 - l287: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l295: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('_') { - goto l288 + goto l296 } position++ - goto l284 - l288: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l296: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('-') { - goto l283 + goto l291 } position++ } - l284: - goto l282 - l283: - position, tokenIndex = position283, tokenIndex283 + l292: + goto l290 + l291: + position, tokenIndex = position291, tokenIndex291 } - add(rulefieldExpr, position278) + add(rulefieldExpr, position286) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l285: + position, tokenIndex = position285, tokenIndex285 return false }, /* 25 field <- <(<(fieldExpr / reserved)> Action53)> */ func() bool { - position289, tokenIndex289 := position, tokenIndex + position297, tokenIndex297 := position, tokenIndex { - position290 := position + position298 := position { - position291 := position + position299 := position { - position292, tokenIndex292 := position, tokenIndex + position300, tokenIndex300 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l293 + goto l301 } - goto l292 - l293: - position, tokenIndex = position292, tokenIndex292 + goto l300 + l301: + position, tokenIndex = position300, tokenIndex300 { - position294 := position + position302 := position { - position295, tokenIndex295 := position, tokenIndex + position303, tokenIndex303 := position, tokenIndex if buffer[position] != rune('_') { - goto l296 + goto l304 } position++ if buffer[position] != rune('r') { - goto l296 + goto l304 } position++ if buffer[position] != rune('o') { - goto l296 + goto l304 } position++ if buffer[position] != rune('w') { - goto l296 + goto l304 } position++ - goto l295 - l296: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l304: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l297 + goto l305 } position++ if buffer[position] != rune('c') { - goto l297 + goto l305 } position++ if buffer[position] != rune('o') { - goto l297 + goto l305 } position++ if buffer[position] != rune('l') { - goto l297 + goto l305 } position++ - goto l295 - l297: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l305: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l298 + goto l306 } position++ if buffer[position] != rune('s') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ if buffer[position] != rune('a') { - goto l298 + goto l306 } position++ if buffer[position] != rune('r') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ - goto l295 - l298: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l306: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l299 + goto l307 } position++ if buffer[position] != rune('e') { - goto l299 + goto l307 } position++ if buffer[position] != rune('n') { - goto l299 + goto l307 } position++ if buffer[position] != rune('d') { - goto l299 + goto l307 } position++ - goto l295 - l299: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l307: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('i') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('e') { - goto l300 + goto l308 } position++ if buffer[position] != rune('s') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('a') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('p') { - goto l300 + goto l308 } position++ - goto l295 - l300: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l308: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l289 + goto l297 } position++ if buffer[position] != rune('f') { - goto l289 + goto l297 } position++ if buffer[position] != rune('i') { - goto l289 + goto l297 } position++ if buffer[position] != rune('e') { - goto l289 + goto l297 } position++ if buffer[position] != rune('l') { - goto l289 + goto l297 } position++ if buffer[position] != rune('d') { - goto l289 + goto l297 } position++ } - l295: - add(rulereserved, position294) + l303: + add(rulereserved, position302) } } - l292: - add(rulePegText, position291) + l300: + add(rulePegText, position299) } { add(ruleAction53, position) } - add(rulefield, position290) + add(rulefield, position298) } return true - l289: - position, tokenIndex = position289, tokenIndex289 + l297: + position, tokenIndex = position297, tokenIndex297 return false }, /* 26 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 27 posfield <- <( Action54)> */ func() bool { - position303, tokenIndex303 := position, tokenIndex + position311, tokenIndex311 := position, tokenIndex { - position304 := position + position312 := position { - position305 := position + position313 := position if !_rules[rulefieldExpr]() { - goto l303 + goto l311 } - add(rulePegText, position305) + add(rulePegText, position313) } { add(ruleAction54, position) } - add(ruleposfield, position304) + add(ruleposfield, position312) } return true - l303: - position, tokenIndex = position303, tokenIndex303 + l311: + position, tokenIndex = position311, tokenIndex311 return false }, /* 28 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position307, tokenIndex307 := position, tokenIndex + position315, tokenIndex315 := position, tokenIndex { - position308 := position + position316 := position { - position309, tokenIndex309 := position, tokenIndex + position317, tokenIndex317 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l310 + goto l318 } position++ - l311: + l319: { - position312, tokenIndex312 := position, tokenIndex + position320, tokenIndex320 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l312 + goto l320 } position++ - goto l311 - l312: - position, tokenIndex = position312, tokenIndex312 + goto l319 + l320: + position, tokenIndex = position320, tokenIndex320 } - goto l309 - l310: - position, tokenIndex = position309, tokenIndex309 + goto l317 + l318: + position, tokenIndex = position317, tokenIndex317 if buffer[position] != rune('0') { - goto l307 + goto l315 } position++ } - l309: - add(ruleuint, position308) + l317: + add(ruleuint, position316) } return true - l307: - position, tokenIndex = position307, tokenIndex307 + l315: + position, tokenIndex = position315, tokenIndex315 return false }, - /* 29 col <- <(( Action55) / ('\'' '\'' Action56) / ('"' '"' Action57))> */ + /* 29 col <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ func() bool { - position313, tokenIndex313 := position, tokenIndex + position321, tokenIndex321 := position, tokenIndex { - position314 := position + position322 := position { - position315, tokenIndex315 := position, tokenIndex + position323, tokenIndex323 := position, tokenIndex { - position317 := position + position325 := position if !_rules[ruleuint]() { - goto l316 + goto l324 } - add(rulePegText, position317) + add(rulePegText, position325) } { add(ruleAction55, position) } - goto l315 - l316: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('\'') { - goto l319 - } - position++ + goto l323 + l324: + position, tokenIndex = position323, tokenIndex323 { - position320 := position - if !_rules[rulesinglequotedstring]() { - goto l319 + position328 := position + if buffer[position] != rune('\'') { + goto l327 } - add(rulePegText, position320) + position++ + if !_rules[rulesinglequotedstring]() { + goto l327 + } + if buffer[position] != rune('\'') { + goto l327 + } + position++ + add(rulePegText, position328) } - if buffer[position] != rune('\'') { - goto l319 - } - position++ { add(ruleAction56, position) } - goto l315 - l319: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('"') { - goto l313 - } - position++ + goto l323 + l327: + position, tokenIndex = position323, tokenIndex323 { - position322 := position - if !_rules[ruledoublequotedstring]() { - goto l313 + position330 := position + if buffer[position] != rune('"') { + goto l321 } - add(rulePegText, position322) + position++ + if !_rules[ruledoublequotedstring]() { + goto l321 + } + if buffer[position] != rune('"') { + goto l321 + } + position++ + add(rulePegText, position330) } - if buffer[position] != rune('"') { - goto l313 - } - position++ { add(ruleAction57, position) } } - l315: - add(rulecol, position314) + l323: + add(rulecol, position322) } return true - l313: - position, tokenIndex = position313, tokenIndex313 + l321: + position, tokenIndex = position321, tokenIndex321 return false }, - /* 30 row <- <(( Action58) / ('\'' '\'' Action59) / ('"' '"' Action60))> */ + /* 30 row <- <(( Action58) / (<('\'' singlequotedstring '\'')> Action59) / (<('"' doublequotedstring '"')> Action60))> */ nil, /* 31 open <- <('(' sp)> */ func() bool { - position325, tokenIndex325 := position, tokenIndex + position333, tokenIndex333 := position, tokenIndex { - position326 := position + position334 := position if buffer[position] != rune('(') { - goto l325 + goto l333 } position++ if !_rules[rulesp]() { - goto l325 + goto l333 } - add(ruleopen, position326) + add(ruleopen, position334) } return true - l325: - position, tokenIndex = position325, tokenIndex325 + l333: + position, tokenIndex = position333, tokenIndex333 return false }, /* 32 close <- <(')' sp)> */ func() bool { - position327, tokenIndex327 := position, tokenIndex + position335, tokenIndex335 := position, tokenIndex { - position328 := position + position336 := position if buffer[position] != rune(')') { - goto l327 + goto l335 } position++ if !_rules[rulesp]() { - goto l327 + goto l335 } - add(ruleclose, position328) + add(ruleclose, position336) } return true - l327: - position, tokenIndex = position327, tokenIndex327 + l335: + position, tokenIndex = position335, tokenIndex335 return false }, /* 33 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position330 := position - l331: + position338 := position + l339: { - position332, tokenIndex332 := position, tokenIndex + position340, tokenIndex340 := position, tokenIndex { - position333, tokenIndex333 := position, tokenIndex + position341, tokenIndex341 := position, tokenIndex if buffer[position] != rune(' ') { - goto l334 + goto l342 } position++ - goto l333 - l334: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l342: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\t') { - goto l335 + goto l343 } position++ - goto l333 - l335: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l343: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\n') { - goto l332 + goto l340 } position++ } - l333: - goto l331 - l332: - position, tokenIndex = position332, tokenIndex332 + l341: + goto l339 + l340: + position, tokenIndex = position340, tokenIndex340 } - add(rulesp, position330) + add(rulesp, position338) } return true }, /* 34 comma <- <(sp ',' sp)> */ func() bool { - position336, tokenIndex336 := position, tokenIndex + position344, tokenIndex344 := position, tokenIndex { - position337 := position + position345 := position if !_rules[rulesp]() { - goto l336 + goto l344 } if buffer[position] != rune(',') { - goto l336 + goto l344 } position++ if !_rules[rulesp]() { - goto l336 + goto l344 } - add(rulecomma, position337) + add(rulecomma, position345) } return true - l336: - position, tokenIndex = position336, tokenIndex336 + l344: + position, tokenIndex = position344, tokenIndex344 return false }, /* 35 lbrack <- <('[' sp)> */ func() bool { - position338, tokenIndex338 := position, tokenIndex + position346, tokenIndex346 := position, tokenIndex { - position339 := position + position347 := position if buffer[position] != rune('[') { - goto l338 + goto l346 } position++ if !_rules[rulesp]() { - goto l338 + goto l346 } - add(rulelbrack, position339) + add(rulelbrack, position347) } return true - l338: - position, tokenIndex = position338, tokenIndex338 + l346: + position, tokenIndex = position346, tokenIndex346 return false }, /* 36 rbrack <- <(sp ']' sp)> */ func() bool { - position340, tokenIndex340 := position, tokenIndex + position348, tokenIndex348 := position, tokenIndex { - position341 := position + position349 := position if !_rules[rulesp]() { - goto l340 + goto l348 } if buffer[position] != rune(']') { - goto l340 + goto l348 } position++ if !_rules[rulesp]() { - goto l340 + goto l348 } - add(rulerbrack, position341) + add(rulerbrack, position349) } return true - l340: - position, tokenIndex = position340, tokenIndex340 + l348: + position, tokenIndex = position348, tokenIndex348 return false }, /* 37 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position342, tokenIndex342 := position, tokenIndex + position350, tokenIndex350 := position, tokenIndex { - position343 := position + position351 := position { - position344, tokenIndex344 := position, tokenIndex + position352, tokenIndex352 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l345 + goto l353 } position++ - goto l344 - l345: - position, tokenIndex = position344, tokenIndex344 + goto l352 + l353: + position, tokenIndex = position352, tokenIndex352 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l342 + goto l350 } position++ } - l344: - l346: + l352: + l354: { - position347, tokenIndex347 := position, tokenIndex + position355, tokenIndex355 := position, tokenIndex { - position348, tokenIndex348 := position, tokenIndex + position356, tokenIndex356 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l349 + goto l357 } position++ - goto l348 - l349: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l357: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l350 + goto l358 } position++ - goto l348 - l350: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l358: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l347 + goto l355 } position++ } - l348: - goto l346 - l347: - position, tokenIndex = position347, tokenIndex347 + l356: + goto l354 + l355: + position, tokenIndex = position355, tokenIndex355 } - add(ruleIDENT, position343) + add(ruleIDENT, position351) } return true - l342: - position, tokenIndex = position342, tokenIndex342 + l350: + position, tokenIndex = position350, tokenIndex350 return false }, /* 38 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position351, tokenIndex351 := position, tokenIndex + position359, tokenIndex359 := position, tokenIndex { - position352 := position + position360 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ { - position353, tokenIndex353 := position, tokenIndex + position361, tokenIndex361 := position, tokenIndex if buffer[position] != rune('0') { - goto l354 + goto l362 } position++ - goto l353 - l354: - position, tokenIndex = position353, tokenIndex353 + goto l361 + l362: + position, tokenIndex = position361, tokenIndex361 if buffer[position] != rune('1') { - goto l351 + goto l359 } position++ } - l353: + l361: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('T') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune(':') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ - add(ruletimestampbasicfmt, position352) + add(ruletimestampbasicfmt, position360) } return true - l351: - position, tokenIndex = position351, tokenIndex351 + l359: + position, tokenIndex = position359, tokenIndex359 return false }, /* 39 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position355, tokenIndex355 := position, tokenIndex + position363, tokenIndex363 := position, tokenIndex { - position356 := position + position364 := position { - position357, tokenIndex357 := position, tokenIndex + position365, tokenIndex365 := position, tokenIndex if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ { - position359 := position + position367 := position if !_rules[ruletimestampbasicfmt]() { - goto l358 + goto l366 } - add(rulePegText, position359) + add(rulePegText, position367) } if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ - goto l357 - l358: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l366: + position, tokenIndex = position365, tokenIndex365 if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ { - position361 := position + position369 := position if !_rules[ruletimestampbasicfmt]() { - goto l360 + goto l368 } - add(rulePegText, position361) + add(rulePegText, position369) } if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ - goto l357 - l360: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l368: + position, tokenIndex = position365, tokenIndex365 { - position362 := position + position370 := position if !_rules[ruletimestampbasicfmt]() { - goto l355 + goto l363 } - add(rulePegText, position362) + add(rulePegText, position370) } } - l357: - add(ruletimestampfmt, position356) + l365: + add(ruletimestampfmt, position364) } return true - l355: - position, tokenIndex = position355, tokenIndex355 + l363: + position, tokenIndex = position363, tokenIndex363 return false }, /* 40 timestamp <- <( Action61)> */ @@ -3572,7 +3637,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 90 Action47 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */ nil, /* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ nil, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 48eaf10c4..4ebf1dcc9 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -238,8 +238,16 @@ func TestPEGWorking(t *testing.T) { name: "RangeEQ", input: "Row(a == 4)", ncalls: 1}, + { + name: "RangeEQNULL", + input: "Row(a == null)", + ncalls: 1}, { name: "RangeNEQ", + input: "Row(a != 4)", + ncalls: 1}, + { + name: "RangeNEQNull", input: "Row(a != null)", ncalls: 1}, { diff --git a/roaring/btree.go b/roaring/btree.go index 1051f8985..bbc429704 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -925,6 +925,15 @@ func (e *enumerator) Every(upd func(key uint64, oldV *Container, exists bool) (n if write { if nv == nil { e.t.Delete(i.k) + f, _ := e.t.Seek(e.k) + *e = *f + f.Close() + // we don't want to e.next() here; we'll + // already be on an item with key >= i.k, + // and since we just deleted the item with + // key i.k, that means key is > i.k, which + // makes it the next item. + continue } else { e.q.d[e.i].v = nv } diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 422a7b767..f0f528a7b 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -11,6 +11,7 @@ import ( "math" "math/rand" "path" + "reflect" "runtime" "runtime/debug" "strings" @@ -688,7 +689,7 @@ func TestBtreeDelete0(t *testing.T) { } func TestBtreeDelete1(t *testing.T) { - const N = 130000 + const N = 13000 for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { r := treeNew() set := r.Set @@ -787,7 +788,7 @@ func benchmarkDelRnd(b *testing.B, n int) { } func TestBtreeDelete2(t *testing.T) { - const N = 100000 + const N = 10000 for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { r := treeNew() set := r.Set @@ -998,6 +999,28 @@ func TestBtreeEnumeratorPrevSanity(t *testing.T) { } } +// TestBtreeEnumeratorEveryRegression is a regression test for a "use-after-free" bug. +// Previously, deleting a container would cause some values to be skipped (and sometimes trigger a race condition). +func TestBtreeEnumeratorEveryRegression(t *testing.T) { + r := treeNew() + + r.Set(uint64(10), getDummyC(100)) + r.Set(uint64(20), getDummyC(200)) + r.Set(uint64(30), getDummyC(300)) + + e, _ := r.Seek(0) + expect := []uint64{10, 20, 30} + var found []uint64 + _ = e.Every(func(key uint64, oldV *Container, exists bool) (*Container, bool) { + found = append(found, key) + return nil, true + }) + + if !reflect.DeepEqual(expect, found) { // Before the fix, this skipped the 20. + t.Errorf("had %v in bitmap; only found %v", expect, found) + } +} + func BenchmarkBtreeSeekSeq1e3(b *testing.B) { benchmarkSeekSeq(b, 1e3) } @@ -1445,7 +1468,7 @@ func TestBtreePut(t *testing.T) { } func TestBtreeSeek(t *testing.T) { - const N = 1 << 13 + const N = 1 << 11 tr := treeNew() for i := 0; i < N; i++ { k := 2*i + 1 diff --git a/roaring/naive_test.go b/roaring/naive_test.go index 154d0f8f5..c1accfb5e 100644 --- a/roaring/naive_test.go +++ b/roaring/naive_test.go @@ -15,6 +15,7 @@ package roaring import ( + "math/rand" "reflect" "testing" ) @@ -110,13 +111,15 @@ func TestUnionSlice(t *testing.T) { } func TestMaxInSlice(t *testing.T) { + // arbitrary, we just want to get the same values every time + r := rand.New(rand.NewSource(23)) a := []uint64{1, 4, 9, 5, 24, 13} v := maxInSlice(a) if uint64(24) != v { t.Fatalf("expected %v, but got %v", uint64(24), v) } - for i := uint64(1000); i <= uint64(100000); i++ { + for i := uint64(1000); i <= uint64(100000); i += uint64(r.Intn(35)) + 1 { a = append(a, i) if v = maxInSlice(a); v != i { t.Fatalf("expected %v, but got %v", i, v) diff --git a/roaring/roaring.go b/roaring/roaring.go index 9605a6299..db63b0edd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -160,7 +160,7 @@ type Containers interface { // replace the given container. UpdateEvery(fn func(uint64, *Container, bool) (*Container, bool)) - // Iterator returns a Contiterator which after a call to Next(), a call to Value() will + // Iterator returns a ContainterIterator which after a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a // container is found at key. Iterator(key uint64) (citer ContainerIterator, found bool) @@ -1738,7 +1738,9 @@ type baseRoaringIterator struct { currentN int currentLen int currentPointer *uint16 - currentDataOffset uint32 + currentDataOffset uint64 + prevOffset32 uint32 + chunkOffset uint64 lastDataOffset int64 lastErr error } @@ -1752,6 +1754,8 @@ func (b *baseRoaringIterator) SilenceLint() { _ = b.offsets _ = b.headers _ = b.currentIdx + _ = b.chunkOffset + _ = b.prevOffset32 } type pilosaRoaringIterator struct { @@ -1787,14 +1791,14 @@ func newOfficialRoaringIterator(data []byte) (*officialRoaringIterator, error) { r.headers = data[headerOffset:offsetOffset] // note: offsets are only actually used with the no-run headers. if r.haveRuns { - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } else { if len(r.data) < offsetOffset+int(r.keys*4) { return nil, fmt.Errorf("insufficient data for offsets (need %d bytes, found %d)", r.keys*4, len(r.data)-offsetOffset) } r.offsets = data[offsetOffset : offsetOffset+int(r.keys*4)] - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } // set key to -1; user should call Next first. r.currentIdx = -1 @@ -1838,7 +1842,11 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) { // if there's no containers, we want to act as though data started at the end // of the list of offsets, which was also empty, so we don't think the entire thing // is actually a malformed op - r.currentDataOffset = uint32(offsetEnd) + r.prevOffset32 = uint32(offsetEnd) + r.currentDataOffset = uint64(offsetEnd) + // it's possible that there's so many headers that we're actually over + // 4GB into the file already. + r.chunkOffset = r.currentDataOffset &^ ((1 << 32) - 1) // set key to -1; user should call Next first. r.currentIdx = -1 r.currentKey = ^uint64(0) @@ -1895,7 +1903,12 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentKey = binary.LittleEndian.Uint64(header[0:8]) r.currentType = byte(binary.LittleEndian.Uint16(header[8:10])) r.currentN = int(binary.LittleEndian.Uint16(header[10:12])) + 1 - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + offset32 := binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + if offset32 < r.prevOffset32 { + r.chunkOffset += (1 << 32) + } + r.prevOffset32 = offset32 + r.currentDataOffset = r.chunkOffset + uint64(offset32) // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1903,7 +1916,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1926,7 +1939,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -1949,7 +1962,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length // with runs, we can't actually look up offsets; the format just stores // things sequentially. so we have to actually track the offset in that case. if !r.haveRuns { - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + r.currentDataOffset = uint64(binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])) } // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1962,7 +1975,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1994,7 +2007,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -2012,14 +2025,14 @@ func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut if c.Mapped() { mappedIn++ } else { - err = fmt.Errorf("container key %d, addr %x, inside %x+%d\n", + err = fmt.Errorf("container key %d, addr %x, inside %x+%d", key, dptr, from, to-from) errs++ unmappedIn++ } } else { if c.Mapped() { - err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped\n", + err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped", key, dptr, from, to-from) errs++ mappedOut++ @@ -4539,50 +4552,95 @@ func (c *Container) bitmapZeroRange(i, j uint64) { c.setN(n) } -// equals reports whether two containers are equal. -func (c *Container) equals(c2 *Container) bool { - if c == nil || c2 == nil { - if c != c2 { - return false +func typePair(ct1, ct2 byte) int { + return int((ct1 << 4) | ct2) +} + +// compareArrayBitmap actually only verifies that everything in the array +// is in the bitmap. It's used only after comparing the N for the containers, +// so if there's anything in the bitmap that's not in the array, either there's +// something in the array that's not in the bitmap, or we didn't get here. +func compareArrayBitmap(a []uint16, b []uint64) error { + for _, v := range a { + w, bit := b[v>>6], v&63 + if w>>bit&1 == 0 { + return fmt.Errorf("value %d missing", v) } } - if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() { - return false + return nil +} + +// compareArrayRuns determines whether an array matches a provided +// set of runs. As with compareArrayBitmap, it only verifies presence +// of the array's values in the run collection. the run collection +// can't be empty; if it were, N would have been 0, and we wouldn't +// have gotten here. +func compareArrayRuns(a []uint16, r []interval16) error { + ri := 0 + ru := r[ri] + ri++ + for _, v := range a { + if v < ru.start { + return fmt.Errorf("value %d missing", v) + } + if v > ru.last { + if ri >= len(r) { + return fmt.Errorf("value %d missing", v) + } + ru = r[ri] + ri++ + // if they're identical, the array value must be + // the start of the next run. + if v != ru.start { + return fmt.Errorf("value %d missing", v) + } + } } - if c.typ() == containerArray { - ca, c2a := c.array(), c2.array() - if len(ca) != len(c2a) { - return false - } - for i := 0; i < len(ca); i++ { - if ca[i] != c2a[i] { - return false - } - } - } else if c.typ() == containerBitmap { - cb, c2b := c.bitmap(), c2.bitmap() - if len(cb) != len(c2b) { - return false - } - for i := 0; i < len(cb); i++ { - if cb[i] != c2b[i] { - return false - } - } - } else if c.typ() == containerRun { - cr, c2r := c.runs(), c2.runs() - if len(cr) != len(c2r) { - return false - } - for i := 0; i < len(cr); i++ { - if cr[i] != c2r[i] { - return false - } - } - } else { - panic(fmt.Sprintf("unknown container type: %v", c.typ())) + return nil +} + +// compareArrayArray reports whether everything in a1 is equal to everything +// in a2. +func compareArrayArray(a1, a2 []uint16) error { + if len(a1) != len(a2) { + return fmt.Errorf("unexpected length mismatch, %d vs %d", len(a1), len(a2)) } - return true + for i := range a1 { + if a1[i] != a2[i] { + return fmt.Errorf("item %d: %d vs %d", i, a1[i], a2[i]) + } + } + return nil +} + +// BitwiseCompare reports whether two containers are equal. It returns +// an error describing any difference it finds. This is mostly intended +// for use in tests that expect equality. +func (c *Container) BitwiseCompare(c2 *Container) error { + if c.N() != c2.N() { + return errors.New("containers are different lengths") + } + if c.N() == 0 { + return nil + } + switch typePair(c.typ(), c2.typ()) { + case typePair(containerArray, containerArray): + return compareArrayArray(c.array(), c2.array()) + case typePair(containerArray, containerBitmap): + return compareArrayBitmap(c.array(), c2.bitmap()) + case typePair(containerBitmap, containerArray): + return compareArrayBitmap(c2.array(), c.bitmap()) + case typePair(containerArray, containerRun): + return compareArrayRuns(c.array(), c2.runs()) + case typePair(containerRun, containerArray): + return compareArrayRuns(c2.array(), c.runs()) + default: + c3 := xor(c, c2) + if c3.N() != 0 { + return fmt.Errorf("%d bits differenct between containers", c3.N()) + } + } + return nil } func unionArrayBitmap(a, b *Container) *Container { @@ -5743,7 +5801,37 @@ func xorBitmapRun(a, b *Container) *Container { return output } -// CompareEquality is used mostly in test cases to confirm that two bitmaps came +// CompareBitmapSlice checks whether a bitmap has the same values in it +// that a provided slice does. +func CompareBitmapSlice(b *Bitmap, vals []uint64) (bool, error) { + count := b.Count() + if count != uint64(len(vals)) { + return false, fmt.Errorf("length mismatch: bitmap has %d bits, slice has %d", count, len(vals)) + } + for _, v := range vals { + if !b.Contains(v) { + return false, fmt.Errorf("bitmap lacks expected value %d", v) + } + } + return true, nil +} + +// CompareBitmapMap checks whether a bitmap has the same values in it +// that a provided map[uint64]struct{} has as keys. +func CompareBitmapMap(b *Bitmap, vals map[uint64]struct{}) (bool, error) { + count := b.Count() + if count != uint64(len(vals)) { + return false, fmt.Errorf("length mismatch: bitmap has %d bits, map has %d", count, len(vals)) + } + for v := range vals { + if !b.Contains(v) { + return false, fmt.Errorf("bitmap lacks expected value %d", v) + } + } + return true, nil +} + +// BitwiseEqual is used mostly in test cases to confirm that two bitmaps came // out the same. It does not expect corresponding opN, or OpWriter, but expects // identical bit contents. It does not expect identical representations; a bitmap // container can be identical to an array container. It returns a boolean value, @@ -5812,35 +5900,6 @@ func (b *Bitmap) BitwiseEqual(c *Bitmap) (bool, error) { return true, nil } -func bitmapsEqual(b, c *Bitmap) error { // nolint: deadcode - statsHit("bitmapsEqual") - if b.OpWriter != c.OpWriter { - return errors.New("opWriters not equal") - } - if b.opN != c.opN { - return errors.New("opNs not equal") - } - - biter, _ := b.Containers.Iterator(0) - citer, _ := c.Containers.Iterator(0) - bn, cn := biter.Next(), citer.Next() - for ; bn && cn; bn, cn = biter.Next(), citer.Next() { - bk, bc := biter.Value() - ck, cc := citer.Value() - if bk != ck { - return errors.New("keys not equal") - } - if !bc.equals(cc) { - return errors.New("containers not equal") - } - } - if bn && !cn || cn && !bn { - return errors.New("different numbers of containers") - } - - return nil -} - func popcount(x uint64) uint64 { return uint64(bits.OnesCount64(x)) } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 41d4059fb..f8de764fb 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -14,6 +14,8 @@ package roaring +import "sync" + /////////////////////////////////////////////////////////////////////////// var containerWidth uint64 = 65536 @@ -261,51 +263,56 @@ func doContainer(typ byte, data interface{}) *Container { return nil } +var makeCts sync.Once +var sampleTestContainers map[byte]map[string]*Container + func setupContainerTests() map[byte]map[string]*Container { - cts := make(map[byte]map[string]*Container) + makeCts.Do(func() { + sampleTestContainers = make(map[byte]map[string]*Container) - // array containers - cts[containerArray] = map[string]*Container{ - "empty": doContainer(containerArray, arrayEmpty()), - "full": doContainer(containerArray, arrayFull()), - "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), - "lastBitSet": doContainer(containerArray, arrayLastBitSet()), - "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), - "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), - "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), - "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), - "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), - "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), - } + // array containers + sampleTestContainers[containerArray] = map[string]*Container{ + "empty": doContainer(containerArray, arrayEmpty()), + "full": doContainer(containerArray, arrayFull()), + "firstBitSet": doContainer(containerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(containerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(containerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(containerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()), + } - // bitmap containers - cts[containerBitmap] = map[string]*Container{ - "empty": doContainer(containerBitmap, bitmapEmpty()), - "full": doContainer(containerBitmap, bitmapFull()), - "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), - "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), - "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), - "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), - "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), - "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), - "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), - "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), - } + // bitmap containers + sampleTestContainers[containerBitmap] = map[string]*Container{ + "empty": doContainer(containerBitmap, bitmapEmpty()), + "full": doContainer(containerBitmap, bitmapFull()), + "firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()), + } - // run containers - cts[containerRun] = map[string]*Container{ - "empty": doContainer(containerRun, runEmpty()), - "full": doContainer(containerRun, runFull()), - "firstBitSet": doContainer(containerRun, runFirstBitSet()), - "lastBitSet": doContainer(containerRun, runLastBitSet()), - "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), - "lastBitUnset": doContainer(containerRun, runLastBitUnset()), - "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), - "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), - "oddBitsSet": doContainer(containerRun, runOddBitsSet()), - "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), - } + // run containers + sampleTestContainers[containerRun] = map[string]*Container{ + "empty": doContainer(containerRun, runEmpty()), + "full": doContainer(containerRun, runFull()), + "firstBitSet": doContainer(containerRun, runFirstBitSet()), + "lastBitSet": doContainer(containerRun, runLastBitSet()), + "firstBitUnset": doContainer(containerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(containerRun, runLastBitUnset()), + "innerBitsSet": doContainer(containerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(containerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(containerRun, runOddBitsSet()), + "evenBitsSet": doContainer(containerRun, runEvenBitsSet()), + } + }) - return cts + return sampleTestContainers } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 9e7a468fc..3a0749cc8 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -19,11 +19,13 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/rand" "reflect" "runtime" "strings" "testing" + "github.com/pilosa/pilosa/v2/generator" "github.com/pkg/errors" ) @@ -2630,7 +2632,7 @@ func TestBitmapClone(t *testing.T) { } } c := b.Clone() - if err := bitmapsEqual(b, c); err != nil { + if _, err := b.BitwiseEqual(c); err != nil { t.Fatalf("Clone Objects not equal: %v\n", err) } d := func() *Bitmap { //anybody know how to declare a nil value? @@ -2733,34 +2735,46 @@ func getFunctionName(i interface{}) string { return y[0] } -// UnionInPlace is defined at the Bitmap level, but this wrapper lets us insert -// it into our ContainerCombinations tests so that it gets exercised on a wide -// variety of container data. func unionInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.UnionInPlace(B) - return out.Containers.Get(0) + ret := a.Clone().unionInPlace(b) + ret.Repair() + return ret } func differenceInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.DifferenceInPlace(B) - return out.Containers.Get(0) + a = a.Clone() + // this should probably return its new value, but currently does not + a.differenceInPlace(b) + return a } func intersectInPlaceWrapper(a, b *Container) *Container { - out := NewBitmap() - out.Containers.Put(0, a.Clone()) - B := NewBitmap() - B.Containers.Put(0, b) - out.IntersectInPlace(B) - return out.Containers.Get(0) + return a.Clone().intersectInPlace(b) +} + +func TestContainerBitwiseCompare(t *testing.T) { + cts := setupContainerTests() + + for t1, containers := range cts { + for name, c := range containers { + for t2, other := range cts { + for otherName, otherC := range other { + err := c.BitwiseCompare(otherC) + if err != nil { + if otherName == name { + t.Fatalf("container types %d/%d, contents %s: unexpected error %v", + t1, t2, name, err) + } + } else { + if name != otherName { + t.Fatalf("container types %d/%d, unexpected %s == %s", + t1, t2, name, otherName) + } + } + } + } + } + } } func TestContainerCombinations(t *testing.T) { @@ -3560,51 +3574,8 @@ func TestContainerCombinations(t *testing.T) { // Convert to all container types and check result. for _, ct := range containerTypes { - clone := ret.Clone() - if ct == containerArray { - if clone == nil { - clone = NewContainerArray(nil) - } else if clone.isBitmap() { - clone = clone.bitmapToArray() - } else if clone.isRun() { - clone = clone.runToArray() - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - // Because xorRunRun resulting in an empty container returns an array container with a - // nil slice array, then we need to check len() on array first (look for 0). - if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) { - t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array()) - } - } else if ct == containerBitmap { - if clone == nil { - clone = NewContainerBitmap(0, nil) - } else if clone.isArray() { - clone = clone.arrayToBitmap() - } else if clone.isRun() { - clone = clone.runToBitmap() - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) { - t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap()) - } - } else if ct == containerRun { - if clone == nil { - clone = NewContainerRun(nil) - } else if clone.isArray() { - clone = clone.arrayToRun(0) - } else if clone.isBitmap() { - clone = clone.bitmapToRun(0) - } - if clone.N() != cts[ct][exp].N() { - t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N()) - } - if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) { - t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs()) - } + if err := ret.BitwiseCompare(cts[ct][exp]); err != nil { + t.Errorf("test %s: %v", desc, err) } } } @@ -4026,7 +3997,11 @@ func TestDirectAddNVsAdd(t *testing.T) { {9384932, 101000, 2, 1, 0}, {3489, 19230, 394, 0, 893982, 890283, 14, 7}, } - // TODO generate more tests and fuzz + // Add some randomly created tests. + rand := rand.New(rand.NewSource(1)) + for i := 0; i < 100; i++ { + tests = append(tests, generator.Uint64Slice(1+rand.Intn(1000), 0, 10000000, i%2 == 0, rand)) + } testsCopy := make([][]uint64, len(tests)) copy(testsCopy, tests) for i, test := range testsCopy { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index a5c4a9a25..55395ea73 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -20,12 +20,12 @@ import ( "math" "math/rand" "reflect" - "sort" "testing" "testing/quick" "time" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/generator" "github.com/pilosa/pilosa/v2/roaring" _ "github.com/pilosa/pilosa/v2/test" ) @@ -280,11 +280,36 @@ func TestBitmap_Slice_Empty(t *testing.T) { } // Ensure a bitmap can return a slice of values within a range. -// TODO duplicate for all container types func TestBitmap_SliceRange(t *testing.T) { - if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { - t.Fatalf("unexpected slice: %+v", a) - } + t.Run("array", func(t *testing.T) { + if a := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) + + t.Run("bitmap", func(t *testing.T) { + bm := roaring.NewFileBitmap() + for i := uint64(10); i < 10000; i++ { + _, _ = bm.Add(i * 2) + } + bm.Optimize() + + if a := bm.SliceRange(20, 30); !reflect.DeepEqual(a, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) + + t.Run("run", func(t *testing.T) { + bm := roaring.NewFileBitmap() + for i := uint64(0); i < 11; i++ { + _, _ = bm.Add(i) + } + bm.Optimize() + + if a := bm.SliceRange(6, 10); !reflect.DeepEqual(a, []uint64{6, 7, 8, 9}) { + t.Fatalf("unexpected slice: %+v", a) + } + }) } // Ensure a bitmap can loop over a set of values. @@ -841,7 +866,7 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) { seed = time.Now().UnixNano() source = rand.NewSource(seed) rng = rand.New(source) - numTests = 100 + numTests = 20 maxNumIntsPerBatch = 100 maxNumBatches = 100 maxRangePercent = 2 @@ -1400,10 +1425,10 @@ func TestBitmap_Shift(t *testing.T) { } func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } -func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } -func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) } -func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 10000, 10000, 20000) } -func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxInt64) } +func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 1000, 0, 1000) } +func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 1000, 0, 10000) } +func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 1000, 10000, 20000) } +func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 1000, 0, math.MaxInt64) } // Ensure a bitmap can perform basic operations on randomly generated values. func testBitmapQuick(t *testing.T, n int, min, max uint64) { @@ -1442,7 +1467,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { // If `got` is nil and `exp` has zero length, don't perform the DeepEqual // because when `a` is empty (`a = []uint64{}`) then `got` is a nil slice // while `exp` is an empty slice. Therefore they will not be considered equal. - if got, exp := bm.Slice(), uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) { + if got, exp := bm.Slice(), generator.Uint64SetSlice(m); !(got == nil && len(exp) == 0) && !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected values:\n\ngot=%+v\n\nexp=%+v\n\n", got, exp) } @@ -1467,7 +1492,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { return true }, &quick.Config{ Values: func(values []reflect.Value, rand *rand.Rand) { - values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, false, rand)) + values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, false, rand)) }, }) if err != nil { @@ -1478,22 +1503,30 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) { func TestBitmap_Marshal_Quick_Array1(t *testing.T) { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) } -func TestBitmap_Marshal_Quick_Array2(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 1000, false) } +func TestBitmap_Marshal_Quick_Array2(t *testing.T) { + testBitmapMarshalQuick(t, 1000, 0, 1000, false) +} func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 0, 10000, false) + testBitmapMarshalQuick(t, 1000, 0, 10000, false) } func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 10000, 20000, false) + testBitmapMarshalQuick(t, 1000, 10000, 20000, false) } func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) { testBitmapMarshalQuick(t, 100, 0, math.MaxInt64, false) } func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) { - testBitmapMarshalQuick(t, 10000, 0, 10000, true) + testBitmapMarshalQuick(t, 1000, 0, 10000, true) } // TODO update for RLE +// (travis) - it's not clear to me how to generate a run container +// using `testBitmapMarshalQuick`. Because it's randomly generated, +// even some of the "Bitmap" tests generate array containers. Also, +// I think in order for the container to be a run, we would need +// to call bm.Optimize() on the bitmap, and I'm hesitant to add that +// because it's not clear to me how that would affect the tests. // Ensure a bitmap can be marshaled and unmarshaled. func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { @@ -1537,22 +1570,20 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { t.Fatal(err) } - // Verify the original bitmap has the correct set of values. - if exp, got := uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) { - t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) + if _, err := roaring.CompareBitmapMap(bm, set); err != nil { + t.Fatalf("source mismatch: %v", err) } - // Verify the bitmap loaded with the ops log has the correct set of values. - if exp, got := uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) { - t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got) + if _, err := roaring.CompareBitmapMap(bm2, set); err != nil { + t.Fatalf("unmarshalled mismatch: %v", err) } } return true }, &quick.Config{ Values: func(values []reflect.Value, rand *rand.Rand) { - values[0] = reflect.ValueOf(GenerateUint64Slice(n, min, max, sorted, rand)) - values[1] = reflect.ValueOf(GenerateUint64Slice(100, min, max, sorted, rand)) + values[0] = reflect.ValueOf(generator.Uint64Slice(n, min, max, sorted, rand)) + values[1] = reflect.ValueOf(generator.Uint64Slice(100, min, max, sorted, rand)) }, }) if err != nil { @@ -1561,9 +1592,8 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) { } // Ensure iterator can iterate over all the values on the bitmap. -// TODO duplicate for all container types func TestIterator(t *testing.T) { - t.Run("bitmap", func(t *testing.T) { + t.Run("array", func(t *testing.T) { itr := roaring.NewFileBitmap(1, 2, 3).Iterator() itr.Seek(0) @@ -1577,6 +1607,29 @@ func TestIterator(t *testing.T) { } }) + t.Run("bitmap", func(t *testing.T) { + bm := roaring.NewFileBitmap() + exp := []uint64{} + for i := uint64(0); i < 10000; i++ { + v := i * 2 + _, _ = bm.Add(v) + exp = append(exp, v) + } + bm.Optimize() + + itr := bm.Iterator() + itr.Seek(0) + + var a []uint64 + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + a = append(a, v) + } + + if !reflect.DeepEqual(a, exp) { + t.Fatalf("unexpected values: %+v", a) + } + }) + t.Run("run", func(t *testing.T) { bm1 := roaring.NewFileBitmap() for i := uint64(0); i < 11; i++ { @@ -1784,49 +1837,6 @@ func getBenchData(tb testing.TB) *benchmarkSampleData { return data } -// GenerateUint64Slice generates between [0, n) random uint64 numbers between min and max. -func GenerateUint64Slice(n int, min, max uint64, sorted bool, rand *rand.Rand) []uint64 { - a := make([]uint64, rand.Intn(n)) - for i := range a { - a[i] = min + uint64(rand.Int63n(int64(max-min))) - } - - if sorted { - sort.Sort(uint64Slice(a)) - } - - return a -} - -// uint64SetSlice returns the values in a uint64 set. -func uint64SetSlice(m map[uint64]struct{}) []uint64 { - a := make([]uint64, 0, len(m)) - for v := range m { - a = append(a, v) - } - sort.Sort(uint64Slice(a)) - return a -} - -// uint64Slice represents a sortable slice of uint64 numbers. -type uint64Slice []uint64 - -func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p uint64Slice) Len() int { return len(p) } -func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } - -func diff(a, b []uint64) string { - if len(a) != len(b) { - return fmt.Sprintf("len: %d != %d", len(a), len(b)) - } - for i := range a { - if a[i] != b[i] { - return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i]) - } - } - return "" -} - func TestBitmap_Intersect(t *testing.T) { bm0 := testBM() result := bm0.Intersect(bm0) diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index 208c15d53..c9071b1ca 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -159,8 +159,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[3+1 : 8]) - if uint32(len(data)) < headerBaseSize+keyN*12 { - return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", int(keyN)/12) + if int64(len(data)) < headerBaseSize+int64(keyN)*12 { + return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN) } headerSize := headerBaseSize @@ -173,14 +173,23 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { int(binary.LittleEndian.Uint16(buf[10:12]))+1, true) } - opsOffset := headerSize + int(keyN)*12 + opsOffset := int64(headerSize) + int64(keyN)*12 // Read container offsets and attach data. citer, _ := b.Containers.Iterator(0) + // if you have enough containers that the *headers alone* exceed 4GB, we + // need to start with a higher cycle offset. + cycleOffset := opsOffset &^ ((1 << 32) - 1) + prevOffset32 := uint32(opsOffset) for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { - offset := binary.LittleEndian.Uint32(buf[0:4]) + offset32 := binary.LittleEndian.Uint32(buf[0:4]) + if offset32 < prevOffset32 { + cycleOffset += (1 << 32) + } + prevOffset32 = offset32 + offset := int64(offset32) + cycleOffset // Verify the offset is within the bounds of the input data. - if int(offset) >= len(data) { + if offset >= int64(len(data)) { return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) } @@ -201,13 +210,13 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { case containerRun: runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount]) - opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size + opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size case containerArray: c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) - opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) + opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) - opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64) + opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64) } } @@ -228,7 +237,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Increase the op count. b.ops++ b.opN += opr.count() - opsOffset += opr.size() + opsOffset += int64(opr.size()) // Move the buffer forward. buf = data[opsOffset:] } diff --git a/row.go b/row.go index e50c6ab67..eea6871f0 100644 --- a/row.go +++ b/row.go @@ -374,7 +374,7 @@ func (r *Row) Difference(others ...*Row) *Row { return &Row{segments: output} } -// GenericUnary returns the results of a generic op on r. +// GenericUnaryOp returns the results of a generic op on r. func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row { work := r var segments []rowSegment @@ -510,14 +510,10 @@ func (r *Row) Columns() []uint64 { // Includes returns true if the row contains the given column. func (r *Row) Includes(col uint64) bool { - // TODO: improve the efficiency of this method by - // performing the column filter at the bitmap level - // rather than iterating through the results here. + shard := col / ShardWidth for i := range r.segments { - for _, c := range r.segments[i].Columns() { - if c == col { - return true - } + if r.segments[i].shard == shard { + return r.segments[i].data.Contains(col) } } return false @@ -574,13 +570,11 @@ func (s *rowSegment) IntersectionCount(other *rowSegment) uint64 { // Intersect returns the itersection of s and other. func (s *rowSegment) Intersect(other *rowSegment) *rowSegment { data := s.data.Intersect(other.data) - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } @@ -591,13 +585,11 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment { datas[i] = other.data } data := s.data.Union(datas...) - data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } @@ -635,49 +627,43 @@ func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment { datas[i] = other.data } data := s.data.Difference(datas...) - data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } // Xor returns the xor of s and other. func (s *rowSegment) Xor(other *rowSegment) *rowSegment { data := s.data.Xor(other.data) - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), } } // Shift returns s shifted by 1 bit. func (s *rowSegment) Shift() (*rowSegment, error) { - //TODO deal with overflow + // TODO: deal with overflow + // See issue: https://github.com/molecula/pilosa/issues/403 data, err := s.data.Shift(1) if err != nil { return nil, errors.Wrap(err, "shifting roaring data") } - data = data.Freeze() return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - writable: true, + data: data, + shard: s.shard, + n: data.Count(), }, nil } -// GenericUnary returns s subject to op. +// GenericUnaryOp returns s subject to op. func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment { - //TODO deal with overflow data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args)) return &rowSegment{ diff --git a/server.go b/server.go index fd07ec0b2..131a6971c 100644 --- a/server.go +++ b/server.go @@ -77,6 +77,8 @@ type Server struct { // nolint: maligned metricInterval time.Duration diagnosticInterval time.Duration maxWritesPerRequest int + confirmDownSleep time.Duration + confirmDownRetries int isCoordinator bool syncer holderSyncer @@ -229,6 +231,17 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { } } +// OptServerNodeDownRetries is a functional option on Server +// used to specify the retries and sleep duration for node down +// checks. +func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption { + return func(s *Server) error { + s.confirmDownRetries = retries + s.confirmDownSleep = sleep + return nil + } +} + // OptServerURI is a functional option on Server // used to set the server URI. func OptServerURI(uri *URI) ServerOption { @@ -330,6 +343,9 @@ func NewServer(opts ...ServerOption) (*Server, error) { metricInterval: 0, diagnosticInterval: 0, + confirmDownRetries: defaultConfirmDownRetries, + confirmDownSleep: defaultConfirmDownSleep, + resetTranslationSyncCh: make(chan struct{}), logger: logger.NopLogger, @@ -356,14 +372,11 @@ func NewServer(opts ...ServerOption) (*Server, error) { } s.executor = newExecutor(executorOpts...) - // s.holder.translateFile.logger = s.logger - path, err := expandDirName(s.dataDir) if err != nil { return nil, err } s.holder.Path = path - // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger s.holder.Stats.SetLogger(s.logger) @@ -402,6 +415,8 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.MaxWritesPerRequest = s.maxWritesPerRequest s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest + s.cluster.confirmDownRetries = s.confirmDownRetries + s.cluster.confirmDownSleep = s.confirmDownSleep s.holder.broadcaster = s err = s.loadAllExtensions() if err != nil { @@ -716,10 +731,13 @@ func (s *Server) receiveMessage(m Message) error { } case *CreateIndexMessage: opt := obj.Meta - _, err := s.holder.CreateIndex(obj.Index, *opt) + idx, err := s.holder.CreateIndex(obj.Index, *opt) if err != nil { return err } + idx.mu.Lock() + idx.createdAt = obj.CreatedAt + idx.mu.Unlock() case *DeleteIndexMessage: if err := s.holder.DeleteIndex(obj.Index); err != nil { return err @@ -730,10 +748,13 @@ func (s *Server) receiveMessage(m Message) error { return fmt.Errorf("local index not found: %s", obj.Index) } opt := obj.Meta - _, err := idx.createFieldIfNotExists(obj.Field, opt) + fld, err := idx.createFieldIfNotExists(obj.Field, opt) if err != nil { return err } + fld.mu.Lock() + fld.createdAt = obj.CreatedAt + fld.mu.Unlock() case *DeleteFieldMessage: idx := s.holder.Index(obj.Index) if err := idx.DeleteField(obj.Field); err != nil { @@ -767,6 +788,12 @@ func (s *Server) receiveMessage(m Message) error { if err != nil { return err } + if !s.isCoordinator { + if obj.Schema != nil { + s.holder.applyCreatedAt(obj.Schema.Indexes) + } + } + case *ResizeInstruction: err := s.cluster.followResizeInstruction(obj) if err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index b01b15c59..479856fad 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -195,19 +195,81 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - body := w.Body.String() - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]} -`, pilosa.ShardWidth) + + body := strings.TrimSpace(w.Body.String()) + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) if body != target { t.Fatalf("%s != %s", target, body) } }) + t.Run("Import", func(t *testing.T) { + indexInfo := cmd.API.Schema(context.Background()) + err := cmd.API.ApplySchema(context.Background(), &pilosa.Schema{Indexes: indexInfo}, false) + if err != nil { + t.Fatalf("applying schema: %v", err) + } + + idx := indexInfo[0] + fld := indexInfo[0].Fields[0] + msg := pilosa.ImportRequest{ + Index: idx.Name, + IndexCreatedAt: idx.CreatedAt, + Field: fld.Name, + FieldCreatedAt: fld.CreatedAt, + Shard: 0, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + path := fmt.Sprintf("/index/%s/field/%s/import", idx.Name, fld.Name) + httpReq := test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + + w := httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + if w.Code != 200 { + t.Fatalf(w.Body.String()) + } + + msg.IndexCreatedAt = -idx.CreatedAt + msg.FieldCreatedAt = -fld.CreatedAt + data, err = ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq = test.MustNewHTTPRequest("POST", path, bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + + w = httptest.NewRecorder() + h.ServeHTTP(w, httpReq) + + if w.Code != 412 { + t.Fatal("expected: Precondition Failed, got:" + w.Body.String()) + } + }) + t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + + idx, err := cmd.API.Index(context.Background(), "i0") + if err != nil { + t.Fatal(err) + } + fld, err := cmd.API.Field(context.Background(), "i0", "f1") + if err != nil { + t.Fatal(err) + } + msg := pilosa.ImportRoaringRequest{ - Clear: false, + IndexCreatedAt: idx.CreatedAt(), + FieldCreatedAt: fld.CreatedAt(), + Clear: false, Views: map[string][]byte{ "": roaringData, }, @@ -217,10 +279,14 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatal(err) } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) httpReq.Header.Set("Content-Type", "application/x-protobuf") httpReq.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, httpReq) + if w.Code != 200 { + t.Fatalf("Unexpected response body: %s", w.Body.String()) + } resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) @@ -761,8 +827,15 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected response body: %s", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // Verify index is gone. if hldr.Index("i") != nil { @@ -779,10 +852,18 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader(""))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String()) - } else if body := w.Body.String(); body != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i").Field("f1"); f != nil { - t.Fatal("expected nil field") + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + if f := hldr.Index("i").Field("f1"); f != nil { + t.Fatal("expected nil field") + } } }) @@ -960,8 +1041,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create index again @@ -970,8 +1057,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating index: index already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // create field @@ -980,8 +1075,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field again @@ -990,8 +1093,16 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusConflict { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"creating field: field already exists"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + Name string `json:"name,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success || resp.Name == "" || resp.CreatedAt == 0 { + t.Errorf("unexpected body: %q", w.Body.String()) + } } // delete field @@ -1000,8 +1111,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete field again @@ -1020,8 +1137,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // delete index again @@ -1030,8 +1153,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusNotFound { t.Errorf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":false,"error":{"message":"deleting index: index not found"}}`+"\n" { - t.Errorf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } }) @@ -1042,8 +1171,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // create field @@ -1052,8 +1187,14 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"success":true}`+"\n" { - t.Fatalf("unexpected body: %q", w.Body.String()) + } else { + var resp struct { + Success bool `json:"success"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + if !resp.Success { + t.Fatalf("unexpected body: %q", w.Body.String()) + } } // set some bits diff --git a/server/server.go b/server/server.go index 5ff688096..614cfd1ab 100644 --- a/server/server.go +++ b/server/server.go @@ -274,6 +274,15 @@ func (m *Command) SetupServer() error { grpcURI.SetPort(uint16(m.grpcLn.Addr().(*net.TCPAddr).Port)) } + if grpcURI.Scheme == "http" { + grpcURI.Scheme = "grpc" + } + + // discover the address if not specified + if grpcURI.Host == "0.0.0.0" { + grpcURI.Host = outboundIP().String() + } + // Setup TLS if uri.Scheme == "https" { m.tlsConfig, err = GetTLSConfig(&m.Config.TLS, m.logger.Logger()) diff --git a/server/server_test.go b/server/server_test.go index d7cb9452e..96d0fa527 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1092,6 +1092,50 @@ func TestClusterExhaustingConnections(t *testing.T) { } } +func TestQueryingWithQuotesAndStuff(t *testing.T) { + m := test.RunCommand(t) + defer m.Close() + + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + if err != nil { + t.Fatal(err) + } + + // Execute Set() commands. + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "fld", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + // Test escaped single quote gets set properly + if res, err := m.Query(t, "i", "", `Set('bl\'ah', fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("setting escaped single quote result: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `bl'ah`) { + t.Errorf("value with escaped single quote set improperly: %s", res) + } + + // Test escaped double quote gets set properly + if res, err := m.Query(t, "i", "", `Set("d\"ah", fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("value with escaped double quote set improperly: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `d\"ah`) { + // the backslash is there because JSON needs to escape the + // double quote since it uses double quotes + t.Errorf("value with escaped double quote set improperly: %s", res) + } +} + func TestClusterExhaustingConnectionsImport(t *testing.T) { if !runStress { t.Skip("stress") diff --git a/test/pilosa.go b/test/pilosa.go index 635e0f3d9..9b5f49a48 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -92,7 +92,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { // We want tests to default to using the in-memory translate store, so we // prepend opts with that functional option. If a different translate store // has been specified, it will override this one. - opts = prependWithMemStore(opts) + opts = prependTestServerOpts(opts) m := newCommand(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator @@ -434,25 +434,25 @@ func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu return c } -// prependOpts applies prependWithMemStore to each of the ops (one per +// prependOpts applies prependTestServerOpts to each of the ops (one per // node, or one for the entire cluser). func prependOpts(opts [][]server.CommandOption) [][]server.CommandOption { if len(opts) == 0 { opts = [][]server.CommandOption{ - prependWithMemStore([]server.CommandOption{}), + prependTestServerOpts([]server.CommandOption{}), } } else { for i := range opts { - opts[i] = prependWithMemStore(opts[i]) + opts[i] = prependTestServerOpts(opts[i]) } } return opts } -// prependWithMemStore prepends opts with the OpenInMemTranslateStore. -func prependWithMemStore(opts []server.CommandOption) []server.CommandOption { +// prependTestServerOpts prepends opts with the OpenInMemTranslateStore. +func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption { defaultOpts := []server.CommandOption{ - server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)), + server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore), pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond)), } return append(defaultOpts, opts...) } diff --git a/tracker.go b/tracker.go new file mode 100644 index 000000000..af5788137 --- /dev/null +++ b/tracker.go @@ -0,0 +1,125 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "sort" + "sync" + "time" +) + +type ActiveQueryStatus struct { + Query string `json:"query"` + Age time.Duration `json:"age"` +} + +type activeQuery struct { + query string + started time.Time +} + +type queryStatusUpdate struct { + q *activeQuery + end bool +} + +type queryTracker struct { + updates chan<- queryStatusUpdate + checks chan<- chan<- []*activeQuery + wg sync.WaitGroup + stop chan struct{} +} + +func newQueryTracker() *queryTracker { + done := make(chan struct{}) + updates := make(chan queryStatusUpdate, 128) + checks := make(chan chan<- []*activeQuery) + tracker := &queryTracker{ + updates: updates, + checks: checks, + stop: done, + } + tracker.wg.Add(1) + go func() { + defer tracker.wg.Done() + + activeQueries := make(map[*activeQuery]struct{}) + + for { + select { + case update := <-updates: + if update.end { + delete(activeQueries, update.q) + } else { + activeQueries[update.q] = struct{}{} + } + case check := <-checks: + out := make([]*activeQuery, len(activeQueries)) + i := 0 + for q := range activeQueries { + out[i] = q + i++ + } + check <- out + close(check) + case <-done: + return + } + } + }() + return tracker +} + +func (t *queryTracker) Start(query string) *activeQuery { + now := time.Now() + q := &activeQuery{query, now} + t.updates <- queryStatusUpdate{q, false} + return q +} + +func (t *queryTracker) Finish(q *activeQuery) { + t.updates <- queryStatusUpdate{q, true} +} + +func (t *queryTracker) ActiveQueries() []ActiveQueryStatus { + ch := make(chan []*activeQuery, 1) + t.checks <- ch + queries := <-ch + sort.Slice(queries, func(i, j int) bool { + switch { + case queries[i].started.Before(queries[j].started): + return true + case queries[i].started.After(queries[j].started): + return false + case queries[i].query < queries[j].query: + return true + case queries[i].query > queries[j].query: + return false + default: + return false + } + }) + now := time.Now() + out := make([]ActiveQueryStatus, len(queries)) + for i, v := range queries { + out[i] = ActiveQueryStatus{v.query, now.Sub(v.started)} + } + return out +} + +func (t *queryTracker) Stop() { + close(t.stop) + t.wg.Wait() +} diff --git a/tracker_test.go b/tracker_test.go new file mode 100644 index 000000000..80f6f84f2 --- /dev/null +++ b/tracker_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import "testing" + +func TestQueryTracker(t *testing.T) { + tracker := newQueryTracker() + defer tracker.Stop() + + if queries := tracker.ActiveQueries(); len(queries) > 0 { + t.Fatalf("expected no active queries; found %v", queries) + } + + qs := tracker.Start("test query") + + var queries []ActiveQueryStatus + for len(queries) < 1 { + queries = tracker.ActiveQueries() + } + if len(queries) > 1 || queries[0].Query != "test query" { + t.Fatalf("unexpected queries: %v", queries) + } + + tracker.Finish(qs) + + for len(queries) > 0 { + queries = tracker.ActiveQueries() + } +} diff --git a/transaction.md b/transaction.md index f8f7f5723..4a3bae6d4 100644 --- a/transaction.md +++ b/transaction.md @@ -30,7 +30,7 @@ The base transaction endpoints are `/transactions`, for listing or creating transactions, and `/transaction/[id]`, for listing, creating, finishing, or cancelling a transaction. -A POST to `/transactions` attempts to create a transaction, assigning it an +A POST to `/transaction` attempts to create a transaction, assigning it an arbitrary ID that is not the ID of any existing transaction. A `GET` from `/transactions` lists existing transactions.