diff --git a/api.go b/api.go index 386b28f30..c9dc170b3 100644 --- a/api.go +++ b/api.go @@ -3385,11 +3385,6 @@ type SchemaAPI interface { DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error } -type SchemaInfoAPI interface { - IndexInfo(ctx context.Context, indexName string) (*IndexInfo, error) - FieldInfo(ctx context.Context, indexName, fieldName string) (*FieldInfo, error) -} - type ClusterNode struct { ID string State string diff --git a/dax/mds/client/client.go b/dax/mds/client/client.go index 359897e73..1efe3764b 100644 --- a/dax/mds/client/client.go +++ b/dax/mds/client/client.go @@ -10,7 +10,6 @@ import ( "net/http" "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/dax/mds/controller" mdshttp "github.com/molecula/featurebase/v3/dax/mds/http" "github.com/molecula/featurebase/v3/errors" "github.com/molecula/featurebase/v3/logger" @@ -49,6 +48,20 @@ func (c *Client) Health() bool { return true } +// TODO(tlt): collapse Table into this +func (c *Client) TableByID(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { + return c.Table(ctx, qtid) +} + +// TODO(tlt): collapse TableID into this +func (c *Client) TableByName(ctx context.Context, qual dax.TableQualifier, tname dax.TableName) (*dax.QualifiedTable, error) { + qtid, err := c.TableID(ctx, qual, tname) + if err != nil { + return nil, errors.Wrap(err, "getting table id") + } + return c.Table(ctx, qtid) +} + func (c *Client) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { url := fmt.Sprintf("%s/table", c.address.WithScheme(defaultScheme)) @@ -336,11 +349,11 @@ func (c *Client) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, return isr.Address, nil } -func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error) { +func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]dax.ComputeNode, error) { url := fmt.Sprintf("%s/compute-nodes", c.address.WithScheme(defaultScheme)) c.logger.Debugf("ComputeNodes url: %s", url) - var nodes []controller.ComputeNode + var nodes []dax.ComputeNode req := &mdshttp.ComputeNodesRequest{ Table: qtid, @@ -374,11 +387,11 @@ func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, sh return cnr.ComputeNodes, nil } -func (c *Client) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error) { +func (c *Client) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]dax.TranslateNode, error) { url := fmt.Sprintf("%s/translate-nodes", c.address.WithScheme(defaultScheme)) c.logger.Debugf("TranslateNodes url: %s", url) - var nodes []controller.TranslateNode + var nodes []dax.TranslateNode req := &mdshttp.TranslateNodesRequest{ Table: qtid, diff --git a/dax/mds/controller/controller.go b/dax/mds/controller/controller.go index a76221fef..36849b38a 100644 --- a/dax/mds/controller/controller.go +++ b/dax/mds/controller/controller.go @@ -1647,7 +1647,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa ///////////// -func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]ComputeNode, error) { +func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]dax.ComputeNode, error) { inRole := &dax.ComputeRole{ TableKey: qtid.Key(), Shards: dax.NewVersionedShards(shards...), @@ -1658,7 +1658,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID return nil, errors.Wrap(err, "getting compute nodes") } - computeNodes := make([]ComputeNode, 0) + computeNodes := make([]dax.ComputeNode, 0) for _, node := range nodes { role, ok := node.Role.(*dax.ComputeRole) @@ -1668,7 +1668,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID return nil, NewErrInternal("not a compute node") } - computeNodes = append(computeNodes, ComputeNode{ + computeNodes = append(computeNodes, dax.ComputeNode{ Address: node.Address, Table: role.TableKey, Shards: role.Shards.Nums(), @@ -1678,7 +1678,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID return computeNodes, nil } -func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]TranslateNode, error) { +func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]dax.TranslateNode, error) { inRole := &dax.TranslateRole{ TableKey: qtid.Key(), Partitions: dax.NewVersionedPartitions(partitions...), @@ -1689,7 +1689,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable return nil, errors.Wrap(err, "getting translate nodes") } - translateNodes := make([]TranslateNode, 0) + translateNodes := make([]dax.TranslateNode, 0) for _, node := range nodes { role, ok := node.Role.(*dax.TranslateRole) @@ -1699,7 +1699,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable return nil, NewErrInternal("not a translate node") } - translateNodes = append(translateNodes, TranslateNode{ + translateNodes = append(translateNodes, dax.TranslateNode{ Address: node.Address, Table: role.TableKey, Partitions: role.Partitions.Nums(), diff --git a/dax/mds/controller/types.go b/dax/mds/controller/types.go deleted file mode 100644 index 65d3e97a3..000000000 --- a/dax/mds/controller/types.go +++ /dev/null @@ -1,19 +0,0 @@ -package controller - -import "github.com/molecula/featurebase/v3/dax" - -// ComputeNode represents a compute node and the table/shards for which it is -// responsible. -type ComputeNode struct { - Address dax.Address `json:"address"` - Table dax.TableKey `json:"table"` - Shards dax.ShardNums `json:"shards"` -} - -// TranslateNode represents a translate node and the table/partitions for which -// it is responsible. -type TranslateNode struct { - Address dax.Address `json:"address"` - Table dax.TableKey `json:"table"` - Partitions dax.PartitionNums `json:"partitions"` -} diff --git a/dax/mds/http/handler.go b/dax/mds/http/handler.go index ff6fd3942..53ea40b71 100644 --- a/dax/mds/http/handler.go +++ b/dax/mds/http/handler.go @@ -7,7 +7,6 @@ import ( "github.com/gorilla/mux" "github.com/molecula/featurebase/v3/dax" "github.com/molecula/featurebase/v3/dax/mds" - "github.com/molecula/featurebase/v3/dax/mds/controller" ) func Handler(mds *mds.MDS) http.Handler { @@ -613,7 +612,7 @@ type ComputeNodesRequest struct { // provided are not included in this response. That might happen if there are // currently no active compute nodes. type ComputeNodesResponse struct { - ComputeNodes []controller.ComputeNode `json:"compute-nodes"` + ComputeNodes []dax.ComputeNode `json:"compute-nodes"` } // POST /translate-nodes @@ -660,5 +659,5 @@ type TranslateNodesRequest struct { // that partitions provided are not included in this response. That might happen // if there are currently no active translate nodes. type TranslateNodesResponse struct { - TranslateNodes []controller.TranslateNode `json:"translate-nodes"` + TranslateNodes []dax.TranslateNode `json:"translate-nodes"` } diff --git a/dax/mds/mds.go b/dax/mds/mds.go index f52fa111e..5f906e1ae 100644 --- a/dax/mds/mds.go +++ b/dax/mds/mds.go @@ -462,7 +462,7 @@ func (m *MDS) DeregisterNodes(ctx context.Context, addrs ...dax.Address) error { // ComputeNodes gets the compute nodes responsible for the table/shards // specified in the ComputeNodeRequest. -func (m *MDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shardNums ...dax.ShardNum) ([]controller.ComputeNode, error) { +func (m *MDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shardNums ...dax.ShardNum) ([]dax.ComputeNode, error) { if err := m.sanitizeQTID(ctx, &qtid); err != nil { return nil, errors.Wrap(err, "sanitizing") } @@ -476,7 +476,7 @@ func (m *MDS) DebugNodes(ctx context.Context) ([]*dax.Node, error) { // TranslateNodes gets the translate nodes responsible for the table/partitions // specified in the TranslateNodeRequest. -func (m *MDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitionNums ...dax.PartitionNum) ([]controller.TranslateNode, error) { +func (m *MDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitionNums ...dax.PartitionNum) ([]dax.TranslateNode, error) { if err := m.sanitizeQTID(ctx, &qtid); err != nil { return nil, errors.Wrap(err, "sanitizing") } diff --git a/dax/node.go b/dax/node.go index 8b6f9d331..4122956c6 100644 --- a/dax/node.go +++ b/dax/node.go @@ -29,6 +29,62 @@ type NodeService interface { Nodes(context.Context) ([]*Node, error) } +// ComputeNode represents a compute node and the table/shards for which it is +// responsible. +type ComputeNode struct { + Address Address `json:"address"` + Table TableKey `json:"table"` + Shards ShardNums `json:"shards"` +} + +// TranslateNode represents a translate node and the table/partitions for which +// it is responsible. +type TranslateNode struct { + Address Address `json:"address"` + Table TableKey `json:"table"` + Partitions PartitionNums `json:"partitions"` +} + +type Noder interface { + ComputeNodes(ctx context.Context, qtid QualifiedTableID, shards ...ShardNum) ([]ComputeNode, error) + TranslateNodes(ctx context.Context, qtid QualifiedTableID, partitions ...PartitionNum) ([]TranslateNode, error) + + // IngestPartition is effectively the "write" version of TranslateNodes. Its + // implementations will return the same Address that TranslateNodes would, + // but it includes the logic to create/assign the partition if it is not + // already being handled by a computer. + IngestPartition(ctx context.Context, qtid QualifiedTableID, partition PartitionNum) (Address, error) + + // IngestShard is effectively the "write" version of ComputeNodes. Its + // implementations will return the same Address that ComputeNodes would, but + // it includes the logic to create/assign the shard if it is not already + // being handled by a computer. + IngestShard(ctx context.Context, qtid QualifiedTableID, shard ShardNum) (Address, error) +} + +// Ensure type implements interface. +var _ Noder = &nopNoder{} + +// NopMDS is a no-op implementation of the MDS interface. +type nopNoder struct{} + +func NewNopNoder() *nopNoder { + return &nopNoder{} +} + +func (n *nopNoder) ComputeNodes(ctx context.Context, qtid QualifiedTableID, shards ...ShardNum) ([]ComputeNode, error) { + return nil, nil +} +func (n *nopNoder) IngestPartition(ctx context.Context, qtid QualifiedTableID, partition PartitionNum) (Address, error) { + return "", nil +} +func (n *nopNoder) IngestShard(ctx context.Context, qtid QualifiedTableID, shard ShardNum) (Address, error) { + return "", nil +} +func (n *nopNoder) TranslateNodes(ctx context.Context, qtid QualifiedTableID, partitions ...PartitionNum) ([]TranslateNode, error) { + return nil, nil +} + //////////////////////////////////////////////////// // Errors //////////////////////////////////////////////////// diff --git a/dax/queryer/featurebase_importer.go b/dax/queryer/featurebase_importer.go deleted file mode 100644 index 087df1a8d..000000000 --- a/dax/queryer/featurebase_importer.go +++ /dev/null @@ -1,41 +0,0 @@ -package queryer - -import ( - "context" - - featurebase "github.com/molecula/featurebase/v3" -) - -// Ensure type implements interface. -var _ Importer = &FeatureBaseImporter{} - -// FeatureBaseImporter is an implementation of the Importer interface which uses -// a pointer to a featurebase.API to make the underlying calls. This assumes -// those calls need to be Qcx aware, so this takes that into account. -type FeatureBaseImporter struct { - api *featurebase.API -} - -func NewFeatureBaseImporter(api *featurebase.API) *FeatureBaseImporter { - return &FeatureBaseImporter{ - api: api, - } -} - -func (fi *FeatureBaseImporter) CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) { - return fi.api.CreateIndexKeys(ctx, index, keys...) -} - -func (fi *FeatureBaseImporter) CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) { - return fi.api.CreateFieldKeys(ctx, index, field, keys...) -} - -func (fi *FeatureBaseImporter) Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error { - qcx := fi.api.Txf().NewQcx() - return fi.api.Import(ctx, qcx, req, opts...) -} - -func (fi *FeatureBaseImporter) ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error { - qcx := fi.api.Txf().NewQcx() - return fi.api.ImportValue(ctx, qcx, req, opts...) -} diff --git a/dax/queryer/interfaces.go b/dax/queryer/interfaces.go deleted file mode 100644 index 9abdf95cc..000000000 --- a/dax/queryer/interfaces.go +++ /dev/null @@ -1,78 +0,0 @@ -package queryer - -import ( - "context" - - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/dax/mds/controller" - "github.com/molecula/featurebase/v3/dax/mds/schemar" -) - -type MDS interface { - // Controller-related methods. - ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error) - IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error) - IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shard dax.ShardNum) (dax.Address, error) - TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error) - - // Schemar-related methods. - schemar.Schemar -} - -// Ensure type implements interface. -var _ MDS = &NopMDS{} - -// NopMDS is a no-op implementation of the MDS interface. -type NopMDS struct { - schemar.NopSchemar -} - -func NewNopMDS() *NopMDS { - return &NopMDS{} -} - -func (m *NopMDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error) { - return nil, nil -} -func (m *NopMDS) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error) { - return "", nil -} -func (m *NopMDS) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shard dax.ShardNum) (dax.Address, error) { - return "", nil -} -func (m *NopMDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error) { - return nil, nil -} - -type Importer interface { - CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) - CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) - Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error - ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error -} - -// Ensure type implements interface. -var _ Importer = &NopImporter{} - -// NopImporter is a no-op implementation of the Importer interface. -type NopImporter struct{} - -func NewNopImporter() *NopImporter { - return &NopImporter{} -} - -func (n *NopImporter) CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n *NopImporter) CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) { - return nil, nil -} - -func (n *NopImporter) Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error { - return nil -} -func (n *NopImporter) ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error { - return nil -} diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 72429174b..113503b8d 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -11,8 +11,6 @@ import ( featurebase "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/dax/mds/controller" - "github.com/molecula/featurebase/v3/dax/mds/schemar" "github.com/molecula/featurebase/v3/errors" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/pql" @@ -43,22 +41,25 @@ const ( ) type Topologer interface { - ComputeNodes(ctx context.Context, index string, shards []uint64) ([]controller.ComputeNode, error) + ComputeNodes(ctx context.Context, index string, shards []uint64) ([]dax.ComputeNode, error) } type MDSTopology struct { - mds MDS + noder dax.Noder } -func (m *MDSTopology) ComputeNodes(ctx context.Context, index string, shards []uint64) ([]controller.ComputeNode, error) { +func (m *MDSTopology) ComputeNodes(ctx context.Context, index string, shards []uint64) ([]dax.ComputeNode, error) { var daxShards = make(dax.ShardNums, len(shards)) for i, s := range shards { daxShards[i] = dax.ShardNum(s) } + // TODO(tlt): this needs review; MDSTopology is converting from + // string/uint64 to qtid/shardNum?? Perhaps we can get rid of the Topologer + // interface altogether and replace it with dax.Noder. qtid := dax.TableKey(index).QualifiedTableID() - return m.mds.ComputeNodes(ctx, qtid, daxShards...) + return m.noder.ComputeNodes(ctx, qtid, daxShards...) } // TODO(jaffee) we need version info in here ASAP. whenever schema or topo @@ -79,7 +80,7 @@ type Translator interface { // executor recursively executes calls in a PQL query across all shards. type orchestrator struct { - schema featurebase.SchemaInfoAPI + schema featurebase.SchemaAPI topology Topologer trans Translator @@ -117,29 +118,17 @@ func (o *orchestrator) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q return resp, errors.New(errors.ErrUncoded, "orchestrator.Execute expects a dax.QualifiedTable") } - index := string(qtbl.Key()) - // Check for query cancellation. if err := validateQueryContext(ctx); err != nil { return resp, err } - // Verify that an index is set. - if index == "" { - return resp, featurebase.ErrIndexRequired - } - - idx, err := o.schema.IndexInfo(ctx, index) - if err != nil { - return resp, errors.Wrap(err, "getting index") - } - // Default options. if opt == nil { opt = &featurebase.ExecOptions{} } - results, err := o.execute(ctx, index, q, shards, opt) + results, err := o.execute(ctx, tableKeyer, q, shards, opt) if err != nil { return resp, err } else if err := validateQueryContext(ctx); err != nil { @@ -147,7 +136,7 @@ func (o *orchestrator) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q } resp.Results = results - if err := o.translateResults(ctx, index, idx, q.Calls, results, opt.MaxMemory); err != nil { + if err := o.translateResults(ctx, qtbl, q.Calls, results, opt.MaxMemory); err != nil { if errors.Cause(err) == featurebase.ErrTranslatingKeyNotFound { // No error - return empty result resp.Results = make([]interface{}, len(q.Calls)) @@ -164,10 +153,12 @@ func (o *orchestrator) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q return resp, nil } -func (o *orchestrator) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *featurebase.ExecOptions) ([]interface{}, error) { +func (o *orchestrator) execute(ctx context.Context, tableKeyer dax.TableKeyer, q *pql.Query, shards []uint64, opt *featurebase.ExecOptions) ([]interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.execute") defer span.Finish() + index := string(tableKeyer.Key()) + // Apply translations if necessary. var colTranslations map[string]map[string]uint64 // colID := colTranslations[index][key] var rowTranslations map[string]map[string]map[string]uint64 // rowID := rowTranslations[index][field][key] @@ -189,7 +180,7 @@ func (o *orchestrator) execute(ctx context.Context, index string, q *pql.Query, // Apply call translation. if !opt.Remote && !opt.PreTranslated { - translated, err := o.translateCall(ctx, call, index, colTranslations, rowTranslations) + translated, err := o.translateCall(ctx, call, tableKeyer, colTranslations, rowTranslations) if err != nil { return nil, errors.Wrap(err, "translating call") } @@ -210,13 +201,13 @@ func (o *orchestrator) execute(ctx context.Context, index string, q *pql.Query, if call.Name == "Count" { // Handle count specially, skipping the level directly underneath it. for _, child := range call.Children { - err := o.handlePreCallChildren(ctx, index, child, shards, opt) + err := o.handlePreCallChildren(ctx, tableKeyer, child, shards, opt) if err != nil { return nil, err } } } else { - err := o.handlePreCallChildren(ctx, index, call, shards, opt) + err := o.handlePreCallChildren(ctx, tableKeyer, call, shards, opt) if err != nil { return nil, err } @@ -229,10 +220,11 @@ func (o *orchestrator) execute(ctx context.Context, index string, q *pql.Query, // already precomputed by handlePreCallChildren, though, // we don't need this logic in executeCall. newIndex := call.CallIndex() + newTableKeyer := dax.StringTableKeyer(newIndex) if newIndex != "" && newIndex != index { - v, err = o.executeCall(ctx, newIndex, call, nil, opt) + v, err = o.executeCall(ctx, newTableKeyer, call, nil, opt) } else { - v, err = o.executeCall(ctx, index, call, shards, opt) + v, err = o.executeCall(ctx, tableKeyer, call, shards, opt) } if err != nil { return nil, err @@ -255,7 +247,9 @@ func (o *orchestrator) execute(ctx context.Context, index string, q *pql.Query, // handlePreCalls traverses the call tree looking for calls that need // precomputed values (e.g. Distinct, UnionRows, ConstRow...). -func (o *orchestrator) handlePreCalls(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) error { +func (o *orchestrator) handlePreCalls(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) error { + index := string(tableKeyer.Key()) + if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) if idx >= 0 && idx < int64(len(opt.EmbeddedData)) { @@ -292,10 +286,11 @@ func (o *orchestrator) handlePreCalls(ctx context.Context, index string, c *pql. if newIndex != "" && newIndex != index { c.Type = pql.PrecallGlobal index = newIndex + tableKeyer = dax.StringTableKeyer(index) // we need to recompute shards, then shards = nil } - if err := o.handlePreCallChildren(ctx, index, c, shards, opt); err != nil { + if err := o.handlePreCallChildren(ctx, tableKeyer, c, shards, opt); err != nil { return err } // child calls already handled, no precall for this, so we're done @@ -311,7 +306,7 @@ func (o *orchestrator) handlePreCalls(ctx context.Context, index string, c *pql. // We set c to look like a normal call, and actually execute it: c.Type = pql.PrecallNone // possibly override call index. - v, err := o.executeCall(ctx, index, c, shards, opt) + v, err := o.executeCall(ctx, tableKeyer, c, shards, opt) if err != nil { return err } @@ -354,12 +349,12 @@ func (o *orchestrator) dumpPrecomputedCalls(ctx context.Context, c *pql.Call) { } // handlePreCallChildren handles any pre-calls in the children of a given call. -func (o *orchestrator) handlePreCallChildren(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) error { +func (o *orchestrator) handlePreCallChildren(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) error { for i := range c.Children { if err := ctx.Err(); err != nil { return err } - if err := o.handlePreCalls(ctx, index, c.Children[i], shards, opt); err != nil { + if err := o.handlePreCalls(ctx, tableKeyer, c.Children[i], shards, opt); err != nil { return err } } @@ -373,7 +368,7 @@ func (o *orchestrator) handlePreCallChildren(ctx context.Context, index string, if err := ctx.Err(); err != nil { return err } - if err := o.handlePreCalls(ctx, index, call, shards, opt); err != nil { + if err := o.handlePreCalls(ctx, tableKeyer, call, shards, opt); err != nil { return err } } @@ -382,7 +377,7 @@ func (o *orchestrator) handlePreCallChildren(ctx context.Context, index string, } // preprocessQuery expands any calls that need preprocessing. -func (o *orchestrator) preprocessQuery(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*pql.Call, error) { +func (o *orchestrator) preprocessQuery(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*pql.Call, error) { switch c.Name { case "All": _, hasLimit, err := c.UintArg("limit") @@ -411,7 +406,7 @@ func (o *orchestrator) preprocessQuery(ctx context.Context, index string, c *pql out := make([]*pql.Call, len(c.Children)) var changed bool for i, child := range c.Children { - res, err := o.preprocessQuery(ctx, index, child, shards, opt) + res, err := o.preprocessQuery(ctx, tableKeyer, child, shards, opt) if err != nil { return nil, err } @@ -429,7 +424,7 @@ func (o *orchestrator) preprocessQuery(ctx context.Context, index string, c *pql } // executeCall executes a call. -func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { +func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") defer span.Finish() @@ -438,7 +433,7 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal } else if err := o.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } - indexTag := "index:" + index + indexTag := "index:" + string(tableKeyer.Key()) metricName := "query_" + strings.ToLower(c.Name) + "_total" statFn := func() { if !opt.Remote { @@ -447,7 +442,7 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal } // Preprocess the query. - c, err := o.preprocessQuery(ctx, index, c, shards, opt) + c, err := o.preprocessQuery(ctx, tableKeyer, c, shards, opt) if err != nil { return nil, err } @@ -455,23 +450,23 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal switch c.Name { case "Sum": statFn() - res, err := o.executeSum(ctx, index, c, shards, opt) + res, err := o.executeSum(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeSum") case "Min": statFn() - res, err := o.executeMin(ctx, index, c, shards, opt) + res, err := o.executeMin(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMin") case "Max": statFn() - res, err := o.executeMax(ctx, index, c, shards, opt) + res, err := o.executeMax(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMax") case "MinRow": statFn() - res, err := o.executeMinRow(ctx, index, c, shards, opt) + res, err := o.executeMinRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMinRow") case "MaxRow": statFn() - res, err := o.executeMaxRow(ctx, index, c, shards, opt) + res, err := o.executeMaxRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMaxRow") // case "Clear": // statFn() @@ -483,7 +478,7 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal // return res, errors.Wrap(err, "executeClearRow") case "Distinct": statFn() - res, err := o.executeDistinct(ctx, index, c, shards, opt) + res, err := o.executeDistinct(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeDistinct") // case "Store": // statFn() @@ -491,7 +486,7 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal // return res, errors.Wrap(err, "executeSetRow") case "Count": statFn() - res, err := o.executeCount(ctx, index, c, shards, opt) + res, err := o.executeCount(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeCount") // case "Set": // statFn() @@ -499,49 +494,49 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal // return res, errors.Wrap(err, "executeSet") case "TopK": statFn() - res, err := o.executeTopK(ctx, index, c, shards, opt) + res, err := o.executeTopK(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopK") case "TopN": statFn() - res, err := o.executeTopN(ctx, index, c, shards, opt) + res, err := o.executeTopN(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopN") case "Rows": statFn() - res, err := o.executeRows(ctx, index, c, shards, opt) + res, err := o.executeRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeRows") case "Extract": statFn() - res, err := o.executeExtract(ctx, index, c, shards, opt) + res, err := o.executeExtract(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeExtract") case "GroupBy": statFn() - res, err := o.executeGroupBy(ctx, index, c, shards, opt) + res, err := o.executeGroupBy(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeGroupBy") case "Options": statFn() - res, err := o.executeOptionsCall(ctx, index, c, shards, opt) + res, err := o.executeOptionsCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": - res, err := o.executeIncludesColumnCall(ctx, index, c, shards, opt) + res, err := o.executeIncludesColumnCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": statFn() - res, err := o.executeFieldValueCall(ctx, index, c, shards, opt) + res, err := o.executeFieldValueCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": - res, err := o.executePrecomputedCall(ctx, index, c, shards, opt) + res, err := o.executePrecomputedCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": - res, err := o.executeUnionRows(ctx, index, c, shards, opt) + res, err := o.executeUnionRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": - res, err := o.executeConstRow(ctx, index, c) + res, err := o.executeConstRow(ctx, tableKeyer, c) return res, errors.Wrap(err, "executeConstRow") case "Limit": - res, err := o.executeLimitCall(ctx, index, c, shards, opt) + res, err := o.executeLimitCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeLimitCall") case "Percentile": - res, err := o.executePercentile(ctx, index, c, shards, opt) + res, err := o.executePercentile(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePercentile") // case "Delete": // statFn() //TODO(twg) need this? @@ -549,7 +544,7 @@ func (o *orchestrator) executeCall(ctx context.Context, index string, c *pql.Cal // return res, errors.Wrap(err, "executeDelete") default: // o.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() - res, err := o.executeBitmapCall(ctx, index, c, shards, opt) + res, err := o.executeBitmapCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") } } @@ -573,7 +568,7 @@ func (o *orchestrator) validateCallArgs(c *pql.Call) error { return nil } -func (o *orchestrator) executeOptionsCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { +func (o *orchestrator) executeOptionsCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeOptionsCall") defer span.Finish() @@ -594,11 +589,11 @@ func (o *orchestrator) executeOptionsCall(ctx context.Context, index string, c * return nil, errors.New(errors.ErrUncoded, "Query(): shards must be a list of unsigned integers") } } - return o.executeCall(ctx, index, c.Children[0], shards, optCopy) + return o.executeCall(ctx, tableKeyer, c.Children[0], shards, optCopy) } // executeIncludesColumnCall executes an IncludesColumn() call. -func (o *orchestrator) executeIncludesColumnCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (bool, error) { +func (o *orchestrator) executeIncludesColumnCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (bool, error) { // Get the shard containing the column, since that's the only // shard that needs to execute this query. var shard uint64 @@ -616,7 +611,7 @@ func (o *orchestrator) executeIncludesColumnCall(ctx context.Context, index stri return other || v.(bool) } - result, err := o.mapReduce(ctx, index, []uint64{shard}, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, []uint64{shard}, c, opt, reduceFn) if err != nil { return false, err } @@ -624,7 +619,7 @@ func (o *orchestrator) executeIncludesColumnCall(ctx context.Context, index stri } // executeFieldValueCall executes a FieldValue() call. -func (o *orchestrator) executeFieldValueCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { +func (o *orchestrator) executeFieldValueCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { fieldName, ok := c.Args["field"].(string) if !ok || fieldName == "" { return featurebase.ValCount{}, featurebase.ErrFieldRequired @@ -651,7 +646,7 @@ func (o *orchestrator) executeFieldValueCall(ctx context.Context, index string, return v } - result, err := o.mapReduce(ctx, index, []uint64{shard}, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, []uint64{shard}, c, opt, reduceFn) if err != nil { return featurebase.ValCount{}, errors.Wrap(err, "map reduce") } @@ -661,7 +656,7 @@ func (o *orchestrator) executeFieldValueCall(ctx context.Context, index string, } // executeLimitCall executes a Limit() call. -func (o *orchestrator) executeLimitCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.Row, error) { +func (o *orchestrator) executeLimitCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.Row, error) { bitmapCall := c.Children[0] limit, hasLimit, err := c.UintArg("limit") @@ -678,7 +673,7 @@ func (o *orchestrator) executeLimitCall(ctx context.Context, index string, c *pq } // Execute bitmap call, storing the full result on this node. - res, err := o.executeCall(ctx, index, bitmapCall, shards, opt) + res, err := o.executeCall(ctx, tableKeyer, bitmapCall, shards, opt) if err != nil { return nil, errors.Wrap(err, "limit map reduce") } @@ -737,7 +732,7 @@ func (o *orchestrator) executeLimitCall(ctx context.Context, index string, c *pq } // executeSum executes a Sum() call. -func (o *orchestrator) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { +func (o *orchestrator) executeSum(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSum") defer span.Finish() @@ -756,7 +751,7 @@ func (o *orchestrator) executeSum(ctx context.Context, index string, c *pql.Call return other.Add(v.(featurebase.ValCount)) } - result, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return featurebase.ValCount{}, err } @@ -769,7 +764,7 @@ func (o *orchestrator) executeSum(ctx context.Context, index string, c *pql.Call // scale summed response if it's a decimal field and this is // not a remote query (we're about to return to original client). if !opt.Remote { - field, err := o.schema.FieldInfo(ctx, index, fieldName) + field, err := o.schemaFieldInfo(ctx, tableKeyer, fieldName) if field == nil { return featurebase.ValCount{}, errors.Wrapf(err, "%q", fieldName) } @@ -786,7 +781,7 @@ func (o *orchestrator) executeSum(ctx context.Context, index string, c *pql.Call // executeDistinct executes a Distinct call on a field. It returns a // SignedRow for int fields and a *Row for set/mutex/time fields. -func (o *orchestrator) executeDistinct(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { +func (o *orchestrator) executeDistinct(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct") defer span.Finish() @@ -821,7 +816,7 @@ func (o *orchestrator) executeDistinct(ctx context.Context, index string, c *pql } } - result, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, errors.Wrap(err, "mapReduce") } @@ -833,7 +828,7 @@ func (o *orchestrator) executeDistinct(ctx context.Context, index string, c *pql } // executeMin executes a Min() call. -func (o *orchestrator) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { +func (o *orchestrator) executeMin(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMin") defer span.Finish() @@ -851,7 +846,7 @@ func (o *orchestrator) executeMin(ctx context.Context, index string, c *pql.Call return other.Smaller(v.(featurebase.ValCount)) } - result, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return featurebase.ValCount{}, err } @@ -864,7 +859,7 @@ func (o *orchestrator) executeMin(ctx context.Context, index string, c *pql.Call } // executeMax executes a Max() call. -func (o *orchestrator) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { +func (o *orchestrator) executeMax(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax") defer span.Finish() @@ -882,7 +877,7 @@ func (o *orchestrator) executeMax(ctx context.Context, index string, c *pql.Call return other.Larger(v.(featurebase.ValCount)) } - result, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return featurebase.ValCount{}, err } @@ -896,7 +891,7 @@ func (o *orchestrator) executeMax(ctx context.Context, index string, c *pql.Call // TODO(jaffee) fix this... valcountize assumes access to field details like base // executePercentile executes a Percentile() call. -func (o *orchestrator) executePercentile(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { +func (o *orchestrator) executePercentile(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ featurebase.ValCount, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executePercentile") defer span.Finish() @@ -923,7 +918,7 @@ func (o *orchestrator) executePercentile(ctx context.Context, index string, c *p if err != nil { return featurebase.ValCount{}, errors.New(errors.ErrUncoded, "Percentile(): field required") } - field, err := o.schema.FieldInfo(ctx, index, fieldName) + field, err := o.schemaFieldInfo(ctx, tableKeyer, fieldName) if err != nil { return featurebase.ValCount{}, ErrFieldNotFound } @@ -942,7 +937,7 @@ func (o *orchestrator) executePercentile(ctx context.Context, index string, c *p if filterCall != nil { minCall.Children = append(minCall.Children, filterCall) } - minVal, err := o.executeMin(ctx, index, minCall, shards, opt) + minVal, err := o.executeMin(ctx, tableKeyer, minCall, shards, opt) if err != nil { return featurebase.ValCount{}, errors.Wrap(err, "executing Min call for Percentile") } @@ -956,7 +951,7 @@ func (o *orchestrator) executePercentile(ctx context.Context, index string, c *p if filterCall != nil { maxCall.Children = append(maxCall.Children, filterCall) } - maxVal, err := o.executeMax(ctx, index, maxCall, shards, opt) + maxVal, err := o.executeMax(ctx, tableKeyer, maxCall, shards, opt) if err != nil { return featurebase.ValCount{}, errors.Wrap(err, "executing Max call for Percentile") } @@ -988,7 +983,7 @@ func (o *orchestrator) executePercentile(ctx context.Context, index string, c *p Op: pql.Token(pql.LT), Value: possibleNthVal, } - leftCountUint64, err := o.executeCount(ctx, index, countCall, shards, opt) + leftCountUint64, err := o.executeCount(ctx, tableKeyer, countCall, shards, opt) if err != nil { return featurebase.ValCount{}, errors.Wrap(err, "executing Count call L for Percentile") } @@ -999,7 +994,7 @@ func (o *orchestrator) executePercentile(ctx context.Context, index string, c *p Op: pql.Token(pql.GT), Value: possibleNthVal, } - rightCountUint64, err := o.executeCount(ctx, index, countCall, shards, opt) + rightCountUint64, err := o.executeCount(ctx, tableKeyer, countCall, shards, opt) if err != nil { return featurebase.ValCount{}, errors.Wrap(err, "executing Count call R for Percentile") } @@ -1036,7 +1031,7 @@ func cookValCount(val int64, cnt uint64, field *featurebase.FieldInfo) featureba } // executeMinRow executes a MinRow() call. -func (o *orchestrator) executeMinRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ interface{}, err error) { +func (o *orchestrator) executeMinRow(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMinRow") defer span.Finish() @@ -1066,11 +1061,11 @@ func (o *orchestrator) executeMinRow(ctx context.Context, index string, c *pql.C return vp } - return o.mapReduce(ctx, index, shards, c, opt, reduceFn) + return o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) } // executeMaxRow executes a MaxRow() call. -func (o *orchestrator) executeMaxRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ interface{}, err error) { +func (o *orchestrator) executeMaxRow(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxRow") defer span.Finish() @@ -1100,11 +1095,11 @@ func (o *orchestrator) executeMaxRow(ctx context.Context, index string, c *pql.C return vp } - return o.mapReduce(ctx, index, shards, c, opt, reduceFn) + return o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) } // executePrecomputedCall pretends to execute a call that we have a precomputed value for. -func (o *orchestrator) executePrecomputedCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ *featurebase.Row, err error) { +func (o *orchestrator) executePrecomputedCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ *featurebase.Row, err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executePrecomputedCall") defer span.Finish() result := featurebase.NewRow() @@ -1116,13 +1111,12 @@ func (o *orchestrator) executePrecomputedCall(ctx context.Context, index string, } // executeBitmapCall executes a call that returns a bitmap. -func (o *orchestrator) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ *featurebase.Row, err error) { - +func (o *orchestrator) executeBitmapCall(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (_ *featurebase.Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall") span.LogKV("pqlCallName", c.Name) defer span.Finish() - indexTag := "index:" + index + indexTag := "index:" + string(tableKeyer.Key()) metricName := "query_" + strings.ToLower(c.Name) + "_total" if c.Name == "Row" && c.HasConditionArg() { metricName = "query_row_bsi_total" @@ -1145,7 +1139,7 @@ func (o *orchestrator) executeBitmapCall(ctx context.Context, index string, c *p return other } - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, errors.Wrap(err, "map reduce") } @@ -1162,7 +1156,7 @@ func (e Error) Error() string { return string(e) } const ViewNotFound = Error("view not found") const FragmentNotFound = Error("fragment not found") -func (o *orchestrator) executeTopK(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { +func (o *orchestrator) executeTopK(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopK") defer span.Finish() @@ -1172,7 +1166,7 @@ func (o *orchestrator) executeTopK(ctx context.Context, index string, c *pql.Cal return ([]*featurebase.Row)(featurebase.AddBSI(x, y)) } - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, err } @@ -1225,7 +1219,7 @@ func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (o *orchestrator) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.PairsField, error) { +func (o *orchestrator) executeTopN(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN") defer span.Finish() @@ -1241,7 +1235,7 @@ func (o *orchestrator) executeTopN(ctx context.Context, index string, c *pql.Cal } // Execute original query. - pairs, err := o.executeTopNShards(ctx, index, c, shards, opt) + pairs, err := o.executeTopNShards(ctx, tableKeyer, c, shards, opt) if err != nil { return nil, errors.Wrap(err, "finding top results") } @@ -1262,7 +1256,7 @@ func (o *orchestrator) executeTopN(ctx context.Context, index string, c *pql.Cal sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := o.executeTopNShards(ctx, index, other, shards, opt) + trimmedList, err := o.executeTopNShards(ctx, tableKeyer, other, shards, opt) if err != nil { return nil, errors.Wrap(err, "retrieving full counts") } @@ -1277,7 +1271,7 @@ func (o *orchestrator) executeTopN(ctx context.Context, index string, c *pql.Cal }, nil } -func (o *orchestrator) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.PairsField, error) { +func (o *orchestrator) executeTopNShards(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards") defer span.Finish() @@ -1297,7 +1291,7 @@ func (o *orchestrator) executeTopNShards(ctx context.Context, index string, c *p return other } - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, err } @@ -1408,9 +1402,10 @@ func findGroupCounts(v interface{}) []featurebase.GroupCount { return nil } -func (o *orchestrator) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.GroupCounts, error) { +func (o *orchestrator) executeGroupBy(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.GroupCounts, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy") defer span.Finish() + // validate call if len(c.Children) == 0 { return nil, errors.New(errors.ErrUncoded, "need at least one child call") @@ -1484,7 +1479,7 @@ func (o *orchestrator) executeGroupBy(ctx context.Context, index string, c *pql. continue } - r, er := o.executeRows(ctx, index, child, shards, opt) + r, er := o.executeRows(ctx, tableKeyer, child, shards, opt) if er != nil { return nil, errors.Wrap(er, "getting rows for ") } @@ -1513,7 +1508,7 @@ func (o *orchestrator) executeGroupBy(ctx context.Context, index string, c *pql. return mergeGroupCounts(other, findGroupCounts(v), limit) } // Get full result set. - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, errors.Wrap(err, "mapReduce") } @@ -1573,7 +1568,7 @@ func (o *orchestrator) executeGroupBy(ctx context.Context, index string, c *pql. } opt.PreTranslated = true - aggregateCount, err := o.execute(ctx, index, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt) + aggregateCount, err := o.execute(ctx, tableKeyer, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt) if err != nil { return nil, err } @@ -1692,7 +1687,7 @@ func mergeGroupCounts(a, b []featurebase.GroupCount, limit int) []featurebase.Gr return ret } -func (o *orchestrator) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (featurebase.RowIDs, error) { +func (o *orchestrator) executeRows(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (featurebase.RowIDs, error) { // Fetch field name from argument. // Check "field" first for backwards compatibility. // TODO: remove at Pilosa 2.0 @@ -1746,7 +1741,7 @@ func (o *orchestrator) executeRows(ctx context.Context, index string, c *pql.Cal return other.Merge(v.(featurebase.RowIDs), limit) } // Get full result set. - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return nil, err } @@ -1784,7 +1779,7 @@ func (o *orchestrator) executeRows(ctx context.Context, index string, c *pql.Cal return results, nil } -func (o *orchestrator) executeExtract(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (featurebase.ExtractedIDMatrix, error) { +func (o *orchestrator) executeExtract(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (featurebase.ExtractedIDMatrix, error) { // Extract the column filter call. if len(c.Children) < 1 { return featurebase.ExtractedIDMatrix{}, errors.New(errors.ErrUncoded, "missing column filter in Extract") @@ -1824,7 +1819,7 @@ func (o *orchestrator) executeExtract(ctx context.Context, index string, c *pql. } // Get full result set. - other, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + other, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return featurebase.ExtractedIDMatrix{}, err } @@ -1835,7 +1830,7 @@ func (o *orchestrator) executeExtract(ctx context.Context, index string, c *pql. return results, nil } -func (o *orchestrator) executeConstRow(ctx context.Context, index string, c *pql.Call) (res *featurebase.Row, err error) { +func (o *orchestrator) executeConstRow(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call) (res *featurebase.Row, err error) { // Fetch user-provided columns list. ids, ok := c.Args["columns"].([]uint64) if !ok { @@ -1845,7 +1840,7 @@ func (o *orchestrator) executeConstRow(ctx context.Context, index string, c *pql return featurebase.NewRow(ids...), nil } -func (o *orchestrator) executeUnionRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.Row, error) { +func (o *orchestrator) executeUnionRows(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (*featurebase.Row, error) { // Turn UnionRows(Rows(...)) into Union(Row(...), ...). var rows []*pql.Call for _, child := range c.Children { @@ -1858,7 +1853,7 @@ func (o *orchestrator) executeUnionRows(ctx context.Context, index string, c *pq } // Execute the call. - rowsResult, err := o.executeCall(ctx, index, child, shards, opt) + rowsResult, err := o.executeCall(ctx, tableKeyer, child, shards, opt) if err != nil { return nil, err } @@ -1925,11 +1920,11 @@ func (o *orchestrator) executeUnionRows(ctx context.Context, index string, c *pq } // Execute the generated Union() call. - return o.executeBitmapCall(ctx, index, c, shards, opt) + return o.executeBitmapCall(ctx, tableKeyer, c, shards, opt) } // executeCount executes a count() call. -func (o *orchestrator) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (uint64, error) { +func (o *orchestrator) executeCount(ctx context.Context, tableKeyer dax.TableKeyer, c *pql.Call, shards []uint64, opt *featurebase.ExecOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") defer span.Finish() @@ -1943,7 +1938,7 @@ func (o *orchestrator) executeCount(ctx context.Context, index string, c *pql.Ca // If the child is distinct/similar, execute it directly here and count the result. if child.Type == pql.PrecallGlobal { - result, err := o.executeCall(ctx, index, child, shards, opt) + result, err := o.executeCall(ctx, tableKeyer, child, shards, opt) if err != nil { return 0, err } @@ -1966,7 +1961,7 @@ func (o *orchestrator) executeCount(ctx context.Context, index string, c *pql.Ca return other + v.(uint64) } - result, err := o.mapReduce(ctx, index, shards, c, opt, reduceFn) + result, err := o.mapReduce(ctx, tableKeyer, shards, c, opt, reduceFn) if err != nil { return 0, err } @@ -2004,10 +1999,12 @@ func (o *orchestrator) remoteExec(ctx context.Context, node dax.Address, index s // mapReduce has to ensure that it never returns before any work it spawned has // terminated. It's not enough to cancel the jobs; we have to wait for them to be // done, or we can unmap resources they're still using. -func (o *orchestrator) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *featurebase.ExecOptions, reduceFn reduceFunc) (result interface{}, err error) { +func (o *orchestrator) mapReduce(ctx context.Context, tableKeyer dax.TableKeyer, shards []uint64, c *pql.Call, opt *featurebase.ExecOptions, reduceFn reduceFunc) (result interface{}, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapReduce") defer span.Finish() + index := string(tableKeyer.Key()) + ch := make(chan mapResponse) // Wrap context with a cancel to kill goroutines on exit. @@ -2114,7 +2111,7 @@ func makeEmbeddedDataForShards(allRows []*featurebase.Row, shards []uint64) []*f return newRows } -func (o *orchestrator) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, index string, nodes []controller.ComputeNode, c *pql.Call, opt *featurebase.ExecOptions, reduceFn reduceFunc) (reterr error) { +func (o *orchestrator) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, index string, nodes []dax.ComputeNode, c *pql.Call, opt *featurebase.ExecOptions, reduceFn reduceFunc) (reterr error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.mapper") defer span.Finish() @@ -2524,13 +2521,17 @@ func fieldValidateValue(f *featurebase.FieldInfo, val interface{}) error { return nil } -func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index string, columnKeys map[string]map[string]uint64, rowKeys map[string]map[string]map[string]uint64) (*pql.Call, error) { +func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, tableKeyer dax.TableKeyer, columnKeys map[string]map[string]uint64, rowKeys map[string]map[string]map[string]uint64) (*pql.Call, error) { + index := string(tableKeyer.Key()) + // Check for an overriding 'index' argument. // This also applies to all child calls. if callIndex := c.CallIndex(); callIndex != "" { index = callIndex + tableKeyer = dax.StringTableKeyer(index) } - idx, err := o.schema.IndexInfo(ctx, index) + + idx, err := o.schemaIndexInfo(ctx, tableKeyer) if err != nil { return nil, errors.Wrapf(err, "translating query on index %q", index) } @@ -2542,7 +2543,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str switch c.Name { case "Set", "Store": if field, err := c.FieldArg(); err == nil { - f, err := o.schema.FieldInfo(ctx, index, field) + f, err := o.schemaFieldInfo(ctx, tableKeyer, field) if err != nil { return nil, errors.Wrapf(err, "validating value for field %q", field) } @@ -2568,7 +2569,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str case "Clear", "Row", "Range", "ClearRow": if field, err := c.FieldArg(); err == nil { - f, err := o.schema.FieldInfo(ctx, index, field) + f, err := o.schemaFieldInfo(ctx, tableKeyer, field) if err != nil { return nil, errors.Wrapf(err, "validating value for field %q", field) } @@ -2655,7 +2656,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str return nil, errors.New(errors.ErrUncoded, "missing field") } - f, err := o.schema.FieldInfo(ctx, index, field) + f, err := o.schemaFieldInfo(ctx, tableKeyer, field) if err != nil { return nil, errors.Wrapf(err, "validating value for field %q", field) } @@ -2725,7 +2726,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str // Translate the previous row key. if prev, ok := c.Args["previous"]; ok { // Validate the type. - f, err := o.schema.FieldInfo(ctx, index, field) + f, err := o.schemaFieldInfo(ctx, tableKeyer, field) if err != nil { return nil, errors.Wrapf(err, "validating value for field %q", field) } @@ -2756,7 +2757,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str if err != nil || fieldName == "" { return nil, fmt.Errorf("cannot read field name for Rows call") } - if f, err := o.schema.FieldInfo(ctx, index, fieldName); err != nil { + if f, err := o.schemaFieldInfo(ctx, tableKeyer, fieldName); err != nil { return nil, errors.Wrapf(err, "getting field %q", fieldName) } else if !f.Options.Keys { return nil, fmt.Errorf("'%s' is not a set/mutex/time field with a string key", fieldName) @@ -2785,7 +2786,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str // Translate child calls. for i, child := range c.Children { - translated, err := o.translateCall(ctx, child, index, columnKeys, rowKeys) + translated, err := o.translateCall(ctx, child, tableKeyer, columnKeys, rowKeys) if err != nil { return nil, err } @@ -2799,7 +2800,7 @@ func (o *orchestrator) translateCall(ctx context.Context, c *pql.Call, index str continue } - translated, err := o.translateCall(ctx, argCall, index, columnKeys, rowKeys) + translated, err := o.translateCall(ctx, argCall, tableKeyer, columnKeys, rowKeys) if err != nil { return nil, err } @@ -2830,10 +2831,12 @@ func (o *orchestrator) callZero(c *pql.Call) *pql.Call { } } -func (o *orchestrator) translateResults(ctx context.Context, index string, idx *featurebase.IndexInfo, calls []*pql.Call, results []interface{}, memoryAvailable int64) (err error) { +func (o *orchestrator) translateResults(ctx context.Context, qtbl *dax.QualifiedTable, calls []*pql.Call, results []interface{}, memoryAvailable int64) (err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.translateResults") defer span.Finish() + idx := featurebase.TableToIndexInfo(&qtbl.Table) + idMap := make(map[uint64]string) if idx.Options.Keys { // Collect all index ids. @@ -2843,13 +2846,13 @@ func (o *orchestrator) translateResults(ctx context.Context, index string, idx * return err } } - if idMap, err = o.trans.TranslateIndexIDSet(ctx, index, idSet); err != nil { + if idMap, err = o.trans.TranslateIndexIDSet(ctx, string(qtbl.Key()), idSet); err != nil { return err } } for i := range results { - results[i], err = o.translateResult(ctx, idx, calls[i], results[i], idMap) + results[i], err = o.translateResult(ctx, qtbl, calls[i], results[i], idMap) if err != nil { return err } @@ -2891,13 +2894,13 @@ func (o *orchestrator) howToTranslate(ctx context.Context, idx *featurebase.Inde // First get the index and field the row specifies (if any). rowIdx = idx if row.Index != "" && row.Index != idx.Name { - rowIdx, err = o.schema.IndexInfo(ctx, row.Index) + rowIdx, err = o.schemaIndexInfo(ctx, dax.StringTableKeyer(row.Index)) if err != nil { return nil, nil, 0, errors.Wrapf(err, "got a row with unknown index: %s", row.Index) } } if row.Field != "" { - rowField, err = o.schema.FieldInfo(ctx, row.Index, row.Field) + rowField, err = o.schemaFieldInfo(ctx, dax.StringTableKeyer(row.Index), row.Field) if err != nil { return nil, nil, 0, errors.Wrapf(err, "got a row with unknown index/field %s/%s", idx.Name, row.Field) } @@ -2907,7 +2910,7 @@ func (o *orchestrator) howToTranslate(ctx context.Context, idx *featurebase.Inde if rowField != nil { // Handle the case where field has a foreign index. if rowField.Options.ForeignIndex != "" { - fidx, err := o.schema.IndexInfo(ctx, rowField.Options.ForeignIndex) + fidx, err := o.schemaIndexInfo(ctx, dax.StringTableKeyer(rowField.Options.ForeignIndex)) if err != nil { return nil, nil, 0, errors.Errorf("foreign index %s not found for field %s in index %s", rowField.Options.ForeignIndex, rowField.Name, rowIdx.Name) } @@ -2960,7 +2963,7 @@ func (o *orchestrator) collectResultIDs(ctx context.Context, idx *featurebase.In } // preTranslateMatrixSet translates the IDs of a set field in an extracted matrix. -func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebase.ExtractedIDMatrix, fieldIdx uint, index, field string) (map[uint64]string, error) { +func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebase.ExtractedIDMatrix, fieldIdx uint, tableKeyer dax.TableKeyer, field string) (map[uint64]string, error) { ids := make(map[uint64]struct{}, len(mat.Columns)) for _, col := range mat.Columns { for _, v := range col.Rows[fieldIdx] { @@ -2968,10 +2971,14 @@ func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebas } } + index := string(tableKeyer.Key()) + return o.trans.TranslateFieldIDs(ctx, index, field, ids) } -func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.IndexInfo, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) { +func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedTable, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) { + idx := featurebase.TableToIndexInfo(&qtbl.Table) + switch result := result.(type) { case *featurebase.Row: rowIdx, rowField, strategy, err := o.howToTranslate(ctx, idx, result) @@ -2994,8 +3001,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind } result.Keys = keys case byRowFieldForeignIndex: - idx, err = o.schema.IndexInfo(ctx, rowField.Options.ForeignIndex) - if err != nil { + if _, err := o.schemaIndexInfo(ctx, dax.StringTableKeyer(rowField.Options.ForeignIndex)); err != nil { return nil, errors.Wrapf(err, "foreign index %s not found for field %s in index %s", rowField.Options.ForeignIndex, rowField.Name, rowIdx.Name) } for _, segment := range result.Segments { @@ -3028,7 +3034,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind return nil, nil } - field, err := o.schema.FieldInfo(ctx, idx.Name, fieldName) + field, err := o.schemaFieldInfo(ctx, qtbl, fieldName) if err != nil { return nil, nil } @@ -3059,7 +3065,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind case featurebase.PairField: if fieldName := callArgString(call, "field"); fieldName != "" { - field, err := o.schema.FieldInfo(ctx, idx.Name, fieldName) + field, err := o.schemaFieldInfo(ctx, qtbl, fieldName) if err != nil { return nil, fmt.Errorf("field %q not found", fieldName) } @@ -3083,7 +3089,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind case *featurebase.PairsField: if fieldName := callArgString(call, "_field"); fieldName != "" { - field, err := o.schema.FieldInfo(ctx, idx.Name, fieldName) + field, err := o.schemaFieldInfo(ctx, qtbl, fieldName) if err != nil { return nil, errors.Wrapf(err, "field '%q'", fieldName) } @@ -3113,7 +3119,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind groups := result.Groups() for _, gl := range groups { for _, g := range gl.Group { - field, err := o.schema.FieldInfo(ctx, idx.Name, g.Field) + field, err := o.schemaFieldInfo(ctx, qtbl, g.Field) if err != nil { return nil, errors.Wrapf(err, "getting field '%q", g.Field) } @@ -3194,7 +3200,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind Field: fieldName, } - if field, err := o.schema.FieldInfo(ctx, idx.Name, fieldName); err != nil { + if field, err := o.schemaFieldInfo(ctx, qtbl, fieldName); err != nil { return nil, errors.Wrapf(err, "'%q'", fieldName) } else if field.Options.Keys { keys, err := o.trans.TranslateFieldListIDs(ctx, idx.Name, field.Name, result) @@ -3214,7 +3220,7 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind fields := make([]featurebase.ExtractedTableField, len(result.Fields)) mappers := make([]fieldMapper, len(result.Fields)) for i, v := range result.Fields { - field, err := o.schema.FieldInfo(ctx, idx.Name, v) + field, err := o.schemaFieldInfo(ctx, qtbl, v) if err != nil { return nil, errors.Wrapf(err, "'%q'", v) } @@ -3244,9 +3250,9 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind case FieldTypeSet, FieldTypeTime: if field.Options.Keys { datatype = "[]string" - translations, err := o.preTranslateMatrixSet(ctx, result, uint(i), idx.Name, field.Name) + translations, err := o.preTranslateMatrixSet(ctx, result, uint(i), qtbl, field.Name) if err != nil { - return nil, errors.Wrapf(err, "translating IDs of field %q", v) + return nil, errors.Wrapf(err, "orch: translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { keys := make([]string, len(ids)) @@ -3267,9 +3273,9 @@ func (o *orchestrator) translateResult(ctx context.Context, idx *featurebase.Ind case FieldTypeMutex: if field.Options.Keys { datatype = "string" - translations, err := o.preTranslateMatrixSet(ctx, result, uint(i), idx.Name, field.Name) + translations, err := o.preTranslateMatrixSet(ctx, result, uint(i), qtbl, field.Name) if err != nil { - return nil, errors.Wrapf(err, "translating IDs of field %q", v) + return nil, errors.Wrapf(err, "orch: translating IDs of field %q", v) } mapper = func(ids []uint64) (interface{}, error) { switch len(ids) { @@ -3460,27 +3466,83 @@ func callArgString(call *pql.Call, key string) string { type qualifiedOrchestrator struct { *orchestrator - qual dax.TableQualifier - schemar schemar.Schemar + qual dax.TableQualifier } -func newQualifiedOrchestrator(orch *orchestrator, qual dax.TableQualifier, schemar schemar.Schemar) *qualifiedOrchestrator { +func newQualifiedOrchestrator(orch *orchestrator, qual dax.TableQualifier) *qualifiedOrchestrator { return &qualifiedOrchestrator{ orchestrator: orch, qual: qual, - schemar: schemar, } } func (o *qualifiedOrchestrator) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q *pql.Query, shards []uint64, opt *featurebase.ExecOptions) (featurebase.QueryResponse, error) { resp := featurebase.QueryResponse{} - tbl, ok := tableKeyer.(*dax.Table) - if !ok { - return resp, errors.New(errors.ErrUncoded, "qualifiedOrchestrator.Execute expects a dax.Table") - } + var qtbl *dax.QualifiedTable - qtbl := dax.NewQualifiedTable(o.qual, tbl) + switch keyer := tableKeyer.(type) { + case *dax.Table: + qtbl = dax.NewQualifiedTable(o.qual, keyer) + case *dax.QualifiedTable: + qtbl = keyer + default: + return resp, errors.Errorf("qualifiedOrchestrator.Execute expects a *dax.Table or *dax.QualifiedTable, but got: %T", tableKeyer) + } return o.orchestrator.Execute(ctx, qtbl, q, shards, opt) } + +// schemaFieldInfo is a function introduced when we replaced +// `schema.FieldInfo()` calls, where schema was a `featurebase.SchemaInfoAPI` to +// `schema.Table().Field()` calls, where schema is a `pilosa.SchemaAPI`. In the +// future, when we're no longer dealing with IndexInfo and FieldInfo, and +// instead use dax.Table and dax.Field, this helper function can be factored +// out. +func (o *orchestrator) schemaFieldInfo(ctx context.Context, tableKeyer dax.TableKeyer, fieldName string) (*featurebase.FieldInfo, error) { + var tbl *dax.Table + var err error + + switch v := tableKeyer.(type) { + case *dax.QualifiedTable: + tbl = &v.Table + case *dax.Table: + tbl = v + case dax.StringTableKeyer: + tbl, err = o.schema.TableByName(ctx, dax.TableName(v)) + if err != nil { + return nil, errors.Wrapf(err, "getting table by name: %s", v) + } + default: + return nil, errors.Errorf("unsupport table keyer type in schemaFieldInfo: %T", tableKeyer) + } + + fld, ok := tbl.Field(dax.FieldName(fieldName)) + if !ok { + return nil, errors.Errorf("field not found: %s", fieldName) + } + + return featurebase.FieldToFieldInfo(fld), nil +} + +// schemaIndexInfo - see comment on schemaFieldInfo. +func (o *orchestrator) schemaIndexInfo(ctx context.Context, tableKeyer dax.TableKeyer) (*featurebase.IndexInfo, error) { + var tbl *dax.Table + var err error + + switch v := tableKeyer.(type) { + case *dax.QualifiedTable: + tbl = &v.Table + case *dax.Table: + tbl = v + case dax.StringTableKeyer: + tbl, err = o.schema.TableByName(ctx, dax.TableName(v)) + if err != nil { + return nil, errors.Wrapf(err, "getting table by name: %s", v) + } + default: + return nil, errors.Errorf("unsupport table keyer type in schemaIndexInfo: %T", tableKeyer) + } + + return featurebase.TableToIndexInfo(tbl), nil +} diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index 8074b4c2b..79bfb2fcc 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "strings" + "sync" "time" featurebase "github.com/molecula/featurebase/v3" @@ -30,9 +31,13 @@ import ( // that the externally-facing Molecula API would proxy query requests to a pool // of "Queryer" nodes, which handle incoming query requests. type Queryer struct { - orchestrator *orchestrator + mu sync.RWMutex + orchestrators map[dax.TableQualifier]*qualifiedOrchestrator - mds MDS + fbClient *featurebase.InternalClient + + noder dax.Noder + schemar dax.Schemar logger logger.Logger } @@ -40,9 +45,10 @@ type Queryer struct { // New returns a new instance of Queryer. func New(cfg Config) *Queryer { q := &Queryer{ - mds: NewNopMDS(), - orchestrator: nil, - logger: logger.NopLogger, + noder: dax.NewNopNoder(), + schemar: dax.NewNopSchemar(), + orchestrators: make(map[dax.TableQualifier]*qualifiedOrchestrator), + logger: logger.NopLogger, } if cfg.Logger != nil { @@ -52,8 +58,63 @@ func New(cfg Config) *Queryer { return q } -func (q *Queryer) SetMDS(mds MDS) error { - q.mds = mds +// Orchestrator gets (or creates) an instance of qualifiedOrchestrator based on +// the provided dax.TableQualifier. +func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { + // Try to get orchestrator under a read lock first. + if orch := func() *qualifiedOrchestrator { + q.mu.RLock() + defer q.mu.RUnlock() + if orch, ok := q.orchestrators[qual]; ok { + return orch + } + return nil + }(); orch != nil { + return orch + } + + // Since we didn't find an orchestrator under a read lock, obtain a write + // lock and try a read/write. + q.mu.Lock() + defer q.mu.Unlock() + if orch, ok := q.orchestrators[qual]; ok { + return orch + } + + sapi := newQualifiedSchemaAPI(qual, q.schemar) + + orch := &orchestrator{ + schema: sapi, + trans: NewMDSTranslator(q.noder, q.schemar), + topology: &MDSTopology{noder: q.noder}, + // TODO(jaffee) using default http.Client probably bad... need to set some timeouts. + client: q.fbClient, + stats: stats.NopStatsClient, + logger: q.logger, + } + + qorch := newQualifiedOrchestrator(orch, qual) + q.orchestrators[qual] = qorch + + return qorch +} + +func (q *Queryer) SetNoder(noder dax.Noder) error { + q.noder = noder + return nil +} + +func (q *Queryer) SetSchemar(schemar dax.Schemar) error { + q.schemar = schemar + return nil +} + +func (q *Queryer) Start() error { + if q.noder == nil { + return errors.New(errors.ErrUncoded, "queryer requires noder to be configured") + } else if q.schemar == nil { + return errors.New(errors.ErrUncoded, "queryer requires schemar to be configured") + } // fbClient is an instance of internal client. It's used in one place in the // orchestrator (o.client.QueryNode()), but in that case, the host is @@ -67,26 +128,8 @@ func (q *Queryer) SetMDS(mds MDS) error { if err != nil { return errors.Wrap(err, "setting up internal client") } + q.fbClient = fbClient - q.orchestrator = &orchestrator{ - schema: NewSchemaInfoAPI(q.mds), - trans: NewMDSTranslator(q.mds), - topology: &MDSTopology{mds: q.mds}, - // TODO(jaffee) using default http.Client probably bad... need to set some timeouts. - client: fbClient, - stats: stats.NopStatsClient, - logger: q.logger, - } - - return nil -} - -func (q *Queryer) Start() error { - if q.mds == nil { - return errors.New(errors.ErrUncoded, "queryer requires mds to be configured") - } else if q.orchestrator == nil { - return errors.New(errors.ErrUncoded, "queryer requires orchestrator to be configured") - } return nil } @@ -123,13 +166,10 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str } // SchemaAPI - sapi := NewQualifiedSchemaAPI(qual, q.mds) - - // Orchestrator - orch := newQualifiedOrchestrator(q.orchestrator, qual, q.mds) + sapi := newQualifiedSchemaAPI(qual, q.schemar) // Importer - imp := idkmds.NewImporter(q.mds, qual, nil) + imp := idkmds.NewImporter(q.noder, q.schemar, qual, nil) // TODO(tlt): this obviously doesn't work; we don't have an API here. We // need a dax-compatible implementation of the SystemAPI (or at least a @@ -138,7 +178,7 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str systemLayer := systemlayer.NewSystemLayer() - pl := planner.NewExecutionPlanner(orch, sapi, sysapi, systemLayer, imp, q.orchestrator.logger, sql) + pl := planner.NewExecutionPlanner(q.Orchestrator(qual), sapi, sysapi, systemLayer, imp, q.logger, sql) planOp, err := pl.CompilePlan(ctx, st) if err != nil { @@ -214,17 +254,12 @@ func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table d return nil, errors.Errorf("must have exactly 1 query, but got: %+v", qry.Calls) } - qtid, err := q.mds.TableID(ctx, qual, dax.TableName(table)) + qtbl, err := q.schemar.TableByName(ctx, qual, dax.TableName(table)) if err != nil { - return nil, errors.Wrap(err, "converting index to qualified table id") + return nil, errors.Wrap(err, "converting index to qualified table") } - qtbl, err := q.mds.Table(ctx, qtid) - if err != nil { - return nil, errors.Wrap(err, "getting table for qtid") - } - - results, err := q.orchestrator.Execute(ctx, qtbl, qry, nil, &featurebase.ExecOptions{}) + results, err := q.Orchestrator(qual).Execute(ctx, qtbl, qry, nil, &featurebase.ExecOptions{}) if err != nil { return nil, errors.Wrap(err, "orchestrator.Execute") } diff --git a/dax/queryer/schema_api.go b/dax/queryer/schema_api.go index ed74c2406..48c465aa4 100644 --- a/dax/queryer/schema_api.go +++ b/dax/queryer/schema_api.go @@ -5,7 +5,6 @@ import ( pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/dax/mds/schemar" "github.com/molecula/featurebase/v3/errors" ) @@ -18,34 +17,28 @@ var _ pilosa.SchemaAPI = (*qualifiedSchemaAPI)(nil) // that lookup/conversion. type qualifiedSchemaAPI struct { qual dax.TableQualifier - schemar schemar.Schemar + schemar dax.Schemar } -func NewQualifiedSchemaAPI(qual dax.TableQualifier, schemar schemar.Schemar) *qualifiedSchemaAPI { +func newQualifiedSchemaAPI(qual dax.TableQualifier, schema dax.Schemar) *qualifiedSchemaAPI { return &qualifiedSchemaAPI{ qual: qual, - schemar: schemar, + schemar: schema, } } func (s *qualifiedSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) { - qtid, err := s.schemar.TableID(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) if err != nil { return nil, errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname) } - - qtbl, err := s.schemar.Table(ctx, qtid) - if err != nil { - return nil, errors.Wrapf(err, "getting table: %s", qtid) - } - return &qtbl.Table, nil } func (s *qualifiedSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) { qtid := dax.NewQualifiedTableID(s.qual, tid) - qtbl, err := s.schemar.Table(ctx, qtid) + qtbl, err := s.schemar.TableByID(ctx, qtid) if err != nil { return nil, errors.Wrapf(err, "getting table: %s", qtid) } @@ -73,28 +66,28 @@ func (s *qualifiedSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) er } func (s *qualifiedSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error { - qtid, err := s.schemar.TableID(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) if err != nil { - return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) } - return s.schemar.CreateField(ctx, qtid, fld) + return s.schemar.CreateField(ctx, qtbl.QualifiedID(), fld) } func (s *qualifiedSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error { - qtid, err := s.schemar.TableID(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) if err != nil { - return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) } - return s.schemar.DropTable(ctx, qtid) + return s.schemar.DropTable(ctx, qtbl.QualifiedID()) } func (s *qualifiedSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error { - qtid, err := s.schemar.TableID(ctx, s.qual, tname) + qtid, err := s.schemar.TableByName(ctx, s.qual, tname) if err != nil { - return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) } - return s.schemar.DropField(ctx, qtid, fname) + return s.schemar.DropField(ctx, qtid.Key().QualifiedTableID(), fname) } diff --git a/dax/queryer/schema_info_api.go b/dax/queryer/schema_info_api.go deleted file mode 100644 index 3c3984c69..000000000 --- a/dax/queryer/schema_info_api.go +++ /dev/null @@ -1,79 +0,0 @@ -package queryer - -import ( - "context" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/dax/mds/schemar" - "github.com/molecula/featurebase/v3/errors" -) - -// Ensure type implements interface. -var _ pilosa.SchemaInfoAPI = (*schemaInfoAPI)(nil) - -type schemaInfoAPI struct { - schemar schemar.Schemar -} - -func NewSchemaInfoAPI(schemar schemar.Schemar) *schemaInfoAPI { - return &schemaInfoAPI{ - schemar: schemar, - } -} - -func (a *schemaInfoAPI) IndexInfo(ctx context.Context, indexName string) (*pilosa.IndexInfo, error) { - qtid := dax.TableKey(indexName).QualifiedTableID() - tbl, err := a.schemar.Table(ctx, qtid) - if err != nil { - return nil, errors.Wrap(err, "getting table for indexinfo") - } - - return daxTableToFeaturebaseIndexInfo(tbl, false) -} - -func (a *schemaInfoAPI) FieldInfo(ctx context.Context, indexName, fieldName string) (*pilosa.FieldInfo, error) { - qtid := dax.TableKey(indexName).QualifiedTableID() - tbl, err := a.schemar.Table(ctx, qtid) - fldName := dax.FieldName(fieldName) - - if err != nil { - return nil, errors.Wrap(err, "getting table for fieldinfo") - } - - fld, ok := tbl.Field(dax.FieldName(fieldName)) - if !ok { - return nil, dax.NewErrFieldDoesNotExist(fldName) - } - - return pilosa.FieldToFieldInfo(fld), nil -} - -// TODO(tlt): try to get rid of this in favor of pilosa.TableToIndexInfo. -// daxTableToFeaturebaseIndexInfo converts a dax.Table to a -// featurebase.IndexInfo. If useName is true, the IndexInfo.Name value will -// be set to the qualified table name. Otherwise it will be set to the table key. -func daxTableToFeaturebaseIndexInfo(qtbl *dax.QualifiedTable, useName bool) (*pilosa.IndexInfo, error) { - name := string(qtbl.Key()) - if useName { - name = string(qtbl.Name) - } - ii := &pilosa.IndexInfo{ - Name: name, - CreatedAt: 0, - Options: pilosa.IndexOptions{ - Keys: qtbl.StringKeys(), - TrackExistence: true, - }, - ShardWidth: pilosa.ShardWidth, - } - - // fields - fields := make([]*pilosa.FieldInfo, len(qtbl.Fields)) - for i := range qtbl.Fields { - fields[i] = pilosa.FieldToFieldInfo(qtbl.Fields[i]) - } - ii.Fields = fields - - return ii, nil -} diff --git a/dax/queryer/service/queryer.go b/dax/queryer/service/queryer.go index a51c480fc..efa5cedb0 100644 --- a/dax/queryer/service/queryer.go +++ b/dax/queryer/service/queryer.go @@ -50,6 +50,8 @@ func (q *queryerService) HTTPHandler() http.Handler { } func (q *queryerService) SetMDS(addr dax.Address) error { - q.queryer.SetMDS(mdsclient.New(addr, q.logger)) + mdscli := mdsclient.New(addr, q.logger) + q.queryer.SetNoder(mdscli) + q.queryer.SetSchemar(mdscli) return nil } diff --git a/dax/queryer/translator.go b/dax/queryer/translator.go index 8ac624529..191d3bc8d 100644 --- a/dax/queryer/translator.go +++ b/dax/queryer/translator.go @@ -14,15 +14,17 @@ import ( ) // Ensure type implements interface. -var _ Translator = (*MDSTranslator)(nil) +var _ Translator = (*mdsTranslator)(nil) -type MDSTranslator struct { - mds MDS +type mdsTranslator struct { + noder dax.Noder + schemar dax.Schemar } -func NewMDSTranslator(mds MDS) *MDSTranslator { - return &MDSTranslator{ - mds: mds, +func NewMDSTranslator(noder dax.Noder, schemar dax.Schemar) *mdsTranslator { + return &mdsTranslator{ + noder: noder, + schemar: schemar, } } @@ -37,11 +39,11 @@ func fbClient(address dax.Address) (*featurebase_client.Client, error) { ) } -func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) { +func (m *mdsTranslator) CreateIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) { tkey := dax.TableKey(table) qtid := tkey.QualifiedTableID() - qtbl, err := m.mds.Table(ctx, qtid) + qtbl, err := m.schemar.TableByID(ctx, qtid) if err != nil { return nil, errors.Wrap(err, "getting table") } @@ -53,7 +55,7 @@ func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys out := make(map[string]uint64) for pNum := range pMap { - address, err := m.mds.IngestPartition(ctx, qtid, pNum) + address, err := m.noder.IngestPartition(ctx, qtid, pNum) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, pNum) } @@ -78,9 +80,9 @@ func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys return out, nil } -func (m *MDSTranslator) CreateFieldKeys(ctx context.Context, table string, field string, keys []string) (map[string]uint64, error) { +func (m *mdsTranslator) CreateFieldKeys(ctx context.Context, table string, field string, keys []string) (map[string]uint64, error) { qtid := dax.TableKey(table).QualifiedTableID() - address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0)) + address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0)) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, dax.PartitionNum(0)) } @@ -96,11 +98,11 @@ func (m *MDSTranslator) CreateFieldKeys(ctx context.Context, table string, field return fbClient.CreateFieldKeys(fld, keys...) } -func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) { +func (m *mdsTranslator) FindIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) { tkey := dax.TableKey(table) qtid := tkey.QualifiedTableID() - qtbl, err := m.mds.Table(ctx, qtid) + qtbl, err := m.schemar.TableByID(ctx, qtid) if err != nil { return nil, errors.Wrap(err, "getting table") } @@ -115,7 +117,7 @@ func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys [] pNums = append(pNums, k) } - translateNodes, err := m.mds.TranslateNodes(ctx, qtid, pNums...) + translateNodes, err := m.noder.TranslateNodes(ctx, qtid, pNums...) if err != nil { return nil, errors.Wrapf(err, "getting translate nodes for partitions on table: %s", table) } @@ -149,9 +151,9 @@ func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys [] return out, nil } -func (m *MDSTranslator) FindFieldKeys(ctx context.Context, table, field string, keys []string) (map[string]uint64, error) { +func (m *mdsTranslator) FindFieldKeys(ctx context.Context, table, field string, keys []string) (map[string]uint64, error) { qtid := dax.TableKey(table).QualifiedTableID() - address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0)) + address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0)) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, dax.PartitionNum(0)) } @@ -167,7 +169,7 @@ func (m *MDSTranslator) FindFieldKeys(ctx context.Context, table, field string, return fbClient.FindFieldKeys(fld, keys...) } -func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) { +func (m *mdsTranslator) TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) { idsByPartition := splitIDsByPartition(index, ids, 1<<20) // TODO(jaffee), don't hardcode shardwidth...need to get this from index info daxPartitions := make([]dax.PartitionNum, 0) for partition := range idsByPartition { @@ -176,7 +178,7 @@ func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids qtid := dax.TableKey(index).QualifiedTableID() - nodes, err := m.mds.TranslateNodes(ctx, qtid, daxPartitions...) + nodes, err := m.noder.TranslateNodes(ctx, qtid, daxPartitions...) if err != nil { return nil, errors.Wrapf(err, "calling translate-nodes on table: %s, partitions: %v", index, daxPartitions) } @@ -210,7 +212,7 @@ func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids return ret, nil } -func (m *MDSTranslator) TranslateIndexIDSet(ctx context.Context, table string, ids map[uint64]struct{}) (map[uint64]string, error) { +func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, ids map[uint64]struct{}) (map[uint64]string, error) { idList := make([]uint64, 0, len(ids)) for id := range ids { idList = append(idList, id) @@ -227,7 +229,7 @@ func (m *MDSTranslator) TranslateIndexIDSet(ctx context.Context, table string, i } return ret, nil } -func (m *MDSTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) { +func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) { idList := make([]uint64, 0, len(ids)) for id := range ids { idList = append(idList, id) @@ -244,9 +246,9 @@ func (m *MDSTranslator) TranslateFieldIDs(ctx context.Context, table, field stri } return ret, nil } -func (m *MDSTranslator) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) { +func (m *mdsTranslator) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) { qtid := dax.TableKey(index).QualifiedTableID() - address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0)) + address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0)) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", index, dax.PartitionNum(0)) } diff --git a/dax/schema.go b/dax/schema.go new file mode 100644 index 000000000..f7b16abc4 --- /dev/null +++ b/dax/schema.go @@ -0,0 +1,51 @@ +package dax + +import "context" + +// Schemar is similar to the pilosa.SchemaAPI interface, but it takes +// TableQualifiers into account. +type Schemar interface { + TableByName(ctx context.Context, qual TableQualifier, tname TableName) (*QualifiedTable, error) + TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) + Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error) + + CreateTable(ctx context.Context, qtbl *QualifiedTable) error + CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error + + DropTable(ctx context.Context, qtid QualifiedTableID) error + DropField(ctx context.Context, qtid QualifiedTableID, fname FieldName) error +} + +////////////////////////////////////////////// + +// Ensure type implements interface. +var _ Schemar = &NopSchemar{} + +// NopSchemar is a no-op implementation of the Schemar interface. +type NopSchemar struct{} + +func NewNopSchemar() *NopSchemar { + return &NopSchemar{} +} + +func (s *NopSchemar) TableByName(context.Context, TableQualifier, TableName) (*QualifiedTable, error) { + return nil, nil +} +func (s *NopSchemar) TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) { + return nil, nil +} +func (s *NopSchemar) Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error) { + return nil, nil +} +func (s *NopSchemar) CreateTable(ctx context.Context, qtbl *QualifiedTable) error { + return nil +} +func (s *NopSchemar) DropTable(ctx context.Context, qtid QualifiedTableID) error { + return nil +} +func (s *NopSchemar) CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error { + return nil +} +func (s *NopSchemar) DropField(ctx context.Context, qtid QualifiedTableID, fld FieldName) error { + return nil +} diff --git a/idk/ingest.go b/idk/ingest.go index a7a272499..d902e80d1 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -1027,7 +1027,7 @@ func (m *Main) setupClient() (*tls.Config, error) { m.SchemaManager = mds.NewSchemaManager(dax.Address(m.MDSAddress), qual, m.log) m.NewImporterFn = func() pilosacore.Importer { - return mds.NewImporter(mdsClient, qtbl.Qualifier(), &qtbl.Table) + return mds.NewImporter(mdsClient, mdsClient, qtbl.Qualifier(), &qtbl.Table) } } else { m.SchemaManager = m.client diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 1776124f3..5c6dd805b 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -58,7 +58,7 @@ func configureTestFlagsMDS(main *Main, address dax.Address, qtbl *dax.QualifiedT mdsClient := mdsclient.New(dax.Address(address), logger.StderrLogger) main.NewImporterFn = func() pilosa.Importer { - return mds.NewImporter(mdsClient, qtbl.TableQualifier, &qtbl.Table) + return mds.NewImporter(mdsClient, mdsClient, qtbl.TableQualifier, &qtbl.Table) } } diff --git a/idk/mds/importer.go b/idk/mds/importer.go index 4f0cc0863..25833cef6 100644 --- a/idk/mds/importer.go +++ b/idk/mds/importer.go @@ -18,18 +18,20 @@ var _ featurebase.Importer = &importer{} // importer type importer struct { - mds MDS + noder dax.Noder + schemar dax.Schemar mu sync.Mutex qual dax.TableQualifier tbl *dax.Table } -func NewImporter(mds MDS, qual dax.TableQualifier, tbl *dax.Table) *importer { +func NewImporter(noder dax.Noder, schemar dax.Schemar, qual dax.TableQualifier, tbl *dax.Table) *importer { return &importer{ - mds: mds, - qual: qual, - tbl: tbl, + noder: noder, + schemar: schemar, + qual: qual, + tbl: tbl, } } @@ -75,7 +77,7 @@ func (m *importer) CreateTableKeys(ctx context.Context, tid dax.TableID, keys .. // all the partitions at once, then getting the distinct list of addresses // and looping over that instead. for partition, ks := range partitions { - address, err := m.mds.IngestPartition(context.Background(), qtbl.QualifiedID(), partition) + address, err := m.noder.IngestPartition(context.Background(), qtbl.QualifiedID(), partition) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", qtbl, partition) } @@ -112,7 +114,7 @@ func (m *importer) CreateFieldKeys(ctx context.Context, tid dax.TableID, fname d // different partitionN for field translation. partition := dax.PartitionNum(0) - address, err := m.mds.IngestPartition(context.Background(), qtbl.QualifiedID(), partition) + address, err := m.noder.IngestPartition(context.Background(), qtbl.QualifiedID(), partition) if err != nil { return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", qtbl, partition) } @@ -137,7 +139,7 @@ func (m *importer) ImportRoaringBitmap(ctx context.Context, tid dax.TableID, fld return errors.Wrapf(err, "getting qtbl") } - address, err := m.mds.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) + address, err := m.noder.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) if err != nil { return errors.Wrap(err, "calling ingest-shard") } @@ -162,7 +164,7 @@ func (m *importer) ImportRoaringShard(ctx context.Context, tid dax.TableID, shar return errors.Wrapf(err, "getting qtbl") } - address, err := m.mds.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) + address, err := m.noder.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) if err != nil { return errors.Wrap(err, "calling ingest-shard") } @@ -182,7 +184,7 @@ func (m *importer) EncodeImportValues(ctx context.Context, tid dax.TableID, fld return "", nil, errors.Wrapf(err, "getting qtbl") } - address, err := m.mds.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) + address, err := m.noder.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) if err != nil { return "", nil, errors.Wrap(err, "calling ingest-shard") } @@ -207,7 +209,7 @@ func (m *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F return "", nil, errors.Wrapf(err, "getting qtbl") } - address, err := m.mds.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) + address, err := m.noder.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) if err != nil { return "", nil, errors.Wrap(err, "calling ingest-shard") } @@ -232,7 +234,7 @@ func (m *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field return errors.Wrapf(err, "getting qtbl") } - address, err := m.mds.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) + address, err := m.noder.IngestShard(context.Background(), qtbl.QualifiedID(), dax.ShardNum(shard)) if err != nil { return errors.Wrap(err, "calling ingest-shard") } @@ -266,7 +268,7 @@ func (m *importer) getQtbl(ctx context.Context, tid dax.TableID) (*dax.Qualified qtid := dax.NewQualifiedTableID(m.qual, tid) - qtbl, err := m.mds.Table(ctx, qtid) + qtbl, err := m.schemar.TableByID(ctx, qtid) if err != nil { return nil, errors.Wrap(err, "getting table") } diff --git a/idk/mds/mds.go b/idk/mds/mds.go deleted file mode 100644 index e81f69914..000000000 --- a/idk/mds/mds.go +++ /dev/null @@ -1,21 +0,0 @@ -// Package mds contains the implementation of the SchemaManager interface. -package mds - -import ( - "context" - - "github.com/molecula/featurebase/v3/dax" -) - -// MDS represents the MDS methods which importer uses. -type MDS interface { - IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error) - IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shard dax.ShardNum) (dax.Address, error) - - // Table was added so the `importer` instance (in this package) of the - // batch.Importer interface could lookup up a table based on the name - // provided in a method, as opposed to setting the table up front. This is - // because in queryer, we don't know the table yet, because we haven't - // parsed the sql yet. - Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) -}