mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
* Formatting adjustments made during code review. While reviewing the BULK INSERT logic (in order to decide how best to approach "ingest via sql" in the cloud), I made a few formatting and comment changes. I'm just adding them here as a separate commit so they don't muddy up my actual work. * Parser modifications to support mulitple tuples in INSERT INTO This commit doesn't include all of the changes required in the planner. Fow now, the planner is simply modified to continue supporting a single tuple (the first tuple in the list). * Update the planner to handle multiple INSERT INTO tuples This is part 1. It's still using the existing logic which builds an ImportRequest for every record (and every field!). The next step will involve using a client.Batch to handle the records. * Introduce client.Importer interface (used by client.Batch) Instead of the Batch having a pointer to a client, this puts an interface there instead (which the client implements). It also allows us to inject a different importer (i.e. other than a featurebase.client) into the Batch. * Decouple batch from client This commit pulls batch-specific code out of the client package and into a new batch package. It introduces the batch.Importer interface, the methods of which replace all the calls that batch was previously making directly to client methods. Finally, it contains two implementations of the batch.Importer interface: one is a wrapper around client, and the other is a wrapper around featurebase.API. * Use docker (instead of MustRunCluster) for internal batch tests Because the `batch` package tests are internal, using test.MustRunCluster() resulted in an import loop (because it eventually imports `server`, and we can't have that). So this commit replaces the use of `test.MustRunCluster()` with docker. The setup is basically the same as that used in the idk docker tests. Here we also remove all client-side references to `UseIngestAPI`, which is an experimental (json) ingest api. It's still suppored on the server, but here we remove the external usage of it. * cherry-pick fix * Use batch.Import() for sql3 INSERT INTO statements * Thread logger into sql3 * fix batch test * Fix some shadowing complaint by linter * Address some test issues related to stringsets * Exclude batch integration tests from CI * Address PR feedback - Added description to batch.README - Consolidated grep commands in .gitlab-ci.yml - Removed some debugging comments - Replaces some inadvertantly removed license headers * Add batch package to gitlab CI * Updated CI for batch package Updated CI include path Update gitlab ci Update CI Update CI Trying new include path for ci Updated gitlab ci include path Made idk race job optional for sonarcloud upload add testdata directory remove testenv from dockercompose file use GIT_STRATEGY clone in batch CI add testdata volume to dockercompose Co-authored-by: Fletcher Haynes <fletcher.haynes@generalassemb.ly>
275 lines
7.3 KiB
Go
275 lines
7.3 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
|
|
featurebase "github.com/molecula/featurebase/v3"
|
|
"github.com/molecula/featurebase/v3/errors"
|
|
)
|
|
|
|
var _ featurebase.SchemaAPI = &schemaAPI{}
|
|
|
|
// schemaAPI is a featurebase client wrapper which implements the
|
|
// featurebase.SchemaAPI interface. This was introduced for use in batch tests
|
|
// when we decoupled Batch from the client package. In other words, this is only
|
|
// used for those tests, it may not be functionally complete, and should not be
|
|
// used otherwise without further testing and review of this code.
|
|
type schemaAPI struct {
|
|
*Client
|
|
}
|
|
|
|
func NewSchemaAPI(c *Client) *schemaAPI {
|
|
return &schemaAPI{
|
|
Client: c,
|
|
}
|
|
}
|
|
|
|
func (s *schemaAPI) CreateIndexAndFields(ctx context.Context, indexName string, options featurebase.IndexOptions, fields []featurebase.CreateFieldObj) error {
|
|
schema, err := s.Client.Schema()
|
|
if err != nil {
|
|
return errors.Wrap(err, "getting schema")
|
|
}
|
|
|
|
// Add the index.
|
|
idx := schema.Index(indexName,
|
|
OptIndexKeys(options.Keys),
|
|
OptIndexTrackExistence(true),
|
|
)
|
|
if err := s.Client.CreateIndex(idx); err != nil {
|
|
return errors.Wrap(err, "creating index")
|
|
}
|
|
|
|
// Now add fields.
|
|
for _, f := range fields {
|
|
fld, err := s.addFieldToIndex(idx, f.Name, f.Options...)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "adding field to index")
|
|
}
|
|
if err := s.Client.CreateField(fld); err != nil {
|
|
return errors.Wrapf(err, "creating field")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *schemaAPI) CreateField(ctx context.Context, indexName string, fieldName string, opts ...featurebase.FieldOption) (*featurebase.Field, error) {
|
|
schema, err := s.Client.Schema()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "getting schema")
|
|
}
|
|
|
|
if !schema.HasIndex(indexName) {
|
|
return nil, featurebase.ErrIndexNotFound
|
|
}
|
|
|
|
idx := schema.Index(indexName)
|
|
|
|
fld, err := s.addFieldToIndex(idx, fieldName, opts...)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "adding field to index")
|
|
}
|
|
|
|
if err := s.Client.CreateField(fld); err != nil {
|
|
return nil, errors.Wrapf(err, "creating field")
|
|
}
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
func (s *schemaAPI) addFieldToIndex(idx *Index, fieldName string, opts ...featurebase.FieldOption) (*Field, error) {
|
|
ffos := &featurebase.FieldOptions{}
|
|
for _, opt := range opts {
|
|
opt(ffos)
|
|
}
|
|
|
|
cfos := []FieldOption{}
|
|
|
|
switch ffos.Type {
|
|
case featurebase.FieldTypeBool:
|
|
cfos = append(cfos, OptFieldTypeBool())
|
|
case featurebase.FieldTypeInt:
|
|
cfos = append(cfos, OptFieldTypeInt(ffos.Min.ToInt64(0), ffos.Max.ToInt64(0)))
|
|
case featurebase.FieldTypeSet:
|
|
cfos = append(cfos,
|
|
OptFieldTypeSet(CacheType(ffos.CacheType), int(ffos.CacheSize)),
|
|
OptFieldKeys(ffos.Keys),
|
|
)
|
|
case featurebase.FieldTypeMutex:
|
|
cfos = append(cfos,
|
|
OptFieldTypeMutex(CacheType(ffos.CacheType), int(ffos.CacheSize)),
|
|
OptFieldKeys(ffos.Keys),
|
|
)
|
|
case featurebase.FieldTypeDecimal:
|
|
cfos = append(cfos, OptFieldTypeDecimal(ffos.Scale, ffos.Min, ffos.Max))
|
|
case featurebase.FieldTypeTime:
|
|
cfos = append(cfos,
|
|
OptFieldTypeTime(TimeQuantum(ffos.TimeQuantum), ffos.NoStandardView),
|
|
OptFieldKeys(ffos.Keys),
|
|
)
|
|
case featurebase.FieldTypeTimestamp:
|
|
cfos = append(cfos, OptFieldTypeTimestamp(featurebase.DefaultEpoch, ffos.TimeUnit))
|
|
default:
|
|
return nil, errors.Errorf("unsupported field type: %s", ffos.Type)
|
|
}
|
|
|
|
return idx.Field(fieldName, cfos...), nil
|
|
}
|
|
|
|
func (s *schemaAPI) DeleteField(ctx context.Context, indexName string, fieldName string) error {
|
|
schema, err := s.Client.Schema()
|
|
if err != nil {
|
|
return errors.Wrap(err, "getting schema")
|
|
}
|
|
|
|
if !schema.HasIndex(indexName) {
|
|
return featurebase.ErrIndexNotFound
|
|
}
|
|
|
|
idx := schema.Index(indexName)
|
|
|
|
return s.Client.DeleteField(&Field{
|
|
name: fieldName,
|
|
index: idx,
|
|
})
|
|
}
|
|
|
|
func (s *schemaAPI) DeleteIndex(ctx context.Context, indexName string) error {
|
|
return s.Client.DeleteIndexByName(indexName)
|
|
}
|
|
|
|
func (s *schemaAPI) IndexInfo(ctx context.Context, indexName string) (*featurebase.IndexInfo, error) {
|
|
schema, err := s.Client.Schema()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "getting schema")
|
|
}
|
|
|
|
if !schema.HasIndex(indexName) {
|
|
return nil, featurebase.ErrIndexNotFound
|
|
}
|
|
|
|
idx := schema.Index(indexName)
|
|
return FromClientIndex(idx), nil
|
|
}
|
|
|
|
func (s *schemaAPI) Schema(ctx context.Context, withViews bool) ([]*featurebase.IndexInfo, error) {
|
|
return nil, errors.New("", "schemaAPI.Schema is not implemented")
|
|
}
|
|
|
|
var _ featurebase.QueryAPI = &queryAPI{}
|
|
|
|
// queryAPI is a featurebase client wrapper which implements the
|
|
// featurebase.QueryAPI interface. This was introduced for use in batch tests
|
|
// when we decoupled Batch from the client package. In other words, this is only
|
|
// used for those tests, it may not be functionally complete, and should not be
|
|
// used otherwise without further testing and review of this code.
|
|
type queryAPI struct {
|
|
*Client
|
|
}
|
|
|
|
func NewQueryAPI(c *Client) *queryAPI {
|
|
return &queryAPI{
|
|
Client: c,
|
|
}
|
|
}
|
|
|
|
func (q *queryAPI) Query(ctx context.Context, req *featurebase.QueryRequest) (featurebase.QueryResponse, error) {
|
|
schema, err := q.Client.Schema()
|
|
if err != nil {
|
|
return featurebase.QueryResponse{}, errors.Wrap(err, "getting schema")
|
|
}
|
|
|
|
if !schema.HasIndex(req.Index) {
|
|
return featurebase.QueryResponse{}, featurebase.ErrIndexNotFound
|
|
}
|
|
|
|
idx := schema.Index(req.Index)
|
|
|
|
qry := NewPQLBaseQuery(req.Query, idx, nil)
|
|
res, err := q.Client.Query(qry)
|
|
if err != nil {
|
|
return featurebase.QueryResponse{}, errors.Wrap(err, "querying client")
|
|
}
|
|
|
|
var rerr error
|
|
if res.ErrorMessage != "" {
|
|
rerr = errors.New("", res.ErrorMessage)
|
|
}
|
|
|
|
fbResults := make([]interface{}, 0)
|
|
|
|
// Row, PairsField, GroupCounts
|
|
for _, result := range res.ResultList {
|
|
switch result.Type() {
|
|
case QueryResultTypeRow:
|
|
if len(result.Row().Keys) > 0 {
|
|
row := featurebase.NewRow()
|
|
row.Keys = result.Row().Keys
|
|
fbResults = append(fbResults, row)
|
|
} else {
|
|
fbResults = append(fbResults, featurebase.NewRow(result.Row().Columns...))
|
|
}
|
|
|
|
case QueryResultTypeUint64:
|
|
fbResults = append(fbResults, uint64(result.Count()))
|
|
|
|
case QueryResultTypeBool:
|
|
fbResults = append(fbResults, result.Changed())
|
|
|
|
case QueryResultTypePairsField:
|
|
pairs := []featurebase.Pair{}
|
|
for _, ci := range result.CountItems() {
|
|
pairs = append(pairs, featurebase.Pair{
|
|
ID: ci.ID,
|
|
Key: ci.Key,
|
|
Count: ci.Count,
|
|
})
|
|
}
|
|
pf := &featurebase.PairsField{
|
|
Pairs: pairs,
|
|
Field: "",
|
|
}
|
|
fbResults = append(fbResults, pf)
|
|
|
|
case QueryResultTypeGroupCounts:
|
|
groups := []featurebase.GroupCount{}
|
|
for _, grpCnt := range result.GroupCounts() {
|
|
fieldRows := []featurebase.FieldRow{}
|
|
for _, fr := range grpCnt.Groups {
|
|
fieldRows = append(fieldRows, featurebase.FieldRow{
|
|
Field: fr.FieldName,
|
|
RowID: fr.RowID,
|
|
RowKey: fr.RowKey,
|
|
Value: fr.Value,
|
|
})
|
|
}
|
|
groups = append(groups, featurebase.GroupCount{
|
|
Group: fieldRows,
|
|
Count: uint64(grpCnt.Count),
|
|
Agg: grpCnt.Agg,
|
|
// DecimalAgg: ??,
|
|
})
|
|
}
|
|
|
|
gc := featurebase.NewGroupCounts("", groups...)
|
|
fbResults = append(fbResults, gc)
|
|
|
|
case QueryResultTypeValCount:
|
|
vc := featurebase.ValCount{
|
|
Val: result.Value(),
|
|
Count: result.Count(),
|
|
}
|
|
fbResults = append(fbResults, vc)
|
|
|
|
default:
|
|
return featurebase.QueryResponse{}, errors.Errorf("unsupported query result type: %d", result.Type())
|
|
}
|
|
}
|
|
|
|
resp := &featurebase.QueryResponse{
|
|
Results: fbResults,
|
|
Err: rerr,
|
|
}
|
|
|
|
return *resp, nil
|
|
}
|