diff --git a/api.go b/api.go index 080c1a8e5..753e7aafa 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "time" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/rbf" //"github.com/molecula/featurebase/v3/pg" @@ -1122,164 +1121,6 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return nil } -// applyOneIngestSchema applies a single ingestSpec, which specifies operations on -// a single index and possibly fields. If it is successful, it returns the name -// of the index and an empty slice (if it created the index), or the name of the -// index and a slice of the fields within that index that it created. If it -// is unsuccessful, it tries to delete whatever it created. -// -// The intended idiom is that if the returned list of fields isn't empty, the index -// already existed and only those fields need to be cleaned up in the event of -// a later error, but if the list of fields is empty, the entire index was new, -// and should be cleaned up, in which case there's no need to track or delete -// the specific fields separately. -func (api *API) ApplyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { - if api.PrimaryNode().ID != api.NodeID() { - return nil, nil, RedirectError{ - HostPort: api.PrimaryNode().URI.Normalize(), - error: "request made to non-primary node", - } - } - - // create index - indexName := schema.IndexName - var createdFields []string - var useKeys bool - switch schema.PrimaryKeyType { - case "string": - useKeys = true - case "uint": - useKeys = false - default: - return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) - } - opts := IndexOptions{ - Keys: useKeys, - TrackExistence: true, - } - createdIndex := false - - // We check this up here because, if there's at least one field but we don't know what to do with - // it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in - // trying to create it. We don't care about this if there's no fields specified. - if len(schema.Fields) > 0 { - switch schema.FieldAction { - case "create", "ensure", "require": - // do nothing - case "": - schema.FieldAction = schema.IndexAction - default: - return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction) - } - } - - switch schema.IndexAction { - case "ensure", "require": - index, err = api.Index(ctx, indexName) - if err != nil { - if _, ok := err.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) - } else { - err = nil - } - } - if index != nil { - existingOpts := index.Options() - if existingOpts != opts { - return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts) - } - break - } - if schema.IndexAction == "require" { - return nil, nil, fmt.Errorf("index %q does not exist", indexName) - } - fallthrough - case "create": - index, err = api.CreateIndex(ctx, indexName, opts) - if err != nil { - return nil, nil, err - } - createdIndex = true - default: - return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction) - } - - // Now we might have an index, so we need our cleanup code. - defer func() { - if err == nil { - return - } - if createdIndex { - err := api.DeleteIndex(ctx, indexName) - if err != nil { - - api.server.logger.Printf("trying to undo failed index %q creation: %v", indexName, err) - } - return - } - for _, field := range createdFields { - err := api.DeleteField(ctx, indexName, field) - if err != nil { - api.server.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err) - } - } - }() - - // create all the fields specified in the index - for _, fSpec := range schema.Fields { - fieldName := fSpec.FieldName - opt := fieldSpecToFieldOption(fSpec) - err = opt.validate() - if err != nil { - return nil, nil, err - } - switch schema.FieldAction { - case "ensure", "require": - field, schemaErr := api.Field(ctx, indexName, fieldName) - if schemaErr != nil { - // NotFoundError is fine - if _, ok := schemaErr.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) - } - } - if field != nil { - existing := field.Options() - if opt.Type != existing.Type { - return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type) - } - if ((opt.Keys != nil) && *opt.Keys) != existing.Keys { - if existing.Keys { - return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName) - } else { - return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName) - } - } - // TODO: verify compatibility of other field opts, this is sorta hard - break - } - if schema.FieldAction == "require" { - return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName) - } - fallthrough - case "create": - fos := fieldOptionsToFunctionalOpts(opt) - _, err = api.CreateField(ctx, indexName, fieldName, fos...) - if err != nil { - return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err) - } - createdFields = append(createdFields, fieldName) - } - } - - // we don't report the fields back, so we can distinguish "created index" - // from "created fields within index" - if createdIndex { - createdFields = nil - } - - return index, createdFields, nil -} - // Views returns the views in the given field. func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Views") @@ -1936,251 +1777,6 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -// ingestNodeOperationsForFields does the actual work of applying operations -// to a given index with a map of known fields and an already-parsed -// ShardedRequest. This is used locally on the node that first receives -// the request, after it does the parsing, and on other nodes because the -// format they get is already that rather than JSON, so it's the common -// path *after* key translation and sorting into shards. -func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { - eg, ctx := errgroup.WithContext(ctx) - for shard, ops := range req.Ops { - // create new local copies of these values so the goroutine uses these - // copies, and doesn't read the actual loop variables, which are being - // changed by the loop. - shard, ops := shard, ops - eg.Go(func() error { - return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) - }) - } - return eg.Wait() -} - -// IngestNodeOperations handles protobuf-formatted data which does not need -// key translation and is applicable to this specific node. -func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { - index := api.holder.Index(indexName) - if index == nil { - api.server.logger.Errorf("ingest: no such index %q", indexName) - return newNotFoundError(ErrIndexNotFound, indexName) - } - fields := index.Fields() - knownFields := map[string]*Field{} - for _, field := range fields { - knownFields[field.name] = field - } - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) -} - -// IngestOperations handles JSON-formatted data which may need key translation -// and may be for any or all nodes. -func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { - span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") - defer span.Finish() - - if api.PrimaryNode().ID != api.NodeID() { - return RedirectError{ - HostPort: api.PrimaryNode().URI.Normalize(), - error: "request made to non-primary node", - } - } - - if err := api.validate(apiIngestOperations); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Find the Index. - index := api.holder.Index(indexName) - if index == nil { - api.server.logger.Errorf("ingest: no such index %q", indexName) - return newNotFoundError(ErrIndexNotFound, indexName) - } - fields := index.Fields() - var indexKeys ingest.KeyTranslator - if index.Keys() { - indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) - } - codec, err := ingest.NewJSONCodec(indexKeys) - if err != nil { - return errors.Wrap(err, "creating JSON codec") - } - knownFields := map[string]*Field{} - for _, field := range fields { - var keys ingest.KeyTranslator - if field.usesKeys { - keys = newIngestKeyTranslatorFromStore(field.translateStore) - } - knownFields[field.name] = field - switch field.Type() { - case "set": - if err = codec.AddSetField(field.name, keys); err != nil { - return fmt.Errorf("adding set field to codec: %w", err) - } - case "time": - if err = codec.AddTimeQuantumField(field.name, keys); err != nil { - return fmt.Errorf("adding time quantum field to codec: %w", err) - } - case "mutex": - if err = codec.AddMutexField(field.name, keys); err != nil { - return fmt.Errorf("adding mutex field to codec: %w", err) - } - case "bool": - if err = codec.AddBoolField(field.name); err != nil { - return fmt.Errorf("adding bool field to codec: %w", err) - } - case "int": - if err = codec.AddIntField(field.name, keys); err != nil { - return fmt.Errorf("adding int field to codec: %w", err) - } - case "decimal": - if err = codec.AddDecimalField(field.name, field.options.Scale); err != nil { - return fmt.Errorf("adding decimal field to codec: %w", err) - } - case "timestamp": - if err = codec.AddTimestampField(field.name, field.options.TimeUnit, field.options.Base); err != nil { - return fmt.Errorf("adding timestamp field to codec: %w", err) - } - default: - return fmt.Errorf("unhandled field type %q", field.Type()) - } - } - req, err := codec.Parse(stream) - if err != nil { - return errors.Wrap(err, "parsing input data") - } - sharded, err := codec.RequestByShard(req) - if err != nil { - return errors.Wrap(err, "sharding input data") - } - // now that we have this, let's assign the shards to nodes - snap := api.cluster.NewSnapshot() - // oh hey an easy case: we're presumably the only node - if len(snap.Nodes) == 1 { - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) - } - // Created new ShardedRequest objects for every node, giving each of them - // all the shards that apply to them. - byNode := make(map[string]*ingest.ShardedRequest) - for shard, ops := range sharded.Ops { - nodes := snap.ShardNodes(indexName, shard) - for _, node := range nodes { - forThisShard := byNode[node.ID] - if forThisShard == nil { - // Create new ShardedRequest for the target node, with its op map - // mapping this shard to the ops for this shard. - byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} - continue - } - // Add this shard to the existing ShardedRequest's Ops map. Note that - // we don't have to worry about overwrites; we can't have seen this - // shard before, because we're in a range loop on a map where the shard - // is the key. - forThisShard.Ops[shard] = ops - } - } - eg, ctx := errgroup.WithContext(ctx) - for _, node := range snap.Nodes { - node := node - sharded := byNode[node.ID] - // Sometimes, there's nothing for a specific node. - if sharded == nil { - continue - } - if node.ID == api.NodeID() { - eg.Go(func() error { - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) - }) - } else { - eg.Go(func() error { - return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) - }) - } - } - return eg.Wait() -} - -// applyOperations applies a set of operations to one specific shard. -func (api *API) applyOperations(ctx context.Context, qcx *Qcx, index *Index, shard uint64, fields map[string]*Field, ops []*ingest.Operation) error { - // For each operation, we may have a set of records/fields to clear, and then - // also a set of fields to set/remove specific bits in. - opts := &ImportOptions{Presorted: true, IgnoreKeyCheck: true, fullySorted: true} - for _, op := range ops { - // ClearRecordIDs should exist only for delete, clear, and write. For clear and write, - // we'll have a list of fields, for delete, it should be all the fields. - if len(op.ClearRecordIDs) > 0 { - // anonymous func lets us defer a finisher from any of the inner error returns - err := func() (e0 error) { - // WARNING: Depends on GetTx being per-shard/index, not per-field. - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) - if err != nil { - return fmt.Errorf("getting Tx: %w", err) - } - defer finisher(&e0) - // For a delete, we don't look at the fields the codec was defined with, - // We delete from the existence field unconditionally and other fields - // if we know they exist. - if op.OpType == ingest.OpDelete { - err = clearExistenceColumns(tx, index, op.ClearRecordIDs, shard) - if err != nil { - return fmt.Errorf("clearing existence columns: %w", err) - } - for name, field := range fields { - if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { - return fmt.Errorf("clearing field %q: %w", name, err) - } - } - return nil - } - // clear things that we need to wipe out, whether it's because - // this is a Clear op, or because it's a write op that - // specifies clears for the fields it's going to write to. - if len(op.ClearFields) > 0 { - for _, fieldName := range op.ClearFields { - field, ok := fields[fieldName] - if !ok { - return fmt.Errorf("can't find a field named %q", fieldName) - } - if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { - return fmt.Errorf("clearing record IDs: %w", err) - } - } - } - return nil - }() - if err != nil { - return err - } - } - opts.Clear = (op.OpType == ingest.OpRemove) - // for "set" and "write" ops, we'll be setting bits, for - // "remove" ops we'll be clearing them, and for "clear" ops - // there shouldn't be anything here. - for fieldName, fieldOp := range op.FieldOps { - field, ok := fields[fieldName] - if !ok { - return fmt.Errorf("can't find a field named %q", fieldName) - } - var err error - err = importExistenceColumns(qcx, index, fieldOp.RecordIDs, shard) - if err != nil { - return errors.Wrap(err, "importing existence columns") - } - switch field.Type() { - case "set", "time", "mutex", "bool": - err = field.Import(qcx, fieldOp.Values, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) - case "int", "timestamp", "decimal": - err = field.importValue(qcx, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) - default: - err = fmt.Errorf("unhandled field type %q", field.Type()) - } - if err != nil { - return err - } - } - } - return nil -} - func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { ef := index.existenceField() if ef == nil { @@ -3287,8 +2883,6 @@ const ( apiIDCommit apiIDReset apiPartitionNodes - apiIngestOperations - apiIngestNodeOperations apiMutexCheck ) @@ -3349,8 +2943,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIDCommit: {}, apiIDReset: {}, apiPartitionNodes: {}, - apiIngestOperations: {}, - apiIngestNodeOperations: {}, apiMutexCheck: {}, } diff --git a/api_test.go b/api_test.go index 42e9b171a..58c344406 100644 --- a/api_test.go +++ b/api_test.go @@ -521,72 +521,6 @@ func TestAPI_Ingest(t *testing.T) { t.Fatalf("creating field: %v", err) } - t.Run("IngestAPI", func(t *testing.T) { - sampleJson := []byte(` - [ - { - "action": "set", - "records": { - "2": { - "set": [2], - "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] } - }, - "5": { "set": [3] }, - "8": { "set": [3] }, - "1": { - "set": [2], - "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } - }, - "4": { "set": [3, 7] } - } - }, - { - "action": "clear", - "record_ids": [ 5, 6, 7 ], - "fields": [ "tq", "set" ] - }, - { - "action": "write", - "records": { - "8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } }, - "9": { "set": [7, 3] } - } - }, - { - "action": "delete", - "record_ids": [ 9 ] - } - ] - `) - // just for set row 3: - // first operation should set it for 4, 5, and 8. - // clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8. - // the write operation should clear set for record 8, even though record 8 doesn't - // contain that field in that op, because set is present in record 9, which also - // gets row 3 set. but then we delete 9. - // so after all that we expect Row(set=3) to be 4... - sampleBuf := bytes.NewBuffer(sampleJson) - qcx := coord.API.Txf().NewQcx() - defer func() { - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf) - if err != nil { - t.Fatalf("importing data: %v", err) - } - query := "Row(set=3)" - res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - if err != nil { - t.Errorf("query: %v", err) - } - r := res.Results[0].(*pilosa.Row).Columns() - if len(r) != 1 || r[0] != 4 { - t.Fatalf("expected row with 4 set, got %d", r) - } - }) - t.Run("ImportRoaringShard", func(t *testing.T) { setBuf := &bytes.Buffer{} setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7) @@ -697,55 +631,6 @@ func ingestBenchmarkHelper() []byte { return data } -func BenchmarkIngest(b *testing.B) { - b.StopTimer() - data := ingestBenchmarkHelper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := test.MustRunCluster(b, 1) - defer c.Close() - - coord := c.GetPrimary() - m0 := c.GetNode(0) - // m1 := c.GetNode(1) - // m2 := c.GetNode(2) - - index := c.Idx() - setField := "set" - intField := "int" - tqField := "tq" - _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false}) - if err != nil { - b.Fatalf("creating index: %v", err) - } - _, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0)) - if err != nil { - b.Fatalf("creating field: %v", err) - } - _, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840)) - if err != nil { - b.Fatalf("creating field: %v", err) - } - _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0")) - if err != nil { - b.Fatalf("creating field: %v", err) - } - b.ReportAllocs() - b.StartTimer() - for i := 0; i < b.N; i++ { - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() - err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data)) - if err != nil { - b.Fatalf("ingest: %v", err) - } - err = qcx.Finish() - if err != nil { - b.Fatalf("finish: %v", err) - } - } -} - func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() diff --git a/apimethod_string.go b/apimethod_string.go index eebc4cab6..da9fde7d5 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -42,14 +42,12 @@ func _() { _ = x[apiIDCommit-31] _ = x[apiIDReset-32] _ = x[apiPartitionNodes-33] - _ = x[apiIngestOperations-34] - _ = x[apiIngestNodeOperations-35] - _ = x[apiMutexCheck-36] + _ = x[apiMutexCheck-34] } -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 499, 522, 535} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/client/client.go b/client/client.go index e41add3bb..f825c6310 100644 --- a/client/client.go +++ b/client/client.go @@ -790,34 +790,6 @@ func (c *Client) readSchema() ([]SchemaIndex, error) { return schemaInfo.Indexes, nil } -func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err error) { - data, err := json.Marshal(reqBody) - if err != nil { - return data, errors.Wrap(err, "error building Schema body to Ingest") - } - return c.IngestRequest("/internal/schema", data) -} - -func (c *Client) IngestData(index string, reqBody []map[string]interface{}) (body []byte, err error) { - data, err := json.Marshal(reqBody) - if err != nil { - return data, errors.Wrap(err, "error building request body to Ingest") - } - return c.IngestRequest("/internal/ingest/"+index, data) -} - -func (c *Client) IngestRequest(uri string, data []byte) (body []byte, err error) { - var header = make(map[string]string) - header["Content-Type"] = "application/json" - header["Accept"] = "application/json" - header["User-Agent"] = "pilosa/" + pilosa.Version - status, body, err := c.HTTPRequest("POST", uri, data, header) - if err != nil { - return nil, errors.Wrapf(err, "requesting %s status: %d", uri, status) - } - return body, err -} - func (c *Client) shardsMax() (map[string]uint64, error) { _, data, err := c.HTTPRequest("GET", "/internal/shards/max", nil, nil) if err != nil { diff --git a/cluster.go b/cluster.go index 549cb9b03..a173efdae 100644 --- a/cluster.go +++ b/cluster.go @@ -8,7 +8,6 @@ import ( "time" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" @@ -504,38 +503,6 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } -// This implements ingest's key translator interface on a cluster/index pair. -type clusterKeyTranslator struct { - ctx context.Context // we're created within a request context and need to pass that to cluster ops - c *cluster - indexName string -} - -var _ ingest.KeyTranslator = &clusterKeyTranslator{} - -func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - return i.c.createIndexKeys(i.ctx, i.indexName, keys...) -} - -func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids) - if err != nil { - return nil, err - } - if len(keys) != len(ids) { - return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) - } - out := make(map[uint64]string, len(keys)) - for i, id := range ids { - out[id] = keys[i] - } - return out, nil -} - -func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator { - return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName} -} - // TODO: remove this when it is no longer used func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keys := make([]string, 0, len(keySet)) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index da5d64570..b83692713 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -9,7 +9,6 @@ import ( "github.com/gogo/protobuf/proto" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" @@ -294,19 +293,6 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } *mt = s.decodeRowMatrix(msg) return nil - - case *ingest.ShardedRequest: - msg := &pb.ShardedIngestRequest{} - err := proto.Unmarshal(buf, msg) - if err != nil { - return errors.Wrap(err, "unmarshalling ShardedRequest") - } - req, err := s.decodeShardedIngestRequest(msg) - if err != nil { - return err - } - *mt = *req - return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -376,8 +362,6 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeTransactionMessage(mt) case *pilosa.AtomicRecord: return s.encodeAtomicRecord(mt) - case *ingest.ShardedRequest: - return s.encodeShardedIngestRequest(mt) } return nil } @@ -900,48 +884,6 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr return &pb.TransactionStats{} } -func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest { - if req == nil || len(req.Ops) == 0 { - return &pb.ShardedIngestRequest{} - } - out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))} - for shard, ops := range req.Ops { - out.Ops[shard] = s.encodeShardIngestOperations(ops) - } - return out -} - -func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { - out := &pb.ShardIngestOperations{} - for _, op := range ops { - if op == nil { - continue - } - out.Ops = append(out.Ops, s.encodeShardIngestOperation(op)) - } - return out -} - -func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation { - out := &pb.ShardIngestOperation{ - OpType: op.OpType.String(), - ClearRecordIDs: op.ClearRecordIDs, - ClearFields: op.ClearFields, - FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), - } - for k, v := range op.FieldOps { - if v == nil { - continue - } - out.FieldOps[k] = &pb.FieldOperation{ - RecordIDs: v.RecordIDs, - Values: v.Values, - Signed: v.Signed, - } - } - return out -} - func (s Serializer) decodeSchema(sc *pb.Schema, m *pilosa.Schema) { m.Indexes = make([]*pilosa.IndexInfo, len(sc.Indexes)) s.decodeIndexes(sc.Indexes, m.Indexes) @@ -1844,57 +1786,3 @@ func (s Serializer) encodeDecimal(p *pql.Decimal) *pb.Decimal { } return retval } - -func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) { - if req == nil || len(req.Ops) == 0 { - return &ingest.ShardedRequest{}, nil - } - out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))} - for shard, ops := range req.Ops { - var err error - out.Ops[shard], err = s.decodeShardIngestOperations(ops) - if err != nil { - return nil, err - } - } - return out, nil -} - -func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) { - out := []*ingest.Operation{} - if len(ops.Ops) == 0 { - return out, nil - } - for _, op := range ops.Ops { - if op == nil { - continue - } - decoded, err := s.decodeShardIngestOperation(op) - if err != nil { - return nil, err - } - out = append(out, decoded) - } - return out, nil -} - -func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) { - opType, err := ingest.ParseOpType(op.OpType) - if err != nil { - return nil, err - } - out := &ingest.Operation{ - OpType: opType, - ClearRecordIDs: op.ClearRecordIDs, - ClearFields: op.ClearFields, - FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)), - } - for k, v := range op.FieldOps { - out.FieldOps[k] = &ingest.FieldOperation{ - RecordIDs: v.RecordIDs, - Values: v.Values, - Signed: v.Signed, - } - } - return out, nil -} diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go index 6c2107fc5..a7500fa4d 100644 --- a/encoding/proto/proto_test.go +++ b/encoding/proto/proto_test.go @@ -2,12 +2,10 @@ package proto import ( - "errors" "reflect" "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/pb" ) @@ -40,99 +38,8 @@ func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, exp t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error()) } } - switch real := obj.(type) { - case *ingest.ShardedRequest: - real2 := obj2.(*ingest.ShardedRequest) - err := real.Compare(real2) - if err != nil { - if expectedMismatchErr == nil { - t.Fatalf("unexpected compare error %q", err.Error()) - } - if err.Error() != expectedMismatchErr.Error() { - t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error()) - } - } else { - if expectedMismatchErr != nil { - t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error()) - } - } - default: - if !reflect.DeepEqual(obj, obj2) { - t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) - } - } -} - -type shardedIngestRequestTest struct { - req *ingest.ShardedRequest - err error -} - -var shardedIngestRequestTestcases = []shardedIngestRequestTest{ - { - req: &ingest.ShardedRequest{ - Ops: map[uint64][]*ingest.Operation{ - 1: { - { - OpType: ingest.OpWrite, - ClearFields: []string{"clearField", "clearField2"}, - ClearRecordIDs: []uint64{1, 7, 9}, - FieldOps: map[string]*ingest.FieldOperation{ - "writeAll": { - RecordIDs: []uint64{3, 6, 8}, - Values: []uint64{0, 17, 34}, - Signed: []int64{-9, 23, 17}, - }, - "writeValues": { - RecordIDs: []uint64{3, 6, 8}, - Values: []uint64{0, 17, 34}, - }, - "writeSigned": { - RecordIDs: []uint64{3, 6, 8}, - Signed: []int64{-9, 23, 17}, - }, - }, - }, - { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ - "foo": nil, - }, - }, - }, - 2: {}, - 3: nil, - }, - }, - }, - { - req: &ingest.ShardedRequest{ - Ops: map[uint64][]*ingest.Operation{ - 1: { - { - OpType: ingest.OpWrite, - FieldOps: map[string]*ingest.FieldOperation{ - "writeAll": {}, - }, - }, - { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ - "foo": nil, - }, - }, - nil, - }, - }, - }, - err: errors.New("shard 1: expected 3 ops, got 2"), - }, -} - -func TestIngestRoundTrip(t *testing.T) { - for _, tc := range shardedIngestRequestTestcases { - t.Logf("next case") - testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err) + if !reflect.DeepEqual(obj, obj2) { + t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) } } diff --git a/go.mod b/go.mod index 0202952cd..c35662184 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/aws/aws-sdk-go v1.42.39 github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 - github.com/buger/jsonparser v1.1.1 github.com/cespare/xxhash v1.1.0 github.com/chzyer/readline v1.5.0 github.com/confluentinc/confluent-kafka-go v1.9.1 diff --git a/go.sum b/go.sum index b2d90ac34..4dd0ae0b4 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= diff --git a/handler.go b/handler.go index caac2f815..a07f275ed 100644 --- a/handler.go +++ b/handler.go @@ -3,9 +3,10 @@ package pilosa import ( "encoding/json" + "math/bits" "time" - "github.com/molecula/featurebase/v3/ingest" + "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" ) @@ -156,7 +157,6 @@ func (ivr *ImportValueRequest) Clone() *ImportValueRequest { // The top level Shard has to agree with Ivr[i].Shard and the Iv[i].Shard // for all i included (in Ivr and Ir). The same goes for the top level Index: all records // have to be writes to the same Index. These requirements are checked. -// type AtomicRecord struct { Index string Shard uint64 @@ -299,28 +299,108 @@ func (ir *ImportRequest) Clone() *ImportRequest { // requests. We don't sort the entries within each shard because the correct // sorting depends on the field type and we don't want to deal with that // here. -func (ir *ImportRequest) SortToShards() map[uint64]*ImportRequest { - // cheat: use ingest - fo := ingest.FieldOperation{ - RecordIDs: ir.ColumnIDs, - Values: ir.RowIDs, - Signed: ir.Timestamps, +func (ir *ImportRequest) SortToShards() (result map[uint64]*ImportRequest) { + if len(ir.ColumnIDs) == 0 { + return nil } - sharded := fo.SortToShards() - output := make(map[uint64]*ImportRequest, len(sharded)) - for shard, shardOp := range sharded { - shardReq := *ir - shardReq.ColumnKeys = nil - shardReq.RowKeys = nil - shardReq.Shard = shard - shardReq.ColumnIDs = shardOp.RecordIDs - shardReq.RowIDs = shardOp.Values - shardReq.Timestamps = shardOp.Signed - output[shard] = &shardReq + diffMask := uint64(0) + prev := ir.ColumnIDs[0] + for _, r := range ir.ColumnIDs[1:] { + diffMask |= r ^ prev + prev = r } + bitsRemaining := bits.Len64(diffMask) + if bitsRemaining <= shardwidth.Exponent { + shard := ir.ColumnIDs[0] >> shardwidth.Exponent + ir.Shard = shard + ir.ColumnKeys = nil + ir.RowKeys = nil + return map[uint64]*ImportRequest{shard: ir} + } + output := make(map[uint64]*ImportRequest) + sortToShardsInto(ir, bitsRemaining-8, output) return output } +// sortToShardsInto puts the shards it finds into the given map, so that +// as we split off buckets, they can be inserted into the same map. +func sortToShardsInto(ir *ImportRequest, shift int, into map[uint64]*ImportRequest) { + if shift < shardwidth.Exponent { + shift = shardwidth.Exponent + } + nextShift := shift - 8 + if nextShift < shardwidth.Exponent { + nextShift = shardwidth.Exponent + } + // count things that belong in each of the 256 buckets + var buckets [256]int + var starts [256]int + + // compute the buckets ourselves + for _, r := range ir.ColumnIDs { + b := (r >> shift) & 0xFF + buckets[b]++ + } + total := 0 + // compute starting points of each bucket, converting the + // bucket counts into ends + for i := range buckets { + starts[i] = total + total += buckets[i] + buckets[i] = total + } + // starts[n] is the index of the first thing that should + // go in that bucket, buckets[n] is the index of the first + // thing that shouldn't + var bucketOp ImportRequest = *ir + bucketOp.ColumnKeys = nil + bucketOp.RowKeys = nil + origStarts := make([]int, len(starts)) + copy(origStarts, starts[:]) + for bucket, start := range origStarts { + end := buckets[bucket] + if end <= start { + continue + } + for j := start; j < end; j++ { + want := int((ir.ColumnIDs[j] >> shift) & 0xFF) + for want != bucket { + // move this to the beginning of the + // bucket it wants to be in, swapping + // the thing there here + dst := starts[want] + ir.ColumnIDs[j], ir.ColumnIDs[dst] = ir.ColumnIDs[dst], ir.ColumnIDs[j] + if ir.RowIDs != nil { + ir.RowIDs[j], ir.RowIDs[dst] = ir.RowIDs[dst], ir.RowIDs[j] + } + if ir.Timestamps != nil { + ir.Timestamps[j], ir.Timestamps[dst] = ir.Timestamps[dst], ir.Timestamps[j] + } + starts[want]++ + want = int((ir.ColumnIDs[j] >> shift) & 0xFF) + } + } + // If shift == shardwidth.Exponent, then this is a completed + // shard and can go into the sharded output. otherwise, we + // can subdivide it. + bucketOp.ColumnIDs = ir.ColumnIDs[start:end] + if ir.RowIDs != nil { + bucketOp.RowIDs = ir.RowIDs[start:end] + } + if ir.Timestamps != nil { + bucketOp.Timestamps = ir.Timestamps[start:end] + } + if shift == shardwidth.Exponent { + x := bucketOp + shard := ir.ColumnIDs[start] >> shardwidth.Exponent + x.Shard = shard + into[shard] = &x + } else { + sortToShardsInto(&bucketOp, nextShift, into) + } + } +} + // ValidateWithTimestamp ensures that the payload of the request is valid. func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if (ir.IndexCreatedAt != 0 && ir.IndexCreatedAt != indexCreatedAt) || diff --git a/handler_test.go b/handler_test.go new file mode 100644 index 000000000..7e102d303 --- /dev/null +++ b/handler_test.go @@ -0,0 +1,28 @@ +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa_test + +import ( + "math/rand" + "testing" + + pilosa "github.com/molecula/featurebase/v3" +) + +func TestSortToShards(t *testing.T) { + var ir pilosa.ImportRequest + expected := make(map[uint64][]uint64) + const n = 50 + rng := rand.New(rand.NewSource(3)) + for i := 0; i < n; i++ { + x := uint64(rng.Intn(8 * pilosa.ShardWidth)) + shard := x / pilosa.ShardWidth + expected[shard] = append(expected[shard], x) + ir.ColumnIDs = append(ir.ColumnIDs, x) + } + out := ir.SortToShards() + for shard, values := range out { + if len(values.ColumnIDs) != len(expected[shard]) { + t.Fatalf("shard %d: expected values %d, got values %d", shard, expected[shard], values.ColumnIDs) + } + } +} diff --git a/http_handler.go b/http_handler.go index f26a3074b..5579bd0ee 100644 --- a/http_handler.go +++ b/http_handler.go @@ -33,7 +33,6 @@ import ( "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/authz" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/monitor" "github.com/molecula/featurebase/v3/pql" @@ -584,10 +583,7 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/index/{index}/shards", handler.chkAuthZ(handler.handleGetIndexAvailableShards, authz.Read)).Methods("GET").Name("GetIndexAvailableShards") router.HandleFunc("/internal/nodes", handler.chkAuthN(handler.handleGetNodes)).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.chkAuthN(handler.handleGetShardsMax)).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/ingest/{index}", handler.chkAuthZ(handler.handlePostIngestData, authz.Write)).Methods("POST").Name("PostIngestData") - router.HandleFunc("/internal/ingest/{index}/node", handler.chkAuthZ(handler.handlePostIngestNode, authz.Write)).Methods("POST").Name("PostIngestNode") - router.HandleFunc("/internal/schema", handler.chkAuthZ(handler.handleIngestSchema, authz.Admin)).Methods("POST").Name("PostIngestSchema") router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.chkAuthZ(handler.handleFindIndexKeys, authz.Admin)).Methods("POST").Name("FindIndexKeys") router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.chkAuthZ(handler.handleCreateIndexKeys, authz.Admin)).Methods("POST").Name("CreateIndexKeys") router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.chkAuthZ(handler.handlePostTranslateIndexDB, authz.Admin)).Methods("POST").Name("PostTranslateIndexDB") @@ -2087,40 +2083,6 @@ func (h *Handler) handlePatchField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -// handlePostIngestData handles JSON ingest data that may need key -// translation, for the entire cluster. -func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { - if !validHeaderAcceptJSON(r.Header) { - http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) - return - } - - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - - qcx := h.api.Txf().NewQcx() - err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) - if err != nil { - qcx.Abort() - switch e := err.(type) { - case RedirectError: - http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) - return - } - } - err = qcx.Finish() - if err != nil { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - return - } - - resp := successResponse{h: h, Name: indexName} - resp.write(w, err) -} - type ingestSpec struct { IndexName string `json:"index-name"` IndexAction string `json:"index-action"` @@ -2182,79 +2144,6 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { return opt } -func (h *Handler) handleIngestSchema(w http.ResponseWriter, r *http.Request) { - if !validHeaderAcceptJSON(r.Header) { - http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) - return - } - - resp := successResponse{h: h} - - dec := json.NewDecoder(r.Body) - dec.DisallowUnknownFields() - schema := ingestSpec{} - // if a key in cleanupIndexes points to a 0-length slice, the - // entire index should be cleaned; otherwise, only the named - // fields within that index should be cleaned. - cleanupIndexes := map[string][]string{} - var schemaErr error - defer func() { - // we set schemaErr in any case where we need to do cleanup - if schemaErr != nil { - for index, fields := range cleanupIndexes { - if len(fields) == 0 { - err := h.api.DeleteIndex(r.Context(), index) - if err != nil { - h.logger.Printf("deleting index %q after schema err: %v", index, err) - } - } else { - for _, field := range fields { - err := h.api.DeleteField(r.Context(), index, field) - if err != nil { - h.logger.Printf("deleting field %q from index %q after schema err: %v", field, index, err) - } - } - } - } - } - }() - for dec.More() { - err := dec.Decode(&schema) - if err != nil { - resp.write(w, err) - return - } - index, fields, err := h.api.ApplyOneIngestSchema(r.Context(), &schema) - if err != nil { - switch e := err.(type) { - case RedirectError: - http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) - return - default: - // if a previous schema created things, clean them up... - schemaErr = err - resp.write(w, err) - return - } - } - // we only have one slot to report these, sorry. - resp.Name = index.Name() - resp.CreatedAt = index.CreatedAt() - cleanupIndexes[index.Name()] = fields - } - // if we got here, we have a cleanupIndexes which we want to return, - // so we want to do that *instead* of the successResponse we'd be - // using otherwise (ironically, to indicate an error) - var mapBody []byte - var err error - if mapBody, err = json.Marshal(cleanupIndexes); err != nil { - resp.write(w, err) - } - if _, err = w.Write(mapBody); err != nil { - h.logger.Printf("error trying to write response: %v", err) - } -} - type postFieldRequest struct { Options fieldOptions `json:"options"` } @@ -3600,52 +3489,6 @@ func (h *Handler) handlePostShardImportRoaring(w http.ResponseWriter, r *http.Re } } -// handlePostIngestNode is the internal endpoint taking already-translated -// ingest operations, sorted by shard, for a single node. -func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if error, code := validateProtobufHeader(r); error != "" { - http.Error(w, error, code) - return - } - - ctx := r.Context() - - // Read entire body. - span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body") - body, err := readBody(r) - span.LogKV("bodySize", len(body)) - span.Finish() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - req := &ingest.ShardedRequest{} - span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = h.serializer.Unmarshal(body, req) - span.Finish() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - urlVars := mux.Vars(r) - indexName := urlVars["index"] - - qcx := h.api.Txf().NewQcx() - err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req) - if err == nil { - err = qcx.Finish() - if err != nil { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - } - } else { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - qcx.Abort() - } -} - func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { diff --git a/http_handler_test.go b/http_handler_test.go index 928a80363..81bdc34e6 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -345,110 +345,15 @@ func TestUpdateFieldNoStandardView(t *testing.T) { } } -func TestIngestSchemaHandler(t *testing.T) { - c := test.MustRunCluster(t, 3) - defer c.Close() - - schema := fmt.Sprintf(` -{ - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "idset", - "field-type": "id", - "field-options": { - "cache-type": "none" - } - }, - { - "field-name": "id", - "field-type": "id", - "field-options": { - "enforce-mutual-exclusion": true - } - }, - { - "field-name": "bool", - "field-type": "bool" - }, - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - }, - { - "field-name": "string", - "field-type": "string", - "field-options": { - "enforce-mutual-exclusion": true - } - }, - { - "field-name": "int", - "field-type": "int" - }, - { - "field-name": "decimal", - "field-type": "decimal", - "field-options": { - "scale": 2 - } - }, - { - "field-name": "timestamp", - "field-type": "timestamp", - "field-options": { - "epoch": "1996-12-19T16:39:57-08:00", - "unit": "µs" - } - }, - { - "field-name": "quantum", - "field-type": "string", - "field-options": { - "time-quantum": "YMDH" - } - } - ] -} -`, c) - m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - // now, try again, expecting a failure: - resp = test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusConflict { - t.Errorf("invalid status: expected 409, got %d, body=%s", resp.StatusCode, resp.Body) - } -} - func TestPostFieldWithTTL(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - indexName := c.Idx("%s") - - schema := fmt.Sprintf(` - { - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields":[] - } - `, c) - m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + indexName := c.Idx("s") + _, err := c.GetNode(0).API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) } + m := c.GetPrimary() tests := []struct { name string @@ -543,27 +448,15 @@ func TestGetViewAndDelete(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - schema := fmt.Sprintf(` - { - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "test_view", - "field-type": "time", - "field-options": { - "time-quantum": "YMDH" - } - } - ] - } - `, c) m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + + _, err := m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m.API.CreateField(context.Background(), c.Idx("s"), "test_view", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) + if err != nil { + t.Fatalf("creating field: %v", err) } // Send sample data @@ -658,30 +551,16 @@ func TestTranslationHandlers(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - - schema := fmt.Sprintf(` -{ - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - } - ] -} -`, c) m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + _, err = m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) } + _, err = m.API.CreateField(context.Background(), c.Idx("s"), "stringset", pilosa.OptFieldTypeSet("ranked", 100000), pilosa.OptFieldKeys()) + if err != nil { + t.Fatalf("creating field: %v", err) + } + baseURLs := []string{ fmt.Sprintf("%s/internal/translate/index/%s/", m.URL(), c), fmt.Sprintf("%s/internal/translate/field/%s/stringset/", m.URL(), c), @@ -802,10 +681,18 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` defer c.Close() m := c.GetPrimary() - index := "allowed-networks-index" - keyedIndex := "allowed-networks-index-keyed" + index := c.Idx("s") + keyedIndex := c.Idx("k") field := "field1" + // This keyed index used to be created by the Post-Schema subtest, but that doesn't + // exist anymore, so we create it up here. We don't create the other one because it's + // supposed to get created by Post-Index. + _, err = m.API.CreateIndex(context.Background(), c.Idx("k"), pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + // needed for key translation nameBytes, err := json.Marshal([]string{"a", "b", "c"}) if err != nil { @@ -813,24 +700,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` } names := string(nameBytes) - schema := ` - { - "index-name": "allowed-networks-index-keyed", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - } - ] - } - ` - IPTests := []struct { TestName string ClientIP string @@ -864,12 +733,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` url: fmt.Sprintf("%s/schema", m.URL()), body: "", }, - { - testName: "Post-Schema", - method: "POST", - url: fmt.Sprintf("%s/internal/schema", m.URL()), - body: schema, - }, { testName: "Get-Shards", method: "GET", diff --git a/ingest/codec.go b/ingest/codec.go deleted file mode 100644 index 45d17d0f9..000000000 --- a/ingest/codec.go +++ /dev/null @@ -1,1169 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "math" - "sort" - "strconv" - "time" - - "github.com/buger/jsonparser" - "github.com/pkg/errors" -) - -// Featurebase has the following field types as of this writing, we plan to -// support all of them but not all are implemented. -// -// Type Single Signed Timestamp -// set no no no -// time no no yes -// mutex yes no no -// int yes yes no -// decimal yes yes no -// timestamp yes yes no - -// KeyTranslator is a thing that can translate strings to IDs, and also -// IDs back to strings. The ID->string conversion is used only to render -// a request back to JSON, which in turn is only used in testing. The -// functions optionally take an existing map which they then augment. -type KeyTranslator interface { - TranslateKeys(keys ...string) (map[string]uint64, error) - TranslateIDs(ids ...uint64) (map[uint64]string, error) -} - -// Codec is a single-use parser which decodes data into columnar vectors. -type Codec interface { - AddSetField(name string, keys KeyTranslator) error - AddTimeQuantumField(name string, keys KeyTranslator) error - AddMutexField(name string, keys KeyTranslator) error - AddBoolField(name string) error - AddIntField(name string, keys KeyTranslator) error - AddDecimalField(name string, scale int64) error - AddTimestampField(name string, scale string, epoch int64) error - - // Parse data from a reader into the vectors. - // This must only be called once on a codec. - Parse(io.Reader) (*Request, error) -} - -type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error - -// jsonEncFn encodes a value, or range of values, according to a given -// fieldCodec's translation rules. key values, if present, will be used to -// replace whichever values could have been provided as keys. so for instance, -// with an int field which is keyed, values are actually of type int64, but -// if strings are present, those are used. for a time quantum field, the time -// is set from the signed field, and the value is replaced by the key if -// keys are provided. -type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error - -// fieldCodec represents something that can encode and decode a -// particular field. Its methods are not reentrant, it uses internal -// buffers. -type fieldCodec struct { - valueKeys *StringTable - currentOp *FieldOperation - decode jsonDecFn - encode jsonEncFn - // For decimal: Decimal digits of precision. So for instance, with - // scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as - // 120. - scaleUnit int64 - // For timestamp: we use timeUnit to determine the scale at which - // to store a timestamp. For example if the timeUnit is milliseconds - // we store the number of milliseconds from the given epoch. - timeUnit string - scale int64 - epoch int64 // used only by Timestamp fields - scratch []uint64 // reusable scratch space for sets of values - keys KeyTranslator - buf []byte // scratch space for format operations. - fieldType FieldType -} - -// JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps. -type JSONCodec struct { - fieldTypes map[string]FieldType - recKeys *StringTable - fields map[string]*fieldCodec - keys KeyTranslator - currentOp *Operation -} - -// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which -// can be used to write to its buffer. Both types are just embedded because -// they have non-overlapping APIs. Please don't look at my horrible face. -// -// This has some internal objects it can use to stash things -// so that it can pass &foo to enc.Encode() without needing to heap-allocate -// something for the interface conversion, and a buffer it can use for -// converting numbers or times. -// -// It also has an internal error buffer. In fact, in many cases, errors -// are so far as I can tell absolutely impossible -- bytes.Buffer specifically -// promises never to yield an error, and nothing seems to hint that -// enc.Encode can error on integers, strings, booleans, or arrays. So we -// have dozens of error checks that we can't cause to happen even with -// malformed inputs... so we eat those errors, don't require them to be -// checked externally, and return them when done if anyone checks. -type jsonBuffer struct { - *bytes.Buffer - enc *json.Encoder - buf [48]byte // scratch space for using strconv.AppendInt, etc - uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64 - strbuf []string // dummy buffer so that we don't have to alloc to print []string - str string // and again, "so we don't have to alloc a copy" - err error -} - -// Encode removes the stray newlines added by json.Encoder. -func (j *jsonBuffer) Encode(v interface{}) { - err := j.enc.Encode(v) - if err == nil { - j.Truncate(j.Len() - 1) - } else { - j.err = err - } -} - -// EncodeInt uses AppendInt into a static buffer to reduce allocs. -func (j *jsonBuffer) EncodeInt(i int64) { - rep := strconv.AppendInt(j.buf[:0], i, 10) - _, _ = j.Write(rep) -} - -// EncodeUint uses AppendUint into a static buffer to reduce allocs. -func (j *jsonBuffer) EncodeUint(u uint64) { - rep := strconv.AppendUint(j.buf[:0], u, 10) - _, _ = j.Write(rep) -} - -// EncodeUints uses a static copy of a []uint64 -- the slice, not its -// contents -- in already-allocated memory so the interface conversion -// doesn't have to do that. -func (j *jsonBuffer) EncodeUints(u []uint64) { - j.uintbuf = u - j.Encode(&j.uintbuf) - j.strbuf = nil -} - -// EncodeStrings uses a static copy of a []string -- the slice, not its -// contents -- in already-allocated memory so the interface conversion -// doesn't have to do that. -func (j *jsonBuffer) EncodeStrings(s []string) { - j.strbuf = s - j.Encode(&j.strbuf) - j.strbuf = nil -} - -// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs, -// while also surrounding the value with quotes. -func (j *jsonBuffer) EncodeQuotedUint(u uint64) { - j.buf[0] = '"' - rep := strconv.AppendUint(j.buf[1:1], u, 10) - j.buf[len(rep)+1] = '"' - _, _ = j.Write(j.buf[:len(rep)+2]) -} - -// EncodeString appends the JSON encoding of a string. This exists -// because otherwise runtime allocates a heap-allocated copy of the -// string to live inside an interface{} for the duration of a function -// call... -func (j *jsonBuffer) EncodeString(s string) { - j.str = s - j.Encode(&j.str) - j.str = "" -} - -// EncodeTime exists because time's MarshalJSON allocates a new -// buffer every time it gets called, resulting in a full 13% of -// all the allocations produced in a test run, plus another 13% -// or so of them which were for the copies of the time objects -// made to stuff them into an interface{}. Eww. -func (j *jsonBuffer) EncodeTime(t time.Time) { - j.buf[0] = '"' - rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano) - j.buf[len(rep)+1] = '"' - _, _ = j.Write(j.buf[:len(rep)+2]) -} - -// EncodeBool writes a literal representation directly to reduce allocs. -func (j *jsonBuffer) EncodeBool(b bool) { - if b { - _, _ = j.WriteString("true") - } else { - _, _ = j.WriteString("false") - } -} - -func (j *jsonBuffer) Err() error { - return j.err -} - -func newJSONBuffer(data []byte) *jsonBuffer { - e := &jsonBuffer{Buffer: bytes.NewBuffer(data)} - e.enc = json.NewEncoder(e.Buffer) - e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON - return e -} - -var _ Codec = &JSONCodec{} - -func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) { - j := &JSONCodec{ - fields: map[string]*fieldCodec{}, - fieldTypes: map[string]FieldType{}, - } - if keys != nil { - j.recKeys = NewStringTable() - j.keys = keys - } - return j, nil -} - -func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeTimeQuantum, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue - fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeSet, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeSetValue - fieldCodec.encode = fieldCodec.EncodeSetValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeInt, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeIntValue - fieldCodec.encode = fieldCodec.EncodeIntValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeMutex, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeMutexValue - fieldCodec.encode = fieldCodec.EncodeMutexValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddBoolField(name string) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeBool, - } - fieldCodec.decode = fieldCodec.DecodeBoolValue - fieldCodec.encode = fieldCodec.EncodeBoolValue - return codec.addField(name, fieldCodec) -} - -// TimestampField is used to store seconds since unix epoch. The numeric values -// stored are adjusted based on the given time.Duration; for instance, if the -// time scale is time.Second, then the second after the epoch is stored as 1, -// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch -// passed to this function should be the offset from the Unix epoch to the -// desired epoch, in the same scale. (So if the scale is milliseconds, -// it should be the Unix timestamp in seconds, times 1000.) -func (codec *JSONCodec) AddTimestampField(name string, timeScale string, epoch int64) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeTimeStamp, - timeUnit: timeScale, - epoch: epoch, - } - fieldCodec.decode = fieldCodec.DecodeTimeValue - fieldCodec.encode = fieldCodec.EncodeTimeValue - return codec.addField(name, fieldCodec) -} - -// AddDecimalField adds a decimal field, which is stored as integer values -// with a scale offset, but parsed as floating point values. For instance, -// with decimalScale=2, `0.01` would store the value 1. -func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeDecimal, - scale: decimalScale, - scaleUnit: int64(math.Pow(10, float64(decimalScale))), - } - fieldCodec.decode = fieldCodec.DecodeDecimalValue - fieldCodec.encode = fieldCodec.EncodeDecimalValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error { - if _, ok := codec.fields[name]; ok { - return fmt.Errorf("duplicate field %q", name) - } - if fieldCodec.keys != nil { - fieldCodec.valueKeys = NewStringTable() - } - codec.fields[name] = fieldCodec - codec.fieldTypes[name] = fieldCodec.fieldType - return nil -} - -// decodeSetOrValue decodes a value which might be either an array of values or a -// single value, where values might be string keys or bare numbers, calling cb for -// each value it finds. -func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { - switch dataType { - case jsonparser.Array: - _, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - var id uint64 - switch dataType { - case jsonparser.String: - id, err = j.valueKeys.ID(value) - if err != nil { - return - } - err = cb(id) - case jsonparser.Number: - if j.valueKeys != nil { - err = errors.New("expecting key, got numeric value") - return - } - id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64) - if err != nil { - return - } - err = cb(id) - default: - err = fmt.Errorf("expecting value or array, got %v", dataType) - } - }) - if err != nil { - return err - } - if arrayErr != nil { - return arrayErr - } - case jsonparser.String: - id, err := j.valueKeys.ID(data) - if err != nil { - return err - } - return cb(id) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting key, got numeric value") - } - id, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - return cb(id) - default: - return fmt.Errorf("expecting value or array, got %v", dataType) - } - return err -} - -// DecodeSetValue decodes a set of unsigned values from the provided data into the -// associated currentOp. -func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { - return j.decodeSetOrValue(dataType, data, func(id uint64) error { - j.currentOp.AddPair(recID, id) - return nil - }) -} - -func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 1 || len(values) == 1 { - // simplify: just hand this off to a single-value case - return j.EncodeMutexValue(dst, values, signed, keys) - } - appendKeysJSON(dst, values, keys) - return nil -} - -// DecodeIntValue decodes a single signed value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String: - value, err := j.valueKeys.IntID(data) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, value) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting string key, got numeric value") - } - value, err := strconv.ParseInt(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, value) - default: - if j.valueKeys != nil { - return fmt.Errorf("expecting string key, got %v", dataType) - } else { - return fmt.Errorf("expecting integer value, got %v", dataType) - } - } - return nil -} - -func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 0 { - if len(signed) == 0 { - return errors.New("encodeIntValue: need a value") - } - dst.EncodeInt(signed[0]) - return nil - } - dst.EncodeString(keys[0]) - return nil -} - -// DecodeMutexValue decodes a single unsigned value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String: - value, err := j.valueKeys.ID(data) - if err != nil { - return err - } - j.currentOp.AddPair(recID, value) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting string key, got numeric value") - } - value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - j.currentOp.AddPair(recID, value) - default: - if j.valueKeys != nil { - return fmt.Errorf("expecting string key, got %v", dataType) - } else { - return fmt.Errorf("expecting integer value, got %v", dataType) - } - } - return nil -} - -func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 0 { - if len(values) == 0 { - return errors.New("encodeMutexValue: need a value") - } - dst.EncodeUint(values[0]) - return nil - } - dst.EncodeString(keys[0]) - return nil -} - -// DecodeBoolValue decodes a single true/false value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { - value := uint64(0) - switch typ { - case jsonparser.String: - if string(data) != "true" && string(data) != "false" { - return fmt.Errorf("expecting boolean, got %q", data) - } - // if it's exactly "true" or "false" let's be forgiving - fallthrough - case jsonparser.Boolean: - if data[0] == 't' { - value = 1 - } - case jsonparser.Number: - v, err := jsonparser.GetInt(data) - if err != nil { - return err - } - if v == 1 { - value = 1 - } else if v != 0 { - return errors.New("boolean should be true/false/0/1") - } - default: - return errors.New("boolean should be true/false/0/1") - } - j.currentOp.AddPair(recID, value) - return nil -} - -func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(values) == 0 { - return errors.New("encoding boolean value, but none provided") - } - var x bool - if values[0] != 0 { - x = true - } - dst.EncodeBool(x) - return nil -} - -// DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the -// provided data into the associated currentOp. -func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { - j.scratch = j.scratch[:0] - stamp := time.Unix(0, 0).UTC() - err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { - switch string(key) { - case "time": - switch dataType { - case jsonparser.String: - stamp, err = time.Parse(time.RFC3339, pretendByteIsString(value)) - case jsonparser.Number: - var unix int64 - unix, err = strconv.ParseInt(pretendByteIsString(value), 10, 64) - stamp = time.Unix(unix, 0) - default: - return fmt.Errorf("expecting time, got %q", value) - } - case "values": - err = j.decodeSetOrValue(dataType, value, func(id uint64) error { - j.scratch = append(j.scratch, id) - return nil - }) - } - return err - }) - if err != nil { - return err - } - if len(j.scratch) > 0 { - defer func() { - // mark these as consumed so if we get called - // again, and don't see a "values" key, we don't reuse them. - j.scratch = j.scratch[:0] - }() - unix := stamp.UnixNano() - for _, value := range j.scratch { - j.currentOp.AddStampedPair(recID, value, unix) - } - } - return nil -} - -// Our format does not allow a record to have multiple values set with -// different timestamps at the same time. We use the first timestamp for -// the whole set. -func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(values) == 0 { - return errors.New("encoding time quantum value: no value provided") - } - dst.WriteString(`{"values":`) - appendKeysJSON(dst, values, keys) - // a zero timestamp is idiomatic for no-time-provided, and i sort of - // hate that, but here we are. - if len(signed) > 0 && signed[0] != 0 { - _, _ = dst.WriteString(`,"time":`) - dst.EncodeTime(time.Unix(0, signed[0]).UTC()) - } - _, _ = dst.WriteString(`}`) - return nil -} - -// DecodeTimeValue will eventually work but right now it doesn't actually. -func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { - var stamp time.Time - switch dataType { - case jsonparser.String: - stamp, err = time.Parse(time.RFC3339Nano, pretendByteIsString(data)) - if err != nil { - return fmt.Errorf("parsing timestamp: %w", err) - } - j.currentOp.AddSignedPair(recID, TimestampToVal(j.timeUnit, stamp)-j.epoch) - case jsonparser.Number: - // We could in theory convert this to a time, then convert it - // back, by multiplying by scaleUnit, then dividing. Or... not. - i64, err := strconv.ParseInt(pretendByteIsString(data), 10, 64) - if err != nil { - return fmt.Errorf("parsing numeric timestamp: %w", err) - } - j.currentOp.AddSignedPair(recID, i64) - default: - return fmt.Errorf("expecting time, got %s", dataType) - } - return nil -} - -func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(signed) == 0 { - return errors.New("encoding time value: no value provided") - } - t, err := ValToTimestamp(j.timeUnit, signed[0]+j.epoch) - if err != nil { - return errors.Wrap(err, "translating value to timestamp") - } - dst.EncodeTime(t) - return nil -} - -// DecodeDecimalValue will eventually work but right now it doesn't actually. -func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String, jsonparser.Number: - value, err := jsonparser.GetFloat(data) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, int64(value*float64(j.scaleUnit))) - default: - return fmt.Errorf("expecting floating-point value, got %v", dataType) - } - - return nil -} - -func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10) - scale := int(j.scale) - if len(j.buf) > scale { - j.buf = append(j.buf, '.') - // shove the last scaleUnit values over - copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:]) - j.buf[len(j.buf)-scale-1] = '.' - } - _, _ = dst.Write(j.buf) - return nil -} - -// FieldTypes gives a mapping of fields to their basic types used by this -// codec. -func (codec *JSONCodec) FieldTypes() map[string]FieldType { - return codec.fieldTypes -} - -// ParseKeyedRecords parses the records it finds. Note that, when you're doing -// a Write op, this will update ClearRecordIDs automatically as it goes, -// even though record_ids isn't specified in the JSON for that case. -func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { - seen := make(map[uint64]struct{}) - return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - id, err := codec.recKeys.ID(key) - if err != nil { - return err - } - if _, ok := seen[id]; ok { - return fmt.Errorf("key %q duplicated in input", key) - } - seen[id] = struct{}{} - if codec.currentOp.OpType == OpWrite { - codec.currentOp.ClearRecordIDs = append(codec.currentOp.ClearRecordIDs, id) - } - return jsonparser.ObjectEach(value, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - fieldCodec, ok := codec.fields[string(key)] - if !ok { - return errFieldNotFound{string(key)} - } - err := fieldCodec.decode(id, dataType, value) - if err != nil { - return fmt.Errorf("parsing value for field %q: %v", key, err) - } - return nil - }) - }) -} - -func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) { - op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq} - codec.currentOp = op - err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - switch string(key) { - case "action": - op.OpType, err = ParseOpType(string(value)) - if err != nil { - return fmt.Errorf("unknown action type %q", value) - } - case "records": - for name, fieldCodec := range codec.fields { - fieldOp := &FieldOperation{} - op.FieldOps[name] = fieldOp - // cache this op so we don't have to do the lookups every time - fieldCodec.currentOp = fieldOp - } - err = codec.ParseKeyedRecords(value) - if err != nil { - return fmt.Errorf("parsing records: %v", err) - } - case "record_ids": - var id uint64 - var idErr error - _, err = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - id, idErr = codec.recKeys.ID(value) - if idErr != nil { - return - } - op.ClearRecordIDs = append(op.ClearRecordIDs, id) - - }) - // if we got an error converting an ID, error out with it here. - // we can't stop the ArrayEach early, though? - if idErr != nil { - return idErr - } - if err != nil { - return err - } - case "fields": - _, err = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - op.ClearFields = append(op.ClearFields, string(value)) - }) - if err != nil { - return err - } - default: - return fmt.Errorf("unknown operation field %q", key) - } - return nil - }) - if err != nil { - return nil, err - } - if op.OpType == OpNone { - return nil, fmt.Errorf("action not specified") - } - return op, err -} - -// Parse reads a request, but does not sort the results at all or divide -// them into shards. -func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - return codec.ParseBytes(data) -} - -// ParseBytes reads a request from a slice of bytes. We need to use a -// byte slice because jsonparser's model is fundamentally built around -// being able to random-access the slice and return slices of it, so -// it can't really admit functional streaming. If we need streaming, the -// streaming needs to be at a higher level. -func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { - var ops []*Operation - var lastErr error - var seq int - _, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { - switch dataType { - case jsonparser.Object: - op, err := codec.ParseOperation(value, seq) - seq++ - if err != nil { - lastErr = fmt.Errorf("parsing operation: %v", err) - return - } - ops = append(ops, op) - default: - lastErr = fmt.Errorf("expected operation, found %s", dataType) - } - }) - if lastErr != nil { - return nil, lastErr - } - if err != nil { - return nil, err - } - // and now, key translation! - var keyMap []uint64 - if codec.keys != nil { - keyMap, err = codec.recKeys.MakeIDMap(codec.keys) - if err != nil { - return nil, fmt.Errorf("trying to find record key mapping: %w", err) - } - } - req = &Request{} - valueMaps := map[string]func(*FieldOperation) error{} - for name, fieldCodec := range codec.fields { - // make closure survive iteration - fieldCodec := fieldCodec - if fieldCodec.keys != nil { - fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys) - if err != nil { - return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err) - } - if fieldCodec.fieldType == FieldTypeInt { - valueMaps[name] = func(fo *FieldOperation) error { - return translateSigned(fieldMap, fo.Signed) - } - } else { - valueMaps[name] = func(fo *FieldOperation) error { - return translateUnsigned(fieldMap, fo.Values) - } - } - } - } - for _, op := range ops { - // For Clear and Write, we need to translate/sort our record ID - // list. - if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { - if keyMap != nil { - if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil { - return nil, fmt.Errorf("mapping record keys for clear op: %w", err) - } - } - } - // For clear/delete, that's all we need to do; there's no meaningful fieldops under them. - if op.OpType == OpClear || op.OpType == OpDelete { - continue - } - for field, fieldOp := range op.FieldOps { - if len(fieldOp.RecordIDs) == 0 { - delete(op.FieldOps, field) - continue - } - if keyMap != nil { - if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil { - return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err) - } - } - if fieldTranslate, ok := valueMaps[field]; ok { - if err = fieldTranslate(fieldOp); err != nil { - return nil, fmt.Errorf("mapping values for op on %q: %w", field, err) - } - } - // Sort by column keys, for now. - // Write op will also want to clear every field we saw. - if op.OpType == OpWrite { - op.ClearFields = append(op.ClearFields, field) - } - } - } - req.Ops = ops - return req, nil -} - -// AppendBytes appends bytes which we would expect to produce the same -// request, using this codec. It does not try to recreate FieldTypes, which -// would be handled by the codec anyway. It's written as an Append so you -// can reuse a buffer. -func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) { - if req == nil || len(req.Ops) == 0 { - return append(data, "[]"...), nil - } - dst := newJSONBuffer(data) - // we ignore the FieldTypes part of the Request, which is just there to - // let the request's ByShard use the correct sorting routines, which is - // itself sort of awful. The codec will insert it again when parsing the - // bytes. - _, _ = dst.WriteString(`,`) - for _, op := range req.Ops { - // EncodeJSON needs to have access to this codec's field data - // and key translators. - err = op.EncodeJSON(dst, codec) - if err != nil { - return nil, err - } - _, _ = dst.WriteString(`,`) - } - // delete the last comma - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`]`) - // there's very few ways an error could occur, possibly none, but - // just in case lots of internal operations stashed an error if one - // happened, so we'll return anything they came up with. - return dst.Bytes(), dst.Err() -} - -// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed -// list of quoted strings if there's a key translator, or integer values -// if the KeyTranslator is nil, comma-separated. It can error because a -// translator can error. -func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { - return appendIDsJSON(dst, values, keys) -} - -func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) { - if len(values) == 0 && len(keys) == 0 { - _, _ = dst.WriteString("[]") - return - } - if len(keys) != 0 { - dst.EncodeStrings(keys) - } else { - dst.EncodeUints(values) - } -} - -func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { - if len(values) == 0 { - _, _ = dst.WriteString("[]") - return nil - } - - if keys == nil { - dst.EncodeUints(values) - return nil - } - _, _ = dst.WriteString(`[`) - translated, err := keys.TranslateIDs(values...) - if err != nil { - return err - } - for _, v := range values { - dst.EncodeString(translated[v]) - _, _ = dst.WriteString(`,`) - } - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`]`) - return nil -} - -// RequestByShard makes up for the fact that we don't want to stash the -// field type data in the request, but we need it to actually do the by-shard. -func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) { - return req.ByShard(codec.fieldTypes) -} - -// EncodeJSON encodes an operation as JSON using a provided buffer. -// It uses the provided codec where necessary to help with key -// translation. -func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) { - // We're not trying to guarantee that what we produce makes sense or is - // identical to what produced us, just to write our current state out. - if o == nil { - _, _ = dst.WriteString(`{}`) - return nil - } - _, _ = dst.WriteString(`{"action":`) - dst.EncodeString(o.OpType.String()) - // it is intentional that o.Seq isn't encoded here; it makes no sense - // to allow an op to specify its seq in JSON. - if o.OpType != OpWrite { - // for Write, ClearRecordIDs and ClearFields were computed - // from the records being written, and aren't actually part of the data. - if len(o.ClearRecordIDs) != 0 { - _, _ = dst.WriteString(`,"record_ids":`) - err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys) - if err != nil { - return err - } - } - if len(o.ClearFields) != 0 { - _, _ = dst.WriteString(`,"fields":`) - dst.EncodeStrings(o.ClearFields) - } - } - if len(o.FieldOps) == 0 { - _, _ = dst.WriteString(`}`) - return - } - // collect all the fields that have a non-empty set of records. we can - // just ignore the others. - fieldNames := make([]string, 0, len(o.FieldOps)) - for field, op := range o.FieldOps { - if op != nil && len(op.RecordIDs) != 0 { - fieldNames = append(fieldNames, field) - } - } - // nevermind then - if len(fieldNames) == 0 { - _, _ = dst.WriteString(`}`) - return nil - } - _, _ = dst.WriteString(`,"records":{`) - - // now we have to invert the logic, creating records from - // fields with corresponding ops. uh-oh. - fieldOps := make([]*FieldOperation, len(o.FieldOps)) - fieldCodecs := make([]*fieldCodec, len(o.FieldOps)) - fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys - indexes := make([]int, len(o.FieldOps)) - sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] }) - next := ^uint64(0) - var idKeys map[uint64]string - if codec.keys != nil { - idKeys = make(map[uint64]string) - } - for i, field := range fieldNames { - // populate a parallel slice of field ops so we don't have - // to do map lookups for every single piece of data - op := o.FieldOps[field] - fieldOps[i] = op - fc := codec.fields[field] - if fc == nil { - return fmt.Errorf("unknown field: %q", field) - } - fieldCodecs[i] = fc - if codec.keys != nil { - // we'll build a list of keys we need for any records - for _, v := range op.RecordIDs { - idKeys[v] = "" - } - } - id := op.RecordIDs[0] - if id < next { - next = id - } - if fc.keys != nil { - thisFieldKeys := make([]string, len(op.RecordIDs)) - if fc.fieldType == FieldTypeInt { - u := make([]uint64, len(op.Signed)) - for i := range op.Signed { - u[i] = uint64(op.Signed[i]) - } - valueKeys, err := fc.keys.TranslateIDs(u...) - if err != nil { - return err - } - for j, v := range op.Signed { - thisFieldKeys[j] = valueKeys[uint64(v)] - } - } else { - valueKeys, err := fc.keys.TranslateIDs(op.Values...) - if err != nil { - return err - } - for j, v := range op.Values { - thisFieldKeys[j] = valueKeys[v] - } - } - fieldKeys[i] = thisFieldKeys - } - } - // ... no fieldops actually have any entries. therefore no record will - // have any fields set, therefore no records exist... - if next == ^uint64(0) { - _, _ = dst.WriteString(`}}`) - return nil - } - if codec.keys != nil { - idList := make([]uint64, 0, len(idKeys)) - for k := range idKeys { - idList = append(idList, k) - } - idKeys, err = codec.keys.TranslateIDs(idList...) - if err != nil { - return err - } - } - - // next is the ID of a record which exists for at least one field. - // we'll recompute it every loop. - for next != ^uint64(0) { - if codec.keys != nil { - dst.EncodeString(idKeys[next]) - } else { - dst.EncodeQuotedUint(next) - } - _, _ = dst.WriteString(`:{`) - current := next - next = ^uint64(0) - for i, field := range fieldNames { - op := fieldOps[i] - idx := indexes[i] - if idx >= len(op.RecordIDs) { - continue - } - id := op.RecordIDs[idx] - if id == current { - var j int - // count ahead to first index which is either outside the list - // or a different id - for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ { - } - - // field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i])) - // print this one, and advance this index to next position - dst.EncodeString(field) - _, _ = dst.WriteString(`:`) - var values []uint64 - var signed []int64 - var keys []string - if len(op.Values) >= j { - values = op.Values[idx:j] - } - if len(op.Signed) >= j { - signed = op.Signed[idx:j] - } - if len(fieldKeys[i]) >= j { - keys = fieldKeys[i][idx:j] - } - err = fieldCodecs[i].encode(dst, values, signed, keys) - if err != nil { - return err - } - _, _ = dst.WriteString(`,`) - indexes[i] = j - // and we'll fall through to the id < next check, so we - // just set id here. - if indexes[i] < len(op.RecordIDs) { - id = op.RecordIDs[indexes[i]] - } else { - id = ^uint64(0) - } - } - if id < next { - next = id - } - } - // we should always have a trailing comma after any entry, and - // if there were no entries we shouldn't have had a list... - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`},`) - } - dst.Truncate(dst.Len() - 1) - // close both the records list, and the whole object. - _, _ = dst.WriteString(`}}`) - // In theory, there's no way for most of these to produce errors, - // but just in case, we'll check for an error every operation or so. - return dst.Err() -} - -type errFieldNotFound struct { - field string -} - -func (err errFieldNotFound) Error() string { - return fmt.Sprintf("field not found: %q", err.field) -} - -// TimestampToVal takes a time unit and a time.Time and converts it to an integer value -func TimestampToVal(unit string, ts time.Time) int64 { - switch unit { - case "s": - return ts.Unix() - case "ms": - return ts.UnixMilli() - case "us": - return ts.UnixMicro() - case "ns": - return ts.UnixNano() - } - return 0 - -} - -// ValToTimestamp takes a timeunit and an integer value and converts it to time.Time -func ValToTimestamp(unit string, val int64) (time.Time, error) { - switch unit { - case "s": - return time.Unix(val, 0).UTC(), nil - case "ms": - return time.UnixMilli(val).UTC(), nil - case "us", "μs": - return time.UnixMicro(val).UTC(), nil - case "ns": - return time.Unix(0, val).UTC(), nil - default: - return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit) - } -} diff --git a/ingest/codec_test.go b/ingest/codec_test.go deleted file mode 100644 index 838328981..000000000 --- a/ingest/codec_test.go +++ /dev/null @@ -1,978 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/molecula/featurebase/v3/shardwidth" -) - -func TestStableTranslator(t *testing.T) { - tr := newStableTranslator() - m1, err := tr.TranslateKeys("a", "b") - if err != nil { - t.Fatalf("translation error on initial keys: %v", err) - } - m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6) - if err != nil { - t.Fatalf("translation error on reverse lookup: %v", err) - } - m3, err := tr.TranslateKeys("a", "k-6") - if err != nil { - t.Fatalf("translation error on new keys: %v", err) - } - for k, v := range m3 { - if m2[v] != k { - t.Fatalf("expected round trip to equate %q and %d", k, v) - } - } -} - -func TestMakeCodec(t *testing.T) { - codec, _ := NewJSONCodec(nil) - err := codec.AddSetField("set", nil) - if err != nil { - t.Fatalf("unexpected error creating field: %v", err) - } - err = codec.AddSetField("set", nil) - if err == nil { - t.Fatalf("expected error creating duplicate field, didn't get it") - } -} - -func TestEncode(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - codecs := []*JSONCodec{codec} - - // redo all of that, only on a keyed translator - codec, _ = NewJSONCodec(newStableTranslator()) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - codecs = append(codecs, codec) - - encodeTests := []*Request{ - { - Ops: []*Operation{ - { - OpType: OpWrite, - ClearRecordIDs: []uint64{0, 1, 2, 3, 5}, - ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"}, - FieldOps: map[string]*FieldOperation{ - "int": { - RecordIDs: []uint64{0, 1}, - Signed: []int64{1, -3}, - }, - "intkeys": { - RecordIDs: []uint64{0, 1}, - Signed: []int64{1, 1}, - }, - "set": { - RecordIDs: []uint64{0, 1, 2, 2}, - Values: []uint64{1, 1, 0, 1}, - }, - "setkeys": { - RecordIDs: []uint64{0, 1, 2, 2}, - Values: []uint64{1, 1, 0, 1}, - }, - "mutex": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{1, 2}, - }, - "mutexkeys": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{1, 2}, - }, - "tq": { - RecordIDs: []uint64{5, 5}, - Values: []uint64{8, 9}, - Signed: []int64{1234567890e9, 1234567890e9}, - }, - "tqkeys": { - RecordIDs: []uint64{3, 3}, - Values: []uint64{2, 4}, - Signed: []int64{1234567890e9, 1234567890e9}, - }, - "ts": { - RecordIDs: []uint64{0}, - Signed: []int64{1}, - }, - "bool": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{0, 1}, - }, - "dec": { - RecordIDs: []uint64{0, 1, 2}, - Signed: []int64{123, -123, 0}, - }, - }, - }, - { - OpType: OpClear, - Seq: 1, - ClearRecordIDs: []uint64{6}, - ClearFields: []string{"tq"}, - }, - }, - }, - { - // this one needs to get filled in programmatically; see below - Ops: []*Operation{ - { - OpType: OpSet, - FieldOps: map[string]*FieldOperation{}, - }, - }, - }, - { - Ops: []*Operation{ - { - OpType: OpRemove, - FieldOps: map[string]*FieldOperation{ - "set": {}, - }, - }, - }, - }, - } - // and now we populate encodeTests[1] with a larger pool of data - const dataSize = 5000 - shardCount := uint64(600) // shards we want to target - passes := uint64(0) - recordIDs := make([]uint64, dataSize) - values := make([]uint64, dataSize) - timeStamps := make([]int64, dataSize) - signedValues := make([]int64, dataSize) - for i := uint64(0); i < dataSize; i++ { - if (i % shardCount) == 0 { - passes++ - } - recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes - values[i] = (i % 4) - timeStamps[i] = int64(1234567890e9 + (i * 100e9)) - signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys - } - // ensure record IDs are sorted, because other stuff might rely on this - sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) - op := encodeTests[1].Ops[0] - op.FieldOps["tq"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: append([]uint64{}, values...), - Signed: append([]int64{}, timeStamps...), - } - op.FieldOps["tqkeys"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: values, - Signed: timeStamps, - } - op.FieldOps["int"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Signed: append([]int64{}, signedValues...), - } - op.FieldOps["intkeys"] = &FieldOperation{ - RecordIDs: recordIDs, - Signed: signedValues, - } - // for sets, we want to shuffle things into fewer shards, and ensure - // non-duplication of values within each record, but also have lots - // of duplication of record IDs in the low shards - recordIDs = make([]uint64, dataSize) - values = make([]uint64, dataSize) - valuesPerRecord := uint64(5) - recordsPerShard := dataSize / valuesPerRecord / 30 - if recordsPerShard < 1 { - recordsPerShard = 1 - } - shard := uint64(0) - nextID := uint64(0) - nextValue := uint64(0) - for i := uint64(0); i < dataSize; i++ { - recordIDs[i] = nextID - values[i] = nextValue + (i % valuesPerRecord) - nextValue++ - if nextValue == valuesPerRecord { - nextValue = 0 - nextID++ - if nextID%(1< 1 { - valuesPerRecord-- - } - } - } - } - sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) - op.FieldOps["set"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: append([]uint64{}, values...), - } - op.FieldOps["setkeys"] = &FieldOperation{ - RecordIDs: recordIDs, - Values: values, - } - var buf []byte - for i, tc := range encodeTests { - for _, c := range codecs { - data, err := c.AppendBytes(tc, buf[:0]) - if err != nil { - t.Fatalf("encode test %d: error encoding: %v", i, err) - } - // t.Logf("data:\n%s", data) - req, err := c.ParseBytes(data) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: error parsing: %v", i, err) - } - err = req.Compare(tc) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) - } - data, err = c.AppendBytes(req, buf[:0]) - if err != nil { - t.Fatalf("encode test %d: error encoding: %v", i, err) - } - // t.Logf("data:\n%s", data) - req2, err := c.ParseBytes(data) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: error parsing: %v", i, err) - } - err = req2.Compare(tc) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) - } - } - } -} - -func TestCodecErrors(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - testCases := []struct { - name string - json []byte - error string - }{ - { - name: "no action", - json: []byte(`[{"records":{"0":{"set":[0]}}}]`), - error: "action not specified", - }, - { - name: "unknown action", - json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`), - error: "unknown action", - }, - { - name: "unknown field", - json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`), - error: "field not found", - }, - { - name: "unknown operation field", - json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`), - error: "unknown operation field", - }, - { - name: "expected operation", - json: []byte(`[true]`), - error: "expected operation", - }, - { - name: "expecting key", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`), - error: "expecting key", - }, - { - name: "invalid int for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`), - error: "boolean should be", - }, - { - name: "invalid number for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`), - error: "looks like Number", - }, - { - name: "invalid string for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`), - error: "expecting boolean", - }, - { - name: "nonsense bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`), - error: "boolean should be", - }, - { - name: "expecting numeric value", - json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting value", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`), - error: "expecting value", - }, - { - name: "expecting array-key", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`), - error: "expecting key", - }, - { - name: "expecting array-value", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`), - error: "expecting value", - }, - { - name: "expecting numeric array-value", - json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`), - error: "invalid syntax", - }, - { - name: "expecting numeric value", - json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting int key", - json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`), - error: "expecting string key", - }, - { - name: "expecting int value", - json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`), - error: "expecting integer value", - }, - { - name: "expecting string array-value", - json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`), - error: "expecting string key", - }, - { - name: "expecting numeric mutex value", - json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting mutex key", - json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`), - error: "expecting string key", - }, - { - name: "expecting mutex value", - json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`), - error: "expecting integer value", - }, - { - name: "expecting mutex string value", - json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`), - error: "expecting string key", - }, - { - name: "time quantum invalid time", - json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`), - error: "expecting time", - }, - { - name: "time stamp invalid integer", - json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`), - error: "parsing numeric time", - }, - { - name: "time stamp invalid string", - json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`), - error: "parsing time", - }, - { - name: "time stamp invalid type", - json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`), - error: "expecting time", - }, - { - name: "invalid decimal", - json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`), - error: "expecting floating", - }, - { - name: "duplicate record", - json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`), - error: "duplicated in input", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - req, err := codec.ParseBytes(tc.json) - if err == nil { - req.Dump(t.Logf) - t.Fatalf("expected error like %q, got request instead", tc.error) - } else { - msg := err.Error() - if !strings.Contains(msg, tc.error) { - t.Fatalf("expected error like %q, got %q", tc.error, msg) - } - } - }) - } -} - -func TestSimpleCodec(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - var nextShard = uint64(1<> shardwidth.Exponent: f} - } - output := make(ShardedFieldOperation) - sortToShardsInto(f, bitsRemaining-8, output) - return output -} - -// sortToShardsInto puts the shards it finds into the given map, so that -// as we split off buckets, they can be inserted into the same map. -func sortToShardsInto(f *FieldOperation, shift int, into ShardedFieldOperation) { - if shift < shardwidth.Exponent { - shift = shardwidth.Exponent - } - nextShift := shift - 8 - if nextShift < shardwidth.Exponent { - nextShift = shardwidth.Exponent - } - // count things that belong in each of the 256 buckets - var buckets [256]int - var starts [256]int - - // compute the buckets ourselves - for _, r := range f.RecordIDs { - b := (r >> shift) & 0xFF - buckets[b]++ - } - total := 0 - // compute starting points of each bucket, converting the - // bucket counts into ends - for i := range buckets { - starts[i] = total - total += buckets[i] - buckets[i] = total - } - // starts[n] is the index of the first thing that should - // go in that bucket, buckets[n] is the index of the first - // thing that shouldn't - var bucketOp FieldOperation - for bucket, start := range starts { - end := buckets[bucket] - if end <= start { - continue - } - for j := start; j < end; j++ { - want := int((f.RecordIDs[j] >> shift) & 0xFF) - for want != bucket { - // move this to the beginning of the - // bucket it wants to be in, swapping - // the thing there here - dst := starts[want] - f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j] - if f.Values != nil { - f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j] - } - if f.Signed != nil { - f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j] - } - starts[want]++ - want = int((f.RecordIDs[j] >> shift) & 0xFF) - } - } - // If shift == shardwidth.Exponent, then this is a completed - // shard and can go into the sharded output. otherwise, we - // can subdivide it. - bucketOp.RecordIDs = f.RecordIDs[start:end] - if f.Values != nil { - bucketOp.Values = f.Values[start:end] - } - if f.Signed != nil { - bucketOp.Signed = f.Signed[start:end] - } - if shift == shardwidth.Exponent { - x := bucketOp - into[f.RecordIDs[start]>>shardwidth.Exponent] = &x - } else { - sortToShardsInto(&bucketOp, nextShift, into) - } - } -} - -const shardMask = ((uint64(1) << shardwidth.Exponent) - 1) - -// SortByValues sorts the operation by values first, then by record -// ID within each value. This is the best ordering for set/mutex fields, -// where we'll want to generate positions in that order. For these -// purposes, a time quantum or bool counts as a kind of a set. -func (f *FieldOperation) SortByValues() { - keys := make([]uint64, len(f.RecordIDs)) - for i, v := range f.RecordIDs { - keys[i] = (f.Values[i] << shardwidth.Exponent) | (v & shardMask) - } - f.SortByKeys(keys) -} - -// SortByRecords sorts the operation by record ID, and not by value at -// all. This makes the most sense for int fields and the like. -func (f *FieldOperation) SortByRecords() { - f.SortByKeys(f.RecordIDs) -} - -// SortByKeys reorganizes the record IDs and values of f according to the -// corresponding members of keys. -func (f *FieldOperation) SortByKeys(keys []uint64) { - if len(f.RecordIDs) < 2 { - return - } - diffMask := uint64(0) - prev := keys[0] - for _, r := range keys[1:] { - diffMask |= r ^ prev - prev = r - } - bitsRemaining := bits.Len64(diffMask) - sortPartialByKeys(f, keys, bitsRemaining-8) -} - -// simpleSort sorts a FieldOperation by external keys, or record IDs. It's a -// horribly naive bubble sort because N is small and a more complex algorithm -// doesn't help as much as you'd hope. This beats using stdlib sort by about -// a factor of two for those small N, for larger N we're using the radix sort -// that calls this. -// -// External keys exist only when we are sorting by value, which is to say, -// when we're using row-oriented formats (set, mutex, time quantum). -// For int/decimal/timestamp fields, we're sorting by record only. -// So, if keys is the same as f.RecordIDs, we're looking at an int field -// or equivalent, so Signed exists and Values doesn't exist. -// Otherwise, we might be looking at a time quantum field (both exist) -// or set/mutex (only Values exist). -func simpleSort(f *FieldOperation, keys []uint64) { - // keys might actually just point to record IDs, in which case, we don't - // want to shuffle the corresponding RecordIDs too, because that would just - // reverse our swaps. If they're different, we actually need to swap them - // both. - if &keys[0] != &f.RecordIDs[0] { - // sorting by record IDs - if f.Values != nil && f.Signed != nil { // time quantum field - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - - } - } - } else if f.Values != nil { // set/mutex/bool - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - } - } - } else if f.Signed != nil { // can't-happen, we think - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else { // can't-happen, we think - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - } - } - } - } else { - if f.Values == nil && f.Signed != nil { - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else if f.Values != nil && f.Signed != nil { // can't happen, we think - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else if f.Values != nil { // only happens during testing - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - } - } - } else { // should definitely not happen - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - } - } - } - } -} - -func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) { - if shift < 0 { - shift = 0 - } - externalKeys := &f.RecordIDs[0] != &keys[0] - nextShift := shift - 8 - if nextShift < 0 { - nextShift = 0 - } - // count things that belong in each of the 256 buckets - var buckets [256]int - var starts [256]int - // compute the buckets ourselves - for _, r := range keys { - b := (r >> shift) & 0xFF - buckets[b]++ - } - total := 0 - // compute starting points of each bucket, converting the - // bucket counts into ends - for i := range buckets { - starts[i] = total - total += buckets[i] - buckets[i] = total - } - // starts[n] is the index of the first thing that should - // go in that bucket, buckets[n] is the index of the first - // thing that shouldn't - // var newbuckets [256]int - var bucketOp FieldOperation - for bucket, start := range starts { - end := buckets[bucket] - if end <= start { - continue - } - for j := start; j < end; j++ { - want := int((keys[j] >> shift) & 0xFF) - for want != bucket { - // move this to the beginning of the - // bucket it wants to be in, swapping - // the thing there here - dst := starts[want] - keys[j], keys[dst] = keys[dst], keys[j] - // we do this to allow you to just pass in the records as keys - if externalKeys { - f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j] - } - if f.Values != nil { - f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j] - } - if f.Signed != nil { - f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j] - } - starts[want]++ - want = int((keys[j] >> shift) & 0xFF) - } - } - // If shift == shardwidth.Exponent, then this is a completed - // shard and can go into the sharded output. otherwise, we - // can subdivide it. - if shift > 0 { - bucketOp.RecordIDs = f.RecordIDs[start:end] - if f.Values != nil { - bucketOp.Values = f.Values[start:end] - } - if f.Signed != nil { - bucketOp.Signed = f.Signed[start:end] - } - // if there's not very many, sort naively instead - if end-start > 32 { - sortPartialByKeys(&bucketOp, keys[start:end], nextShift) - } else { - simpleSort(&bucketOp, keys[start:end]) - } - } - } -} - -// AddPair adds a record ID/value pair where the value is unsigned, as -// when used with set/mutex/time quantum fields. -func (f *FieldOperation) AddPair(rec uint64, value uint64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Values = append(f.Values, value) -} - -// AddSignedPair adds a record ID/value pair where the value is signed, -// as when used with int/decimal/timestamp fields. -func (f *FieldOperation) AddSignedPair(rec uint64, value int64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Signed = append(f.Signed, value) -} - -// AddStampedPair adds a record/value pair plus a time, which is just -// a Unix time in seconds. (Note, no scaling here; timestamp fields are -// scaled int fields, this is for time quantums.) -func (f *FieldOperation) AddStampedPair(rec uint64, value uint64, stamp int64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Values = append(f.Values, value) - f.Signed = append(f.Signed, stamp) -} - -// Compare returns a diagnostic if the field operations do not seem -// equivalent. -func (got *FieldOperation) Compare(expected *FieldOperation) error { - if got == nil { - if expected == nil { - return nil - } - // We don't worry about non-empty Values or Signed here, because in theory - // RecordIDs are the Source of Truth as to what's in the op. - if len(expected.RecordIDs) == 0 { - return nil - } - return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs)) - } - if expected == nil { - if len(got.RecordIDs) == 0 { - return nil - } - return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs)) - } - if len(got.RecordIDs) != len(expected.RecordIDs) { - return fmt.Errorf("record counts differ: expected %d, got %d", len(expected.RecordIDs), len(got.RecordIDs)) - } - for i, v1 := range got.RecordIDs { - v2 := expected.RecordIDs[i] - if v1 != v2 { - return fmt.Errorf("record id %d differs: expected %d, got %d", i, v2, v1) - } - } - if len(got.Values) != len(expected.Values) { - return fmt.Errorf("value counts differ: expected %d, got %d", len(expected.Values), len(got.Values)) - } - for i, v1 := range got.Values { - v2 := expected.Values[i] - if v1 != v2 { - return fmt.Errorf("value %d differs: expected %d, got %d", i, v2, v1) - } - } - if len(got.Signed) != len(expected.Signed) { - return fmt.Errorf("signed value counts differ: expected %d, got %d", len(expected.Signed), len(got.Signed)) - } - for i, v1 := range got.Signed { - v2 := expected.Signed[i] - if v1 != v2 { - return fmt.Errorf("signed value %d differs: expected %d, got %d", i, v2, v1) - } - } - return nil -} - -// ShardOperations is a set of Operations associated with a specific shard. -type ShardOperations struct { - Shard uint64 - Ops []*Operation -} - -// Request is a complete ingest request, which may be any combination -// of operations, which may apply to multiple shards. -type Request struct { - Ops []*Operation -} - -// ShardedRequest is an ingest request, split up into individual per-shard -// operations. -type ShardedRequest struct { - Ops map[uint64][]*Operation -} - -// ByShard converts a request into the same request, only sharded. -func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) { - if len(r.Ops) == 0 { - return &ShardedRequest{Ops: nil}, nil - } - req := make(map[uint64][]*Operation) - shards := make(map[uint64]*Operation) - // we're getting per-field things, which we want to divide per-shard, - // and return to per-shard sets of per-field things, so we're inverting - // the structure. - for _, op := range r.Ops { - // for clear and write ops, we also need to split up the - // ClearRecords values, which may be distinct from the set of - // records for any given field. For Write ops, we'll then end - // up adding in field values for some fields. - if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { - sharded := ShardIDs(op.ClearRecordIDs) - for shard, data := range sharded { - shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} - } - } - for field, fieldOp := range op.FieldOps { - sharded := fieldOp.ByShard() - sorter := fieldTypeSorts[fields[field]] - if sorter == nil { - sorter = (*FieldOperation).SortByRecords - } - for shard, data := range sharded { - sorter(data) - shardOp, ok := shards[shard] - if !ok { - if op.OpType == OpWrite { - return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard) - } - shardOp = &Operation{OpType: op.OpType, Seq: op.Seq} - shards[shard] = shardOp - shardOp.FieldOps = map[string]*FieldOperation{field: data} - } else { - shardOp.FieldOps[field] = data - } - } - } - for shard, shardOp := range shards { - req[shard] = append(req[shard], shardOp) - } - for k := range shards { - delete(shards, k) - } - - } - return &ShardedRequest{Ops: req}, nil -} - -// merge combines the components of a sharded request back into a single -// unsharded request, processing shards in numerical order. -func (s *ShardedRequest) merge() *Request { - req := &Request{} - if s == nil || len(s.Ops) == 0 { - return req - } - shards := make([]uint64, 0, len(s.Ops)) - for shard := range s.Ops { - shards = append(shards, shard) - } - sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] }) - for _, shard := range shards { - ops := s.Ops[shard] - for _, op := range ops { - var _ *Operation - if op.Seq >= len(req.Ops) { - // Pad out with nil *Operations to the required length - req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...) - } - if req.Ops[op.Seq] == nil { - req.Ops[op.Seq] = op.clone() - continue - } - req.Ops[op.Seq].merge(op) - } - } - return req -} - -func (r *Request) Dump(logf func(string, ...interface{})) { - logf("req: %#v", r) - for _, op := range r.Ops { - logf("op: %#v", op) - if len(op.ClearRecordIDs) > 0 { - if len(op.ClearRecordIDs) > 8 { - logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8) - } else { - logf(" clearRecordIDs: %d", op.ClearRecordIDs) - } - } - if len(op.ClearFields) > 0 { - if len(op.ClearFields) > 8 { - logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8) - } else { - logf(" clearFields: %s", op.ClearFields) - } - } - for field, fieldOp := range op.FieldOps { - if fieldOp != nil { - logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed)) - if len(fieldOp.RecordIDs) > 0 { - if len(fieldOp.RecordIDs) > 8 { - logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8) - } else { - logf(" records %d", fieldOp.RecordIDs) - } - } - } else { - logf(" field %q: nil op", field) - } - } - } -} - -func (r *Request) Compare(other *Request) error { - if other == nil { - if r != nil && len(r.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - if r == nil { - if other != nil && len(other.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - ops := r.Ops - ops2 := other.Ops - if len(ops2) != len(ops) { - return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2)) - } - for i, op := range ops { - if err := op.Compare(ops2[i]); err != nil { - return fmt.Errorf("op %d: %v", i, err) - } - } - return nil -} - -// Compare checks whether two ShardedRequest objects represent the same -// data. Empty shards shouldn't have entries in the map in the first place, -// so we don't accept a nil or 0-length slice of ops as equal to the -// shard key not existing, but we do accept nil or empty requests as -// equal to each other. -func (s *ShardedRequest) Compare(other *ShardedRequest) error { - if other == nil { - if s != nil && len(s.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - if s == nil { - if other != nil && len(other.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - for shard, ops := range s.Ops { - ops2, ok := other.Ops[shard] - if !ok { - return fmt.Errorf("shard %d missing in other", shard) - } - if len(ops2) != len(ops) { - return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2)) - } - for i, op := range ops { - if err := op.Compare(ops2[i]); err != nil { - return fmt.Errorf("shard %d, op %d: %v", shard, i, err) - } - } - } - if len(other.Ops) != len(s.Ops) { - for shard := range other.Ops { - if _, ok := s.Ops[shard]; !ok { - return fmt.Errorf("shard %d missing in self", shard) - } - } - } - return nil -} diff --git a/ingest/op_test.go b/ingest/op_test.go deleted file mode 100644 index 31e1c56ff..000000000 --- a/ingest/op_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "math/rand" - "testing" - - "github.com/molecula/featurebase/v3/shardwidth" -) - -type opShardingTestCase struct { - name string - input *Request - output *ShardedRequest -} - -var opShardingTestCases = []opShardingTestCase{ - { - name: "sample", - input: &Request{ - Ops: []*Operation{ - { - OpType: OpSet, - FieldOps: map[string]*FieldOperation{ - "shard0": { - RecordIDs: []uint64{0, 1}, - }, - "shard0-1": { - RecordIDs: []uint64{ - 0, - 1 << shardwidth.Exponent, - }, - }, - "shard1": { - RecordIDs: []uint64{ - 1 << shardwidth.Exponent, - 1< 1 { - valuesPerRecord-- - } - } - } - } - op.FieldOps["set"] = &FieldOperation{ - RecordIDs: recordIDs, - Values: values, - } - fieldTypes := codec.FieldTypes() - sharded, err := req.ByShard(fieldTypes) - if err != nil { - t.Errorf("sharding: unexpected error %v", err) - } - merged := sharded.merge() - if err := req.Compare(merged); err != nil { - t.Fatalf("merge comparison: %v", err) - } -} - -func TestFancySharding(t *testing.T) { - const shardLimit = 700 - const recordCount = 5000 - grr := rand.New(rand.NewSource(0)) - for i := 0; i < 100; i++ { - f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} - shards := make([]int, shardLimit) - for j := range f.RecordIDs { - v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent)) - f.RecordIDs[j] = v - f.Values[j] = uint64(grr.Int63n(8)) - shards[v>>shardwidth.Exponent]++ - } - - sharded := f.SortToShards() - for shard, data := range sharded { - if len(data.RecordIDs) != shards[shard] { - t.Errorf("shard %d: expected %d items, got %d", shard, shards[shard], len(data.RecordIDs)) - } - for _, v := range data.RecordIDs { - if (v >> shardwidth.Exponent) != shard { - t.Errorf("shard %d: got %x, which should be in %d", shard, v, v>>shardwidth.Exponent) - } - } - // expect sorted-ness - data.SortByRecords() - prev := data.RecordIDs[0] - for i, next := range data.RecordIDs[1:] { - if next < prev { - t.Errorf("index %d: prev %d, next %d", i+1, prev, next) - } - prev = next - } - data.SortByValues() - prevV, prevRec := data.Values[0], data.RecordIDs[0] - for i, nextRec := range data.RecordIDs[1:] { - nextV := data.Values[i+1] - if nextV < prevV { - t.Errorf("index %d: prev value %d, next value %d", i+1, prevV, nextV) - } - if nextV == prevV { - if nextRec < prevRec { - t.Errorf("index %d, value %d: prev rec %d, next rec %d", i+1, nextV, prevRec, nextRec) - } - } - prevV = nextV - prevRec = nextRec - } - } - } -} diff --git a/ingest/shard.go b/ingest/shard.go deleted file mode 100644 index 788623bdb..000000000 --- a/ingest/shard.go +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest diff --git a/ingest/sort.go b/ingest/sort.go deleted file mode 100644 index 7020c841f..000000000 --- a/ingest/sort.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -// "math/bits" - -// HERE THERE BE DRAGONS - -// This is some sorting logic Nia was experimenting with, which we aren't currently -// using, but which beat stdlib sort by a factor-of-several on at least some test -// data, so we aren't deleting it just yet. - -// groupIDPairsByShard destructively groups ID pairs by shard. -// The returned slices reference the original pairs slice. -// func groupIDPairsByShard(pairs []IDPair) map[uint64][]IDPair { -// if len(pairs) == 0 { -// return nil -// } -// -// // Sort pairs by shard (in-place radix sort). -// // This may also change the order of pairs within a shard, but that should not matter. -// for { -// // Find the highest bit which needs to be sorted. -// var diffMask uint64 -// prev := pairs[0].RecordID -// for _, v := range pairs[1:] { -// if v.RecordID < prev { -// diffMask |= v.RecordID ^ prev -// } -// -// prev = v.RecordID -// } -// diffLen := bits.Len64(diffMask) -// if diffLen <= shardwidth.Exponent { -// // The pairs are sorted by shard. -// break -// } -// -// // Select a right bit shift index such that the highest unsorted bit moves to the 128's place. -// shift := uint(diffLen) - 8 -// -// // Create a mask that can be used to group values by sorted bits. -// sortedMask := ^uint64(0) << bits.Len64(diffMask) -// -// for i := 0; i < len(pairs); { -// // Select a group of pairs to sort. -// // While doing so, count the pairs within each bucket. -// j := i -// var buckets [256]struct { -// start, end uint -// } -// for group := pairs[i].RecordID & sortedMask; i < len(pairs) && pairs[i].RecordID&sortedMask == group; i++ { -// buckets[uint8(pairs[i].RecordID>>uint64(shift))].end++ -// } -// group := pairs[j:i] -// -// // Assign indices within the group to the buckets. -// { -// var start uint -// for i := range buckets { -// bucket := &buckets[i] -// bucket.start = start -// bucket.end += start -// start = bucket.end -// } -// } -// -// // Split the group into the buckets. -// for i, b := range buckets { -// // There is no need to update the state of the current bucket - we will never reference it again after this. -// i := uint8(i) -// for j := b.start; j < b.end; j++ { -// // This inner loop may run quite a few times for the first few buckets, but no element will be moved more than twice per byte. -// for uint8(group[j].RecordID>>shift) != i { -// // This pair is in the wrong bucket. -// // Swap it into the correct bucket. -// dstBucket := &buckets[uint8(group[j].RecordID>>shift)] -// k := dstBucket.start -// dstBucket.start++ -// group[j], group[k] = group[k], group[j] -// } -// } -// } -// } -// } -// -// // Split the pairs by shard. -// shards := make(map[uint64][]IDPair) -// for i := 0; i < len(pairs); { -// // Select the shard. -// shard := pairs[i].RecordID >> shardwidth.Exponent -// -// // Find all pairs in the shard. -// j := i -// incr := 1 -// for i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard { -// i += incr -// incr *= 2 -// } -// for ; incr > 0; incr /= 2 { -// if i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard { -// i += incr -// } -// } -// // that found us the last thing in this shard, so... -// i++ -// -// // Add the shard, referencing the original slice. -// // This sets the cap so that we dont accidentally overwrite other shards data. -// shards[shard] = pairs[j:i:i] -// } -// -// return shards -// } - -// pairsToZigZag converts a set of record-value pairs to Pilosa's zig-zag format. -// This assumes that all pairs are within the same shard. -// func pairsToZigZag(pairs []IDPair) []uint64 { -// const recordMask = (1 << shardwidth.Exponent) - 1 -// -// dst := make([]uint64, len(pairs)) -// for i, p := range pairs { -// dst[i] = (p.ID << shardwidth.Exponent) | (p.RecordID & recordMask) -// } -// -// return dst -// } - -// radixSort64 sorts the data with radix-sort. -// The "shift" is the highest differing bit position, rounded down to a multiple of 8. -// If there are duplicates, this may change the duplicate count for some values. -// func radixSort64(data []uint64, shift uint) { -// if len(data) < 2 { -// return -// } -// if shift <= 8 { -// // The data falls into a 16-bit span, so the remaining digits can be sorted simultaneously with a bitmask. -// maskSort(data) -// return -// } -// -// // Count the values within each bucket. -// var buckets [256]struct { -// start, end uint -// } -// for _, v := range data { -// buckets[uint8(v>>shift)].end++ -// } -// -// // Assign indices within the group to the buckets. -// { -// var start uint -// for i := range buckets { -// bucket := &buckets[i] -// bucket.start = start -// bucket.end += start -// start = bucket.end -// } -// } -// -// // Split the data into the buckets. -// var start uint -// for i, b := range buckets { -// // Replace misplaced values until the contents of the bucket all have the correct digit. -// i := uint8(i) -// for j := b.start; j < b.end; j++ { -// // This inner loop may run quite a few times for the first few buckets, but it will never run more than once-per-element-per-byte. -// for uint8(data[j]>>shift) != i { -// // This pair is in the wrong bucket. -// // Swap it into the correct bucket. -// dstBucket := &buckets[uint8(data[j]>>shift)] -// k := dstBucket.start -// dstBucket.start++ -// data[j], data[k] = data[k], data[j] -// } -// } -// -// // Sort the contents of the bucket. -// data := data[start:b.end] -// switch { -// case len(data) < 64: -// // Use insertion-sort because the data is too small for a more complex algorithm to be efficient. -// for i := 0; i < len(data); i++ { -// for j := i; j > 0 && data[j-1] > data[j]; j-- { -// data[j-1], data[j] = data[j], data[j-1] -// } -// } -// -// default: -// // Sort the next byte recursively. -// radixSort64(data, shift-8) -// } -// start = b.end -// } -// } - -// maskSort sorts integers using a bitmask. -// The values must all fall within one 16-bit span. -// If there are duplicates, this may change the duplicate count for some values. -// func maskSort(data []uint64) { -// if len(data) < 2 { -// return -// } -// -// base := data[0] &^ ((1 << 16) - 1) -// -// // Dump everything into the mask. -// var mask [(1 << 16) / 64]uint64 -// for _, v := range data { -// mask[uint16(v)/64] |= 1 << (v % 64) -// } -// -// // Scan through the set bits in the mask. -// k := 0 -// for i, w := range mask { -// for w != 0 { -// j := bits.TrailingZeros64(w) -// w &^= 1 << j -// data[k] = 64*uint64(i) + uint64(j) + base -// k++ -// } -// } -// -// if k < len(data) { -// // Copy the ending value to fill up the rest of the space. -// // This happens once for each duplicate value. -// endVal := data[k-1] -// for k < len(data) { -// data[k] = endVal -// k++ -// } -// } -// } diff --git a/ingest/sort_test.go b/ingest/sort_test.go deleted file mode 100644 index 017ffa60f..000000000 --- a/ingest/sort_test.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "testing" -) - -const sampleSize = 1000000 - -var sampleSortingData = createSampleFieldData() - -func createSampleFieldData() *FieldOperation { - fo := &FieldOperation{} - fo.RecordIDs = make([]uint64, sampleSize) - fo.Values = make([]uint64, sampleSize) - fo.Signed = make([]int64, sampleSize) - for i := range fo.RecordIDs { - fo.RecordIDs[i] = (uint64(i) * 63 * 3456789) % 50000000 - fo.Values[i] = (uint64(i) * 6) % 8 - fo.Signed[i] = ((int64(i) * 17) % 15) - 8 - } - return fo -} - -func benchmarkOneSort(b *testing.B, fo *FieldOperation) { - for i := 0; i < b.N; i++ { - b.StopTimer() - sortable := fo.clone() - b.StartTimer() - _ = sortable.SortToShards() - } -} - -func BenchmarkSortFieldOp(b *testing.B) { - b.Run("full", func(b *testing.B) { - f2 := *sampleSortingData - benchmarkOneSort(b, &f2) - }) - b.Run("nosign", func(b *testing.B) { - f2 := *sampleSortingData - f2.Signed = nil - benchmarkOneSort(b, &f2) - }) - b.Run("signonly", func(b *testing.B) { - f2 := *sampleSortingData - f2.Values = nil - benchmarkOneSort(b, &f2) - }) -} - -// func BenchmarkSort64(b *testing.B) { -// gen := func(n, width uint64) func() []uint64 { -// var data []uint64 -// var once sync.Once -// return func() []uint64 { -// once.Do(func() { -// data = make([]uint64, n) -// var rng rand.PCGSource -// rng.Seed(9001) -// for i := range data { -// data[i] = rng.Uint64() % width -// } -// }) -// -// return data -// } -// } -// -// algos := []struct { -// name string -// maxn uint64 -// fn func([]uint64) []uint64 -// }{ -// { -// name: "stdlib", -// maxn: 1024 * 1024 * 1024, -// fn: stdSort, -// }, -// { -// name: "heap", -// maxn: 1024 * 1024 * 1024, -// fn: heapSort, -// }, -// { -// name: "radix-insertion-mask", -// maxn: 1024 * 1024 * 1024, -// fn: dedupSort64, -// }, -// } -// -// widths := []struct { -// name string -// width uint64 -// }{ -// {"64", 64}, -// {"1K", 1024}, -// {"64K", 64 * 1024}, -// {"1M", 1024 * 1024}, -// {"16M", 16 * 1024 * 1024}, -// {"128M", 128 * 1024 * 1024}, -// {"1B", 1024 * 1024 * 1024}, -// } -// -// counts := []struct { -// name string -// n uint64 -// }{ -// {"64", 64}, -// {"1K", 1024}, -// {"64K", 64 * 1024}, -// {"1M", 1024 * 1024}, -// {"4M", 4 * 1024 * 1024}, -// {"16M", 16 * 1024 * 1024}, -// {"64M", 64 * 1024 * 1024}, -// {"256M", 256 * 1024 * 1024}, -// } -// -// for _, width := range widths { -// width := width -// b.Run(width.name, func(b *testing.B) { -// for _, count := range counts { -// if count.n > width.width { -// continue -// } -// -// count := count -// b.Run(count.name, func(b *testing.B) { -// datasrc := gen(count.n, width.width) -// for _, alg := range algos { -// if count.n > alg.maxn { -// continue -// } -// -// alg := alg -// b.Run(alg.name, func(b *testing.B) { -// data := datasrc() -// buf := make([]uint64, len(data)) -// b.SetBytes(8 * int64(len(buf))) -// -// b.StopTimer() -// b.ResetTimer() -// -// for i := 0; i < b.N; i++ { -// copy(buf, data) -// b.StartTimer() -// alg.fn(buf) -// b.StopTimer() -// } -// }) -// } -// }) -// } -// }) -// } -// } -// -// func heapSort(data []uint64) []uint64 { -// for i, v := range data { -// for i > 0 && v > data[(i-1)/2] { -// data[i] = data[(i-1)/2] -// i = (i - 1) / 2 -// } -// data[i] = v -// } -// { -// heap := data -// for len(heap) > 1 { -// heap[0], heap[len(heap)-1] = heap[len(heap)-1], heap[0] -// heap = heap[:len(heap)-1] -// i := 0 -// for { -// max := i -// if r := 2*i + 1; r < len(heap) && heap[r] > heap[max] { -// max = r -// } -// if l := 2*i + 2; l < len(heap) && heap[l] > heap[max] { -// max = l -// } -// if max == i { -// break -// } -// -// heap[max], heap[i] = heap[i], heap[max] -// i = max -// } -// } -// } -// -// j := 1 -// prev := data[0] -// for _, v := range data[1:] { -// if v == prev { -// continue -// } -// -// data[j] = v -// prev = v -// } -// -// return data[:j] -// } -// -// func stdSort(data []uint64) []uint64 { -// sort.Slice(data, func(i, j int) bool { return data[i] < data[j] }) -// -// j := 1 -// prev := data[0] -// for _, v := range data[1:] { -// if v == prev { -// continue -// } -// -// data[j] = v -// prev = v -// } -// -// return data[:j] -// } diff --git a/ingest/translate_test.go b/ingest/translate_test.go deleted file mode 100644 index f84265cba..000000000 --- a/ingest/translate_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "testing" -) - -// stableTranslator implements a key translator that can be reused and -// will continue to give the same keys for the same values. Possibly -// surprisingly, it will invent new keys for IDs it is asked about but -// hasn't seen. This allows us to give a codec which would use keys on -// translation a request which contains arbitrary numbers, and request -// text that would parse into that request. -type stableTranslator struct { - in map[string]uint64 - out map[uint64]string - next uint64 -} - -func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - ret := make(map[string]uint64, len(keys)) - for _, key := range keys { - if existing, ok := s.in[key]; ok { - ret[key] = existing - continue - } - id := s.next - // but what if someone already translated that ID, so now it already - // exists? - if _, ok := s.out[id]; ok { - for k := range s.out { - if k > id { - id = k - } - } - // one larger than the largest we already have. this could - // wrap around, in which case, it's your own fault. - id++ - } - s.next = id + 1 - s.in[key] = id - s.out[id] = key - ret[key] = id - } - return ret, nil -} - -func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - ret := make(map[uint64]string, len(ids)) - for _, id := range ids { - if existing, ok := s.out[id]; ok { - ret[id] = existing - continue - } - key := fmt.Sprintf("k-%d", id) - s.in[key] = id - s.out[id] = key - ret[id] = key - if id >= s.next { - s.next = id + 1 - } - } - return ret, nil -} - -// newStableTranslator produces a translator which can translate forwards -// and backwards and invent new things if it needs to. Don't use this. -func newStableTranslator() *stableTranslator { - return &stableTranslator{ - in: make(map[string]uint64), - out: make(map[uint64]string), - } -} - -func TestTranslateReuse(t *testing.T) { - // the original stable-translator design had a flaw in that it - // assumed that each new ID would always come from a string translation, - // never from a key translation, and that they'd show up sequentially. - tr := newStableTranslator() - orig, err := tr.TranslateIDs(1, 2) - tr.next = 1 // intentionally break the translation logic for testing purposes - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - var s [6]string - stash := s[:0] - for _, v := range orig { - stash = append(stash, v) - } - for i := range s[len(orig):] { - stash = append(stash, fmt.Sprintf("key-%d", i)) - } - keys, err := tr.TranslateKeys(stash...) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ids := make([]uint64, 0, len(keys)) - for _, v := range keys { - ids = append(ids, v) - } - idMap, err := tr.TranslateIDs(ids...) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - for k, v := range keys { - if idMap[v] != k { - t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q", - k, v, v, idMap[v]) - } - } - for id, key := range idMap { - if keys[key] != id { - t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d", - id, key, key, keys[key]) - } - } - for id, key := range orig { - if keys[key] != id { - t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d", - id, key, key, keys[key]) - } - } -} diff --git a/ingest/update.go b/ingest/update.go deleted file mode 100644 index f36363ba7..000000000 --- a/ingest/update.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "github.com/molecula/featurebase/v3/roaring" -) - -// ShardUpdate is an update request for a shard. -type ShardUpdate struct { - // TODO: include schema version - - // Sets are the updates to set fields. - Sets map[string]SetMatrixUpdate - - // Mutexes are the updates to mutex fields. - Mutexes map[string]MutexMatrixUpdate - - // TimeTensors are the updates to time fields. - TimeTensors map[string]TimeTensorUpdate - - // Ints are the updates to int fields. - Ints map[string]IntUpdate -} - -// Convert the ID vector to a raw update that can be imported. -// All records are assumed to fall within a shard. -// func (vec IDVector) Convert() (MutexMatrixUpdate, error) { -// // Before converting the vector, do a sanity-check for duplicates. -// dedup := make(map[uint64]struct{}, len(vec.Updates)+len(vec.Clears)) -// for _, p := range vec.Updates { -// if _, ok := dedup[p.RecordID]; !ok { -// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates") -// } -// -// dedup[p.RecordID] = struct{}{} -// } -// for _, p := range vec.Clears { -// if _, ok := dedup[p]; !ok { -// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates") -// } -// -// dedup[p] = struct{}{} -// } -// -// // Convert the updates to a bitmap. -// updates, err := idPairsToBitmap(vec.Updates, false) -// if err != nil { -// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex updates") -// } -// -// // Convert the clears to a bitmap. -// clears, err := idListToBitmap(vec.Clears, false) -// if err != nil { -// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex clears") -// } -// -// // Realign clears bitmap to start of shard. -// if min, ok := clears.Min(); ok { -// min &^= (1 << shardwidth.Exponent) - 1 -// clears = clears.OffsetRange(-min, 0, 1< 0 { -// return SetMatrixUpdate{}, errors.New("set adds and removes overlap") -// } -// -// return SetMatrixUpdate{ -// Add: adds, -// Remove: removes, -// Clear: clears, -// }, nil -// } - -// SetMatrixUpdate is an encoded update request for a set-type (set/mutex) view. -type SetMatrixUpdate struct { - // Add is a bitmap to union the matrix against. - Add *roaring.Bitmap - - // Remove is a bitmap to difference out of the matrix. - // No values will be present in both add and remove. - // This bitmap will not include any records in the clear bitmap. - Remove *roaring.Bitmap - - // Clear is a set of records for which all values should be removed. - // This is applied before add operations. - Clear *roaring.Bitmap -} - -// func idPairsToBitmap(pairs []IDPair, allowDup bool) (*roaring.Bitmap, error) { -// return idListToBitmap(pairsToZigZag(pairs), allowDup) -// } -// -// func idListToBitmap(ids []uint64, allowDup bool) (*roaring.Bitmap, error) { -// vals := dedupSort64(ids) -// if len(vals) != len(ids) && !allowDup { -// return nil, errors.New("input contains duplicate values") -// } -// -// // TODO: make the roaring package actually handle this well -// return roaring.NewBitmap(vals...), nil -// } - -// Convert the int vector to a raw update that can be imported. -// All records are assumed to fall within a shard. -// func (vec IntVector) Convert() (IntUpdate, error) { -// // Convert the clears to a bitmap. -// clears, err := idListToBitmap(vec.Clears, false) -// if err != nil { -// return IntUpdate{}, errors.Wrap(err, "encoding int clears") -// } -// -// // Realign clears bitmap to start of shard. -// if min, ok := clears.Min(); ok { -// min &^= (1 << shardwidth.Exponent) - 1 -// clears = clears.OffsetRange(-min, 0, 1< 0 { -// return IntUpdate{}, errors.New("update duplicated with a clear") -// } -// -// return IntUpdate{ -// BSI: updates, -// Clear: clears, -// }, nil -// } - -// IntUpdate is an encoded update request for a BSI (int/timestamp/etc.) view. -type IntUpdate struct { - // BSI is a bitmap of new BSI data to overwrite existing values. - BSI *roaring.Bitmap - - // Clear is a set of records to assign null. - Clear *roaring.Bitmap -} diff --git a/ingest/vec.go b/ingest/vec.go deleted file mode 100644 index 94d6477f3..000000000 --- a/ingest/vec.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "reflect" - "strconv" - "unsafe" -) - -// StringTable is a mapping of strings to temporary IDs. -// All mapped-to IDs fall in the range [0, len). The zero value, -// a nil map, instead just parses the numbers. -// -// We keep the array of names in creation order because we want reproducibility; -// the first key we see is always key 0. Otherwise, the keys are created in -// arbitrary orders. -type StringTable struct { - names []string - values map[string]uint64 -} - -// NewStringTable just creates a string table with a non-nil map. -func NewStringTable() *StringTable { - return &StringTable{values: map[string]uint64{}} -} - -// unsafe is -func pretendByteIsString(data []byte) (result string) { - dH := (*reflect.SliceHeader)(unsafe.Pointer(&data)) - sH := (*reflect.StringHeader)(unsafe.Pointer(&result)) - sH.Data = dH.Data - sH.Len = dH.Len - return result -} - -// ID returns an ID associated to the string, adding it to the table if it is not already present, -// or parsing an integer if there's no table. -func (tbl *StringTable) ID(in []byte) (uint64, error) { - str := pretendByteIsString(in) - if tbl != nil { - id, ok := tbl.values[str] - if !ok { - id = uint64(len(tbl.values)) - tbl.values[str] = id - tbl.names = append(tbl.names, str) - } - return id, nil - } - return strconv.ParseUint(str, 10, 64) -} - -// SignedID returns an ID associated to the string, adding it to the table if it is not already present, -// or parsing an integer if there's no table. It yields signed values only. -func (tbl *StringTable) IntID(in []byte) (int64, error) { - str := pretendByteIsString(in) - if tbl != nil { - id, ok := tbl.values[str] - if !ok { - id = uint64(len(tbl.values)) - tbl.values[str] = id - tbl.names = append(tbl.names, str) - } - return int64(id), nil - } - return strconv.ParseInt(str, 10, 64) -} - -// MapForStringTable, given a string table mapping strings to consecutive -// integers and a translation function from strings to "real" keys, yields -// a translation/lookup slice. If it cannot translate all the keys, it -// returns an error. -func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) { - lookedUp, err := keys.TranslateKeys(tbl.names...) - if err != nil { - return nil, err - } - if len(lookedUp) != len(tbl.names) { - return nil, fmt.Errorf("missing keys: expected %d keys, got %d", len(tbl.values), len(lookedUp)) - } - out := make([]uint64, len(tbl.names)) - for i, v := range tbl.names { - out[i] = lookedUp[v] - } - return out, nil -} - -// translateSigned replaces values from 0 to len(mapping)-1 with the -// elements of mapping. It yields an error if any values aren't -// mapped. -func translateSigned(mapping []uint64, values []int64) error { - oops := 0 - for i, v := range values { - if v >= int64(len(mapping)) { - oops++ - } else { - values[i] = int64(mapping[v]) - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} - -// translateUnsigned replaces values from 0 to len(mapping)-1 with the -// elements of mapping. It yields an error if any values aren't -// mapped. -func translateUnsigned(mapping []uint64, values []uint64) error { - oops := 0 - for i, v := range values { - if v >= uint64(len(mapping)) { - oops++ - } else { - values[i] = mapping[v] - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} diff --git a/ingest/vec_test.go b/ingest/vec_test.go deleted file mode 100644 index 9b5947272..000000000 --- a/ingest/vec_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "errors" - "testing" -) - -type badTranslator struct{} - -func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - if len(keys) == 0 { - return nil, errors.New("no keys") - } - m := make(map[string]uint64) - skip := true - for i, k := range keys { - if skip { - skip = false - continue - } - m[k] = uint64(i) - } - out := make([]uint64, len(keys)-1) - for i := range out { - out[i] = uint64(i) - } - return nil, nil -} - -func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) { - return nil, nil -} - -func TestStringTableErrors(t *testing.T) { - tbl := NewStringTable() - btr := badTranslator{} - _, keyErr := tbl.MakeIDMap(btr) - if keyErr == nil { - t.Fatalf("expected error passed up from failed translate, didn't get it") - } - a1, err := tbl.ID([]byte("a")) - if err != nil { - t.Fatalf("getting translation for key: %v", err) - } - b1, err := tbl.ID([]byte("b")) - if err != nil { - t.Fatalf("getting translation for key: %v", err) - } - _, keyErr = tbl.MakeIDMap(btr) - if keyErr == nil { - t.Fatalf("expected error for short translate, didn't get it") - } - tr := newStableTranslator() - _, err = tr.TranslateKeys("c", "d") - if err != nil { - t.Fatalf("translating stray keys: %v", err) - } - m, err := tbl.MakeIDMap(tr) - if err != nil { - t.Fatalf("creating lookup: %v", err) - } - var y = []uint64{a1, b1} - err = translateUnsigned(m, y) - if err != nil { - t.Fatalf("unexpected unsigned translation error: %v", err) - } - trResults, err := tr.TranslateKeys("a", "b") - if err != nil { - t.Fatalf("unexpected translation error: %v", err) - } - if y[0] != trResults["a"] { - t.Fatalf("expected %d, got %d", trResults["a"], y[0]) - } - if y[1] != trResults["b"] { - t.Fatalf("expected %d, got %d", trResults["b"], y[1]) - } - y[0] = a1 - y[1] = (a1 + b1 + 1) // assumed not to be any of them - err = translateUnsigned(m, y) - if err == nil { - t.Fatalf("no error from translating invalid table") - } - z := []int64{int64(a1), int64(b1)} - err = translateSigned(m, z) - if err != nil { - t.Fatalf("unexpected unsigned translation error: %v", err) - } - if uint64(z[0]) != trResults["a"] { - t.Fatalf("expected %d, got %d", trResults["a"], z[0]) - } - if uint64(z[1]) != trResults["b"] { - t.Fatalf("expected %d, got %d", trResults["b"], z[1]) - } - z[0] = int64(a1) - z[1] = int64(a1 + b1 + 1) // assumed not to be any of them - err = translateSigned(m, z) - if err == nil { - t.Fatalf("no error from translating invalid table") - } -} diff --git a/ingest_test.go b/ingest_test.go deleted file mode 100644 index 341ab9d8c..000000000 --- a/ingest_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa_test - -import ( - "bytes" - "context" - "encoding/json" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/ingest" - "github.com/molecula/featurebase/v3/test" - "github.com/pkg/errors" -) - -// For ingest API testing, we want to do tests which have a known -// schema, no existing data before we start, and perform ingests and -// then do queries. -// -// A good starting point for this would be a fairly simple file -// divided into sections which are just the JSON text of the data -// we want to be working with, or the PQL queries we want to run, -// or their expected results. -// -// So, roughly like this: -// -// schema: -// { -// "index-name": "example", -// "primary-key-type": "string", -// "fields": [ -// { -// "field-name": "set", -// "field-type": "id", -// "field-options": { "cache-type": "none" } -// } -// ] -// } -// ingest: -// [ -// { -// "action": "set", -// "records": { -// "1": { -// "set": [ 2 ], -// } -// } -// } -// ] -// queries: -// Row(set=2): -// [1] -// -// Additionally, the names "schema-error" and "ingest-error" are taken to -// represent a schema, or data set, which is expected to produce an error. -// For instance: -// -// ingest-error: -// [ { "action": puppy } -// -// In this case, it would be considered a test failure if an ingest request -// did NOT fail. -// Lines starting with # - -// ingestSchemaPartial represents the only part of a schema we need to -// know about in order to undo its creation of a schema for use in a -// test case. -type ingestSchemaPartial struct { - IndexName string `json:"index-name"` -} - -type ingestActionKind int - -const ( - ingestActionNone = ingestActionKind(iota) - ingestActionSchema - ingestActionIngest - ingestActionSchemaError - ingestActionIngestError - ingestActionQueries -) - -var ingestActionKinds = map[string]ingestActionKind{ - "schema": ingestActionSchema, - "ingest": ingestActionIngest, - "schema-error": ingestActionSchemaError, - "ingest-error": ingestActionIngestError, - "queries": ingestActionQueries, -} -var ingestActionKindNames = map[ingestActionKind]string{} - -type testCaseAction struct { - kind ingestActionKind - comment []byte - lineStart int - lineEnd int - data []byte -} - -type liner struct { - data []byte - at int - start int - remaining []byte - line []byte -} - -func newLiner(data []byte) *liner { - return &liner{data: data, at: 0, remaining: data} -} - -func (l *liner) next() bool { - if len(l.remaining) == 0 { - return false - } - foundNL := true - nextNL := bytes.IndexByte(l.remaining, '\n') - if nextNL == -1 { - foundNL = false - nextNL = len(l.remaining) - } - l.line = l.remaining[:nextNL] - // move past the newline we found, if we found one - if foundNL { - nextNL++ - } - l.start, l.at = l.at, l.at+nextNL - l.remaining = l.data[l.at:] - return true -} - -func (l *liner) text() (line []byte, start int, end int) { - return l.line, l.start, l.at -} - -// parseExpectedResults handles something that looks like -// [1, 2, 3] or ["a", "b", "c"]. It does not handle things like -// quotes within strings, etcetera. -func parseExpectedResults(data []byte) (ints []uint64, keys []string, err error) { - if len(data) < 2 || data[0] != '[' || data[len(data)-1] != ']' { - return nil, nil, errors.New("expecting [] results") - } - words := bytes.Split(data[1:len(data)-1], []byte{','}) - if len(words) == 1 && len(words[0]) == 0 { - return nil, nil, nil - } - for _, word := range words { - word = bytes.TrimSpace(word) - if len(word) == 0 { - return nil, nil, errors.New("found empty word expecting result") - } - if word[0] == '"' { - keys = append(keys, string(word[1:len(word)-1])) - continue - } - v, err := strconv.ParseInt(string(word), 10, 64) - if err != nil { - return nil, nil, err - } - ints = append(ints, uint64(v)) - } - if len(ints) > 0 && len(keys) > 0 { - return nil, nil, errors.New("mixed integers and strings are invalid") - } - return ints, keys, err -} - -func testQueries(t *testing.T, ctx context.Context, cmd *test.Command, index string, action testCaseAction) { - qcx := cmd.API.Txf().NewQcx() - defer func() { - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - l := newLiner(action.data) - for l.next() { - query, _, _ := l.text() - if !l.next() { - t.Fatalf("processing query list: no expected after %q", query) - } - expected, _, _ := l.text() - ints, keys, err := parseExpectedResults(expected) - if err != nil { - t.Fatalf("processing query list: invalid expected results %q", expected) - } - t.Logf("expecting %q -> %s", query, expected) - res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: string(query)}) - if err != nil { - t.Errorf("query: %v", err) - } - if len(res.Results) != 1 { - t.Fatalf("expected one result per query, got %d results", len(res.Results)) - } - var row *pilosa.Row - var ok bool - if row, ok = res.Results[0].(*pilosa.Row); !ok { - t.Fatalf("expected results to be a row") - } - if ints != nil { - cols := row.Columns() - if len(cols) != len(ints) { - t.Fatalf("wrong number of values, expected %d, got %d", len(ints), len(cols)) - } - for i := range cols { - if ints[i] != cols[i] { - t.Fatalf("result %d: expected %d, got %d", i, ints[i], cols[i]) - } - } - } - // key return value is unpredictable, so... - if keys != nil { - seen := make(map[string]struct{}) - for _, k := range keys { - seen[k] = struct{}{} - } - for _, k := range row.Keys { - if _, ok := seen[k]; !ok { - t.Fatalf("unexpected result key %q", k) - } - delete(seen, k) - } - for k := range seen { - t.Fatalf("expected result to contain %q, but did not get it", k) - } - } - } -} - -// testOneIngestTestcase runs a set of actions, then cleans up after itself -func testOneIngestTestcase(t *testing.T, ctx context.Context, cmd *test.Command, tcpath string) { - data, err := os.ReadFile(tcpath) - if err != nil { - t.Fatalf("reading %q: %v", tcpath, err) - } - var actions []testCaseAction - var action testCaseAction - l := newLiner(data) - var line []byte - var start, lineStart, lineEnd int - lineCount := 0 - for l.next() { - line, lineStart, lineEnd = l.text() - lineCount++ - if colon := bytes.IndexByte(line, ':'); colon != -1 { - if kind, ok := ingestActionKinds[string(line[:colon])]; ok { - if action.kind != ingestActionNone { - action.data = data[start:lineStart] - actions = append(actions, action) - action.lineEnd = lineCount - 1 - } else { - if lineStart != 0 { - t.Logf("warning: %d bytes with no action type before first action", lineStart) - } - } - start = lineEnd - action.data = nil - action.kind = kind - if line[len(line)-1] == ':' { - action.comment = line[:len(line)-1] - } else { - action.comment = line - } - action.lineStart = lineCount - } - } - } - if action.kind != ingestActionNone { - action.lineEnd = lineCount - 1 - action.data = data[start:] - actions = append(actions, action) - } - var mostRecentIndex string - seenIndexes := map[string]struct{}{} - - cli := cmd.Client() - created := map[string][]string{} - defer func() { - t.Logf("deleting created indexes/fields:") - for k, v := range created { - if len(v) == 0 { - t.Logf(" index: %q", k) - if err := cmd.API.DeleteIndex(ctx, k); err != nil { - t.Errorf("deleting index %q: %v", k, err) - } - } else { - t.Logf(" fields in %q: %q", k, v) - for _, field := range v { - if err := cmd.API.DeleteField(ctx, k, field); err != nil { - t.Errorf("deleting field %q from %q: %v", field, k, err) - } - } - } - } - }() - - noticeCreation := func(newlyCreated map[string][]string) { - for k, v := range newlyCreated { - if len(v) == 0 { - if existing, ok := created[k]; ok { - if len(existing) > 0 { - t.Fatalf("creation reports index %q newly created, but we created fields %q in it previously", - k, existing) - } - } - // create an empty list, indicating that the whole index is - // believed nil - created[k] = nil - continue - } - if existing, ok := created[k]; ok { - if len(existing) == 0 { - // we'll delete this index anyway, don't need to delete fields in it - continue - } - created[k] = append(existing, v...) - continue - } - created[k] = v - } - } - - for _, action := range actions { - t.Logf("%s, lines %d-%d", action.comment, action.lineStart, action.lineEnd) - switch action.kind { - case ingestActionSchema: - var scratch ingestSchemaPartial - err = json.Unmarshal(action.data, &scratch) - if err != nil { - t.Fatalf("couldn't parse schema data: %v", err) - } - if scratch.IndexName == "" { - t.Fatalf("test case must provide an index name") - } - // stash the string from the schema, because we - // might need it later - mostRecentIndex = scratch.IndexName - seenIndexes[mostRecentIndex] = struct{}{} - var newlyCreated map[string][]string - newlyCreated, err = cli.IngestSchema(ctx, nil, action.data) - if err != nil { - t.Fatalf("executing schema: %v", err) - } - noticeCreation(newlyCreated) - case ingestActionSchemaError: - var scratch ingestSchemaPartial - err = json.Unmarshal(action.data, &scratch) - if err != nil { - t.Logf("got expected error from schema: %v", err) - break - } - var newlyCreated map[string][]string - newlyCreated, err = cli.IngestSchema(ctx, nil, action.data) - if err != nil { - t.Logf("got expected error from schema: %v", err) - break - } - noticeCreation(newlyCreated) - t.Fatalf("expected error from schema, didn't get it") - case ingestActionIngest: - func() { - qcx := cmd.API.Txf().NewQcx() - var err error - defer func() { - if err == nil { - qcx.Abort() - return - } - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data)) - if err != nil { - t.Fatalf("importing data: %v", err) - } - }() - case ingestActionIngestError: - func() { - qcx := cmd.API.Txf().NewQcx() - var err error - defer func() { - if err == nil { - qcx.Abort() - return - } - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data)) - if err != nil { - t.Logf("got expected error from ingest: %v", err) - return - } - t.Fatalf("expected error from ingest, didn't get it") - }() - case ingestActionQueries: - testQueries(t, ctx, cmd, mostRecentIndex, action) - } - } - -} - -// TestIngestTestcases reads sample test cases from a test data directory -// and evaluates them. -func TestIngestTestcases(t *testing.T) { - _ = &ingest.Operation{} - var testcases []string - if len(ingestActionKindNames) == 0 { - for k, v := range ingestActionKinds { - ingestActionKindNames[v] = k - } - } - err := filepath.Walk("ingest_testdata", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if strings.HasSuffix(path, ".tc") { - testcases = append(testcases, path) - } - return nil - }) - if err != nil { - t.Fatalf("looking for test cases: %v", err) - } - if len(testcases) == 0 { - t.Fatalf("no ingest test cases found") - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := test.MustRunCluster(t, 3) - defer c.Close() - - coord := c.GetPrimary() - - for _, tc := range testcases { - t.Run(strings.TrimSuffix(tc, ".tc"), func(t *testing.T) { - testOneIngestTestcase(t, ctx, coord, tc) - }) - } -} diff --git a/ingest_testdata/bool.tc b/ingest_testdata/bool.tc deleted file mode 100644 index a5957cb15..000000000 --- a/ingest_testdata/bool.tc +++ /dev/null @@ -1,17 +0,0 @@ -schema: -{ - "index-name": "example", - "index-action": "create", - "primary-key-type": "uint", - "fields": [ - { - "field-name": "tf", - "field-type": "bool" - } - ] -} -ingest: -[{"action": "write", "records": {"2": { "tf": true }}}] -queries: -Row(tf=true) -[2] diff --git a/ingest_testdata/expect_errors.tc b/ingest_testdata/expect_errors.tc deleted file mode 100644 index 707bfcdba..000000000 --- a/ingest_testdata/expect_errors.tc +++ /dev/null @@ -1,101 +0,0 @@ -schema-error: fail to create field -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "cookie", - "field-type": "id", - "field-options": { "cache-type": "none" } - }, - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "nun" } - } - ] -} -schema: confirm index was not created because field failed -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: can't recreate index -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema: can add field to existing index -{ - "index-name": "examplekeys", - "index-action": "ensure", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: second field failing in existing index -{ - "index-name": "examplekeys", - "index-action": "ensure", - "primary-key-type": "string", - "fields": [ - { - "field-name": "addokay", - "field-type": "id", - "field-options": { "cache-type": "none" } - }, - { - "field-name": "addfail", - "field-type": "id", - "field-options": { "cache-type": "nun" } - } - ] -} -schema: verify that fields we think exist do exist -{ - "index-name": "examplekeys", - "index-action": "require", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: successful field deleted anyway because second field failed -{ - "index-name": "examplekeys", - "index-action": "require", - "primary-key-type": "string", - "fields": [ - { - "field-name": "addokay", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} diff --git a/ingest_testdata/keyed.tc b/ingest_testdata/keyed.tc deleted file mode 100644 index 8f7d5c4b6..000000000 --- a/ingest_testdata/keyed.tc +++ /dev/null @@ -1,32 +0,0 @@ -schema: -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "a": { - "set": [ 2 ], - }, - "b": { - "set": 3, - } - } - } -] -queries: -Row(set=2) -["a"] -Union(Row(set=3),Row(set=2)) -["a","b"] diff --git a/ingest_testdata/sample.tc b/ingest_testdata/sample.tc deleted file mode 100644 index 764965025..000000000 --- a/ingest_testdata/sample.tc +++ /dev/null @@ -1,99 +0,0 @@ -schema: -{ - "index-name": "example", - "index-action": "create", - "primary-key-type": "uint", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "1": { - "set": [ 2 ], - }, - "2": { - "set": 3, - } - } - } -] -queries: -Row(set=2) -[1] -Union(Row(set=3),Row(set=2)) -[1,2] -schema-error: -{ - "index-name": "example", - "primary-key-type": "uint", - "index-action": "require", - "fields": [ - { - "field-name": "setkey", - "field-type": "string", - "field-options": { "cache-type": "none" } - } - ] -} -schema: -{ - "index-name": "example", - "primary-key-type": "uint", - "index-action": "ensure", - "fields": [ - { - "field-name": "setkey", - "field-type": "string", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "1": { - "setkey": [ "a" ], - }, - "2": { - "setkey": "b", - } - } - } -] -queries: -Row(setkey="a") -[1] -ingest-error: -[ - { - "action": "setkeys", - "records": { - "a": { - "setkey": [ "a" ], - }, - "b": { - "setkey": "b", - } - } - } -] -ingest: -[ - { - "action": "delete", - "record_ids": [ 1 ] - } -] -queries: -Row(setkey="a") -[] diff --git a/internal_client.go b/internal_client.go index 8d7666292..9620c4a75 100644 --- a/internal_client.go +++ b/internal_client.go @@ -21,7 +21,6 @@ import ( "github.com/hashicorp/go-retryablehttp" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/tracing" @@ -392,124 +391,6 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return rsp.Indexes, nil } -// IngestSchema uses the new schema ingest endpoint. It returns a -// map from index names to fields created within them; note that if the -// entire index was created, the list of fields is empty. The intended -// usage is cleaning up after creating the indexes, so if you create the -// index, you don't need to delete the fields, but if you created fields -// within an existing index, you should delete those fields but not the -// whole index. -func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []byte) (created map[string][]string, err error) { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path("/internal/schema") - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - buf, err = io.ReadAll(resp.Body) - if resp.StatusCode != 200 { - if err != nil { - return nil, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status) - } - var msg string - // try to decode a JSON response - var sr successResponse - qr := &QueryResponse{} - if err = json.Unmarshal(buf, &sr); err == nil { - msg = sr.Error.Error() - } else if err := c.serializer.Unmarshal(buf, qr); err == nil { - msg = qr.Err.Error() - } else { - msg = string(buf) - } - return nil, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg) - } - // this is the err from io.ReadAll, but in the case where resp.StatusCode - // was 2xx, so we don't have a bad status. - if err != nil { - return nil, errors.Wrapf(err, "error reading response body") - } - if err = json.Unmarshal(buf, &created); err != nil { - return nil, errors.Wrapf(err, "error interpreting response body") - } - return created, nil -} - -// IngestOperations uses the new ingest endpoint for ingest data -func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, indexName string, buf []byte) error { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path(fmt.Sprintf("/internal/ingest/%s", indexName)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return errors.Errorf("unexpected status code: %s", resp.Status) - } - return nil -} - -// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data -func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName)) - - buf, err := c.serializer.Marshal(ireq) - if err != nil { - return errors.Wrap(err, "marshalling") - } - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return errors.Errorf("unexpected status code: %s", resp.Status) - } - return nil -} - // MutexCheck uses the mutex-check endpoint to request mutex collision data // from a single node. It produces per-shard results, and does not translate // them. diff --git a/translate.go b/translate.go index 7de4863b4..64d3f6bd8 100644 --- a/translate.go +++ b/translate.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" ) @@ -89,39 +88,6 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul Delete(records *roaring.Bitmap) (Commitor, error) } -// This implements ingest's key translator interface, which differs -// slightly because we want to be able to do fast lookups on arbitrary -// IDs which are not necessarily contiguous small values, so the []string -// from TranslateIDs isn't a good fit. -type ingestKeyTranslator struct { - store TranslateStore -} - -var _ ingest.KeyTranslator = &ingestKeyTranslator{} - -func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - return i.store.CreateKeys(keys...) -} - -func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - keys, err := i.store.TranslateIDs(ids) - if err != nil { - return nil, err - } - if len(keys) != len(ids) { - return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) - } - out := make(map[uint64]string, len(keys)) - for i, id := range ids { - out[id] = keys[i] - } - return out, nil -} - -func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator { - return &ingestKeyTranslator{store: s} -} - // TranslatorSummary is returned, for example from the boltdb string key translators, // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. type TranslatorSummary struct {