From 16ccbc461a71363ebc43c2f3c51431080ec68d0b Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 31 Oct 2022 10:24:23 -0500 Subject: [PATCH] Drop the ingest subpackage and related endpoints. The internal/ingest and internal/schema endpoints were developed with intent that they'd be the primary interface new users would work with, because they were Easy To Use, and did not require any kind of setup, the counterpoint being that ingest done this way had performance issues because it ended up with huge amounts of JSON parsing to reformat things into our native format. But this was understood to be the price of providing a new-user-friendly JSON ingest experience. A year later, we have no evidence that it's ever been used. We never even moved it out of the `/internal` path. It's a lot of very complex fiddly code and we don't seem to be using it, and at this point, our anticipation is that if we really need something, we'll use CSV, which we already have working, or something in the new SQL code. Either way, we don't seem to be using this. --- api.go | 408 ----------- api_test.go | 115 --- apimethod_string.go | 8 +- client/client.go | 28 - cluster.go | 33 - encoding/proto/proto.go | 112 --- encoding/proto/proto_test.go | 97 +-- go.mod | 1 - go.sum | 2 - handler.go | 118 ++- handler_test.go | 28 + http_handler.go | 157 ---- http_handler_test.go | 199 +---- ingest/codec.go | 1169 ------------------------------ ingest/codec_test.go | 978 ------------------------- ingest/doc.go | 17 - ingest/op.go | 826 --------------------- ingest/op_test.go | 275 ------- ingest/shard.go | 2 - ingest/sort.go | 232 ------ ingest/sort_test.go | 218 ------ ingest/translate_test.go | 124 ---- ingest/update.go | 243 ------- ingest/vec.go | 122 ---- ingest/vec_test.go | 101 --- ingest_test.go | 448 ------------ ingest_testdata/bool.tc | 17 - ingest_testdata/expect_errors.tc | 101 --- ingest_testdata/keyed.tc | 32 - ingest_testdata/sample.tc | 99 --- internal_client.go | 119 --- translate.go | 34 - 32 files changed, 163 insertions(+), 6300 deletions(-) create mode 100644 handler_test.go delete mode 100644 ingest/codec.go delete mode 100644 ingest/codec_test.go delete mode 100644 ingest/doc.go delete mode 100644 ingest/op.go delete mode 100644 ingest/op_test.go delete mode 100644 ingest/shard.go delete mode 100644 ingest/sort.go delete mode 100644 ingest/sort_test.go delete mode 100644 ingest/translate_test.go delete mode 100644 ingest/update.go delete mode 100644 ingest/vec.go delete mode 100644 ingest/vec_test.go delete mode 100644 ingest_test.go delete mode 100644 ingest_testdata/bool.tc delete mode 100644 ingest_testdata/expect_errors.tc delete mode 100644 ingest_testdata/keyed.tc delete mode 100644 ingest_testdata/sample.tc diff --git a/api.go b/api.go index 080c1a8e5..753e7aafa 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "time" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/rbf" //"github.com/molecula/featurebase/v3/pg" @@ -1122,164 +1121,6 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return nil } -// applyOneIngestSchema applies a single ingestSpec, which specifies operations on -// a single index and possibly fields. If it is successful, it returns the name -// of the index and an empty slice (if it created the index), or the name of the -// index and a slice of the fields within that index that it created. If it -// is unsuccessful, it tries to delete whatever it created. -// -// The intended idiom is that if the returned list of fields isn't empty, the index -// already existed and only those fields need to be cleaned up in the event of -// a later error, but if the list of fields is empty, the entire index was new, -// and should be cleaned up, in which case there's no need to track or delete -// the specific fields separately. -func (api *API) ApplyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { - if api.PrimaryNode().ID != api.NodeID() { - return nil, nil, RedirectError{ - HostPort: api.PrimaryNode().URI.Normalize(), - error: "request made to non-primary node", - } - } - - // create index - indexName := schema.IndexName - var createdFields []string - var useKeys bool - switch schema.PrimaryKeyType { - case "string": - useKeys = true - case "uint": - useKeys = false - default: - return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) - } - opts := IndexOptions{ - Keys: useKeys, - TrackExistence: true, - } - createdIndex := false - - // We check this up here because, if there's at least one field but we don't know what to do with - // it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in - // trying to create it. We don't care about this if there's no fields specified. - if len(schema.Fields) > 0 { - switch schema.FieldAction { - case "create", "ensure", "require": - // do nothing - case "": - schema.FieldAction = schema.IndexAction - default: - return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction) - } - } - - switch schema.IndexAction { - case "ensure", "require": - index, err = api.Index(ctx, indexName) - if err != nil { - if _, ok := err.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) - } else { - err = nil - } - } - if index != nil { - existingOpts := index.Options() - if existingOpts != opts { - return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts) - } - break - } - if schema.IndexAction == "require" { - return nil, nil, fmt.Errorf("index %q does not exist", indexName) - } - fallthrough - case "create": - index, err = api.CreateIndex(ctx, indexName, opts) - if err != nil { - return nil, nil, err - } - createdIndex = true - default: - return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction) - } - - // Now we might have an index, so we need our cleanup code. - defer func() { - if err == nil { - return - } - if createdIndex { - err := api.DeleteIndex(ctx, indexName) - if err != nil { - - api.server.logger.Printf("trying to undo failed index %q creation: %v", indexName, err) - } - return - } - for _, field := range createdFields { - err := api.DeleteField(ctx, indexName, field) - if err != nil { - api.server.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err) - } - } - }() - - // create all the fields specified in the index - for _, fSpec := range schema.Fields { - fieldName := fSpec.FieldName - opt := fieldSpecToFieldOption(fSpec) - err = opt.validate() - if err != nil { - return nil, nil, err - } - switch schema.FieldAction { - case "ensure", "require": - field, schemaErr := api.Field(ctx, indexName, fieldName) - if schemaErr != nil { - // NotFoundError is fine - if _, ok := schemaErr.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) - } - } - if field != nil { - existing := field.Options() - if opt.Type != existing.Type { - return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type) - } - if ((opt.Keys != nil) && *opt.Keys) != existing.Keys { - if existing.Keys { - return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName) - } else { - return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName) - } - } - // TODO: verify compatibility of other field opts, this is sorta hard - break - } - if schema.FieldAction == "require" { - return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName) - } - fallthrough - case "create": - fos := fieldOptionsToFunctionalOpts(opt) - _, err = api.CreateField(ctx, indexName, fieldName, fos...) - if err != nil { - return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err) - } - createdFields = append(createdFields, fieldName) - } - } - - // we don't report the fields back, so we can distinguish "created index" - // from "created fields within index" - if createdIndex { - createdFields = nil - } - - return index, createdFields, nil -} - // Views returns the views in the given field. func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Views") @@ -1936,251 +1777,6 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -// ingestNodeOperationsForFields does the actual work of applying operations -// to a given index with a map of known fields and an already-parsed -// ShardedRequest. This is used locally on the node that first receives -// the request, after it does the parsing, and on other nodes because the -// format they get is already that rather than JSON, so it's the common -// path *after* key translation and sorting into shards. -func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { - eg, ctx := errgroup.WithContext(ctx) - for shard, ops := range req.Ops { - // create new local copies of these values so the goroutine uses these - // copies, and doesn't read the actual loop variables, which are being - // changed by the loop. - shard, ops := shard, ops - eg.Go(func() error { - return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) - }) - } - return eg.Wait() -} - -// IngestNodeOperations handles protobuf-formatted data which does not need -// key translation and is applicable to this specific node. -func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { - index := api.holder.Index(indexName) - if index == nil { - api.server.logger.Errorf("ingest: no such index %q", indexName) - return newNotFoundError(ErrIndexNotFound, indexName) - } - fields := index.Fields() - knownFields := map[string]*Field{} - for _, field := range fields { - knownFields[field.name] = field - } - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) -} - -// IngestOperations handles JSON-formatted data which may need key translation -// and may be for any or all nodes. -func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { - span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") - defer span.Finish() - - if api.PrimaryNode().ID != api.NodeID() { - return RedirectError{ - HostPort: api.PrimaryNode().URI.Normalize(), - error: "request made to non-primary node", - } - } - - if err := api.validate(apiIngestOperations); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Find the Index. - index := api.holder.Index(indexName) - if index == nil { - api.server.logger.Errorf("ingest: no such index %q", indexName) - return newNotFoundError(ErrIndexNotFound, indexName) - } - fields := index.Fields() - var indexKeys ingest.KeyTranslator - if index.Keys() { - indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) - } - codec, err := ingest.NewJSONCodec(indexKeys) - if err != nil { - return errors.Wrap(err, "creating JSON codec") - } - knownFields := map[string]*Field{} - for _, field := range fields { - var keys ingest.KeyTranslator - if field.usesKeys { - keys = newIngestKeyTranslatorFromStore(field.translateStore) - } - knownFields[field.name] = field - switch field.Type() { - case "set": - if err = codec.AddSetField(field.name, keys); err != nil { - return fmt.Errorf("adding set field to codec: %w", err) - } - case "time": - if err = codec.AddTimeQuantumField(field.name, keys); err != nil { - return fmt.Errorf("adding time quantum field to codec: %w", err) - } - case "mutex": - if err = codec.AddMutexField(field.name, keys); err != nil { - return fmt.Errorf("adding mutex field to codec: %w", err) - } - case "bool": - if err = codec.AddBoolField(field.name); err != nil { - return fmt.Errorf("adding bool field to codec: %w", err) - } - case "int": - if err = codec.AddIntField(field.name, keys); err != nil { - return fmt.Errorf("adding int field to codec: %w", err) - } - case "decimal": - if err = codec.AddDecimalField(field.name, field.options.Scale); err != nil { - return fmt.Errorf("adding decimal field to codec: %w", err) - } - case "timestamp": - if err = codec.AddTimestampField(field.name, field.options.TimeUnit, field.options.Base); err != nil { - return fmt.Errorf("adding timestamp field to codec: %w", err) - } - default: - return fmt.Errorf("unhandled field type %q", field.Type()) - } - } - req, err := codec.Parse(stream) - if err != nil { - return errors.Wrap(err, "parsing input data") - } - sharded, err := codec.RequestByShard(req) - if err != nil { - return errors.Wrap(err, "sharding input data") - } - // now that we have this, let's assign the shards to nodes - snap := api.cluster.NewSnapshot() - // oh hey an easy case: we're presumably the only node - if len(snap.Nodes) == 1 { - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) - } - // Created new ShardedRequest objects for every node, giving each of them - // all the shards that apply to them. - byNode := make(map[string]*ingest.ShardedRequest) - for shard, ops := range sharded.Ops { - nodes := snap.ShardNodes(indexName, shard) - for _, node := range nodes { - forThisShard := byNode[node.ID] - if forThisShard == nil { - // Create new ShardedRequest for the target node, with its op map - // mapping this shard to the ops for this shard. - byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} - continue - } - // Add this shard to the existing ShardedRequest's Ops map. Note that - // we don't have to worry about overwrites; we can't have seen this - // shard before, because we're in a range loop on a map where the shard - // is the key. - forThisShard.Ops[shard] = ops - } - } - eg, ctx := errgroup.WithContext(ctx) - for _, node := range snap.Nodes { - node := node - sharded := byNode[node.ID] - // Sometimes, there's nothing for a specific node. - if sharded == nil { - continue - } - if node.ID == api.NodeID() { - eg.Go(func() error { - return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) - }) - } else { - eg.Go(func() error { - return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) - }) - } - } - return eg.Wait() -} - -// applyOperations applies a set of operations to one specific shard. -func (api *API) applyOperations(ctx context.Context, qcx *Qcx, index *Index, shard uint64, fields map[string]*Field, ops []*ingest.Operation) error { - // For each operation, we may have a set of records/fields to clear, and then - // also a set of fields to set/remove specific bits in. - opts := &ImportOptions{Presorted: true, IgnoreKeyCheck: true, fullySorted: true} - for _, op := range ops { - // ClearRecordIDs should exist only for delete, clear, and write. For clear and write, - // we'll have a list of fields, for delete, it should be all the fields. - if len(op.ClearRecordIDs) > 0 { - // anonymous func lets us defer a finisher from any of the inner error returns - err := func() (e0 error) { - // WARNING: Depends on GetTx being per-shard/index, not per-field. - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard}) - if err != nil { - return fmt.Errorf("getting Tx: %w", err) - } - defer finisher(&e0) - // For a delete, we don't look at the fields the codec was defined with, - // We delete from the existence field unconditionally and other fields - // if we know they exist. - if op.OpType == ingest.OpDelete { - err = clearExistenceColumns(tx, index, op.ClearRecordIDs, shard) - if err != nil { - return fmt.Errorf("clearing existence columns: %w", err) - } - for name, field := range fields { - if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { - return fmt.Errorf("clearing field %q: %w", name, err) - } - } - return nil - } - // clear things that we need to wipe out, whether it's because - // this is a Clear op, or because it's a write op that - // specifies clears for the fields it's going to write to. - if len(op.ClearFields) > 0 { - for _, fieldName := range op.ClearFields { - field, ok := fields[fieldName] - if !ok { - return fmt.Errorf("can't find a field named %q", fieldName) - } - if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil { - return fmt.Errorf("clearing record IDs: %w", err) - } - } - } - return nil - }() - if err != nil { - return err - } - } - opts.Clear = (op.OpType == ingest.OpRemove) - // for "set" and "write" ops, we'll be setting bits, for - // "remove" ops we'll be clearing them, and for "clear" ops - // there shouldn't be anything here. - for fieldName, fieldOp := range op.FieldOps { - field, ok := fields[fieldName] - if !ok { - return fmt.Errorf("can't find a field named %q", fieldName) - } - var err error - err = importExistenceColumns(qcx, index, fieldOp.RecordIDs, shard) - if err != nil { - return errors.Wrap(err, "importing existence columns") - } - switch field.Type() { - case "set", "time", "mutex", "bool": - err = field.Import(qcx, fieldOp.Values, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) - case "int", "timestamp", "decimal": - err = field.importValue(qcx, fieldOp.RecordIDs, fieldOp.Signed, shard, opts) - default: - err = fmt.Errorf("unhandled field type %q", field.Type()) - } - if err != nil { - return err - } - } - } - return nil -} - func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error { ef := index.existenceField() if ef == nil { @@ -3287,8 +2883,6 @@ const ( apiIDCommit apiIDReset apiPartitionNodes - apiIngestOperations - apiIngestNodeOperations apiMutexCheck ) @@ -3349,8 +2943,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIDCommit: {}, apiIDReset: {}, apiPartitionNodes: {}, - apiIngestOperations: {}, - apiIngestNodeOperations: {}, apiMutexCheck: {}, } diff --git a/api_test.go b/api_test.go index 42e9b171a..58c344406 100644 --- a/api_test.go +++ b/api_test.go @@ -521,72 +521,6 @@ func TestAPI_Ingest(t *testing.T) { t.Fatalf("creating field: %v", err) } - t.Run("IngestAPI", func(t *testing.T) { - sampleJson := []byte(` - [ - { - "action": "set", - "records": { - "2": { - "set": [2], - "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] } - }, - "5": { "set": [3] }, - "8": { "set": [3] }, - "1": { - "set": [2], - "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } - }, - "4": { "set": [3, 7] } - } - }, - { - "action": "clear", - "record_ids": [ 5, 6, 7 ], - "fields": [ "tq", "set" ] - }, - { - "action": "write", - "records": { - "8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } }, - "9": { "set": [7, 3] } - } - }, - { - "action": "delete", - "record_ids": [ 9 ] - } - ] - `) - // just for set row 3: - // first operation should set it for 4, 5, and 8. - // clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8. - // the write operation should clear set for record 8, even though record 8 doesn't - // contain that field in that op, because set is present in record 9, which also - // gets row 3 set. but then we delete 9. - // so after all that we expect Row(set=3) to be 4... - sampleBuf := bytes.NewBuffer(sampleJson) - qcx := coord.API.Txf().NewQcx() - defer func() { - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf) - if err != nil { - t.Fatalf("importing data: %v", err) - } - query := "Row(set=3)" - res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - if err != nil { - t.Errorf("query: %v", err) - } - r := res.Results[0].(*pilosa.Row).Columns() - if len(r) != 1 || r[0] != 4 { - t.Fatalf("expected row with 4 set, got %d", r) - } - }) - t.Run("ImportRoaringShard", func(t *testing.T) { setBuf := &bytes.Buffer{} setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7) @@ -697,55 +631,6 @@ func ingestBenchmarkHelper() []byte { return data } -func BenchmarkIngest(b *testing.B) { - b.StopTimer() - data := ingestBenchmarkHelper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := test.MustRunCluster(b, 1) - defer c.Close() - - coord := c.GetPrimary() - m0 := c.GetNode(0) - // m1 := c.GetNode(1) - // m2 := c.GetNode(2) - - index := c.Idx() - setField := "set" - intField := "int" - tqField := "tq" - _, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false}) - if err != nil { - b.Fatalf("creating index: %v", err) - } - _, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0)) - if err != nil { - b.Fatalf("creating field: %v", err) - } - _, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840)) - if err != nil { - b.Fatalf("creating field: %v", err) - } - _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0")) - if err != nil { - b.Fatalf("creating field: %v", err) - } - b.ReportAllocs() - b.StartTimer() - for i := 0; i < b.N; i++ { - qcx := m0.API.Txf().NewQcx() - defer qcx.Abort() - err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data)) - if err != nil { - b.Fatalf("ingest: %v", err) - } - err = qcx.Finish() - if err != nil { - b.Fatalf("finish: %v", err) - } - } -} - func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() diff --git a/apimethod_string.go b/apimethod_string.go index eebc4cab6..da9fde7d5 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -42,14 +42,12 @@ func _() { _ = x[apiIDCommit-31] _ = x[apiIDReset-32] _ = x[apiPartitionNodes-33] - _ = x[apiIngestOperations-34] - _ = x[apiIngestNodeOperations-35] - _ = x[apiMutexCheck-36] + _ = x[apiMutexCheck-34] } -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 499, 522, 535} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/client/client.go b/client/client.go index e41add3bb..f825c6310 100644 --- a/client/client.go +++ b/client/client.go @@ -790,34 +790,6 @@ func (c *Client) readSchema() ([]SchemaIndex, error) { return schemaInfo.Indexes, nil } -func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err error) { - data, err := json.Marshal(reqBody) - if err != nil { - return data, errors.Wrap(err, "error building Schema body to Ingest") - } - return c.IngestRequest("/internal/schema", data) -} - -func (c *Client) IngestData(index string, reqBody []map[string]interface{}) (body []byte, err error) { - data, err := json.Marshal(reqBody) - if err != nil { - return data, errors.Wrap(err, "error building request body to Ingest") - } - return c.IngestRequest("/internal/ingest/"+index, data) -} - -func (c *Client) IngestRequest(uri string, data []byte) (body []byte, err error) { - var header = make(map[string]string) - header["Content-Type"] = "application/json" - header["Accept"] = "application/json" - header["User-Agent"] = "pilosa/" + pilosa.Version - status, body, err := c.HTTPRequest("POST", uri, data, header) - if err != nil { - return nil, errors.Wrapf(err, "requesting %s status: %d", uri, status) - } - return body, err -} - func (c *Client) shardsMax() (map[string]uint64, error) { _, data, err := c.HTTPRequest("GET", "/internal/shards/max", nil, nil) if err != nil { diff --git a/cluster.go b/cluster.go index 549cb9b03..a173efdae 100644 --- a/cluster.go +++ b/cluster.go @@ -8,7 +8,6 @@ import ( "time" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" @@ -504,38 +503,6 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } -// This implements ingest's key translator interface on a cluster/index pair. -type clusterKeyTranslator struct { - ctx context.Context // we're created within a request context and need to pass that to cluster ops - c *cluster - indexName string -} - -var _ ingest.KeyTranslator = &clusterKeyTranslator{} - -func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - return i.c.createIndexKeys(i.ctx, i.indexName, keys...) -} - -func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids) - if err != nil { - return nil, err - } - if len(keys) != len(ids) { - return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) - } - out := make(map[uint64]string, len(keys)) - for i, id := range ids { - out[id] = keys[i] - } - return out, nil -} - -func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator { - return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName} -} - // TODO: remove this when it is no longer used func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keys := make([]string, 0, len(keySet)) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index da5d64570..b83692713 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -9,7 +9,6 @@ import ( "github.com/gogo/protobuf/proto" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" @@ -294,19 +293,6 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } *mt = s.decodeRowMatrix(msg) return nil - - case *ingest.ShardedRequest: - msg := &pb.ShardedIngestRequest{} - err := proto.Unmarshal(buf, msg) - if err != nil { - return errors.Wrap(err, "unmarshalling ShardedRequest") - } - req, err := s.decodeShardedIngestRequest(msg) - if err != nil { - return err - } - *mt = *req - return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -376,8 +362,6 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeTransactionMessage(mt) case *pilosa.AtomicRecord: return s.encodeAtomicRecord(mt) - case *ingest.ShardedRequest: - return s.encodeShardedIngestRequest(mt) } return nil } @@ -900,48 +884,6 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr return &pb.TransactionStats{} } -func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest { - if req == nil || len(req.Ops) == 0 { - return &pb.ShardedIngestRequest{} - } - out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))} - for shard, ops := range req.Ops { - out.Ops[shard] = s.encodeShardIngestOperations(ops) - } - return out -} - -func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { - out := &pb.ShardIngestOperations{} - for _, op := range ops { - if op == nil { - continue - } - out.Ops = append(out.Ops, s.encodeShardIngestOperation(op)) - } - return out -} - -func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation { - out := &pb.ShardIngestOperation{ - OpType: op.OpType.String(), - ClearRecordIDs: op.ClearRecordIDs, - ClearFields: op.ClearFields, - FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), - } - for k, v := range op.FieldOps { - if v == nil { - continue - } - out.FieldOps[k] = &pb.FieldOperation{ - RecordIDs: v.RecordIDs, - Values: v.Values, - Signed: v.Signed, - } - } - return out -} - func (s Serializer) decodeSchema(sc *pb.Schema, m *pilosa.Schema) { m.Indexes = make([]*pilosa.IndexInfo, len(sc.Indexes)) s.decodeIndexes(sc.Indexes, m.Indexes) @@ -1844,57 +1786,3 @@ func (s Serializer) encodeDecimal(p *pql.Decimal) *pb.Decimal { } return retval } - -func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) { - if req == nil || len(req.Ops) == 0 { - return &ingest.ShardedRequest{}, nil - } - out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))} - for shard, ops := range req.Ops { - var err error - out.Ops[shard], err = s.decodeShardIngestOperations(ops) - if err != nil { - return nil, err - } - } - return out, nil -} - -func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) { - out := []*ingest.Operation{} - if len(ops.Ops) == 0 { - return out, nil - } - for _, op := range ops.Ops { - if op == nil { - continue - } - decoded, err := s.decodeShardIngestOperation(op) - if err != nil { - return nil, err - } - out = append(out, decoded) - } - return out, nil -} - -func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) { - opType, err := ingest.ParseOpType(op.OpType) - if err != nil { - return nil, err - } - out := &ingest.Operation{ - OpType: opType, - ClearRecordIDs: op.ClearRecordIDs, - ClearFields: op.ClearFields, - FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)), - } - for k, v := range op.FieldOps { - out.FieldOps[k] = &ingest.FieldOperation{ - RecordIDs: v.RecordIDs, - Values: v.Values, - Signed: v.Signed, - } - } - return out, nil -} diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go index 6c2107fc5..a7500fa4d 100644 --- a/encoding/proto/proto_test.go +++ b/encoding/proto/proto_test.go @@ -2,12 +2,10 @@ package proto import ( - "errors" "reflect" "testing" pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/pb" ) @@ -40,99 +38,8 @@ func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, exp t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error()) } } - switch real := obj.(type) { - case *ingest.ShardedRequest: - real2 := obj2.(*ingest.ShardedRequest) - err := real.Compare(real2) - if err != nil { - if expectedMismatchErr == nil { - t.Fatalf("unexpected compare error %q", err.Error()) - } - if err.Error() != expectedMismatchErr.Error() { - t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error()) - } - } else { - if expectedMismatchErr != nil { - t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error()) - } - } - default: - if !reflect.DeepEqual(obj, obj2) { - t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) - } - } -} - -type shardedIngestRequestTest struct { - req *ingest.ShardedRequest - err error -} - -var shardedIngestRequestTestcases = []shardedIngestRequestTest{ - { - req: &ingest.ShardedRequest{ - Ops: map[uint64][]*ingest.Operation{ - 1: { - { - OpType: ingest.OpWrite, - ClearFields: []string{"clearField", "clearField2"}, - ClearRecordIDs: []uint64{1, 7, 9}, - FieldOps: map[string]*ingest.FieldOperation{ - "writeAll": { - RecordIDs: []uint64{3, 6, 8}, - Values: []uint64{0, 17, 34}, - Signed: []int64{-9, 23, 17}, - }, - "writeValues": { - RecordIDs: []uint64{3, 6, 8}, - Values: []uint64{0, 17, 34}, - }, - "writeSigned": { - RecordIDs: []uint64{3, 6, 8}, - Signed: []int64{-9, 23, 17}, - }, - }, - }, - { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ - "foo": nil, - }, - }, - }, - 2: {}, - 3: nil, - }, - }, - }, - { - req: &ingest.ShardedRequest{ - Ops: map[uint64][]*ingest.Operation{ - 1: { - { - OpType: ingest.OpWrite, - FieldOps: map[string]*ingest.FieldOperation{ - "writeAll": {}, - }, - }, - { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ - "foo": nil, - }, - }, - nil, - }, - }, - }, - err: errors.New("shard 1: expected 3 ops, got 2"), - }, -} - -func TestIngestRoundTrip(t *testing.T) { - for _, tc := range shardedIngestRequestTestcases { - t.Logf("next case") - testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err) + if !reflect.DeepEqual(obj, obj2) { + t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) } } diff --git a/go.mod b/go.mod index 0202952cd..c35662184 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/aws/aws-sdk-go v1.42.39 github.com/beevik/ntp v0.3.0 github.com/benbjohnson/immutable v0.3.0 - github.com/buger/jsonparser v1.1.1 github.com/cespare/xxhash v1.1.0 github.com/chzyer/readline v1.5.0 github.com/confluentinc/confluent-kafka-go v1.9.1 diff --git a/go.sum b/go.sum index b2d90ac34..4dd0ae0b4 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= diff --git a/handler.go b/handler.go index caac2f815..a07f275ed 100644 --- a/handler.go +++ b/handler.go @@ -3,9 +3,10 @@ package pilosa import ( "encoding/json" + "math/bits" "time" - "github.com/molecula/featurebase/v3/ingest" + "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" ) @@ -156,7 +157,6 @@ func (ivr *ImportValueRequest) Clone() *ImportValueRequest { // The top level Shard has to agree with Ivr[i].Shard and the Iv[i].Shard // for all i included (in Ivr and Ir). The same goes for the top level Index: all records // have to be writes to the same Index. These requirements are checked. -// type AtomicRecord struct { Index string Shard uint64 @@ -299,28 +299,108 @@ func (ir *ImportRequest) Clone() *ImportRequest { // requests. We don't sort the entries within each shard because the correct // sorting depends on the field type and we don't want to deal with that // here. -func (ir *ImportRequest) SortToShards() map[uint64]*ImportRequest { - // cheat: use ingest - fo := ingest.FieldOperation{ - RecordIDs: ir.ColumnIDs, - Values: ir.RowIDs, - Signed: ir.Timestamps, +func (ir *ImportRequest) SortToShards() (result map[uint64]*ImportRequest) { + if len(ir.ColumnIDs) == 0 { + return nil } - sharded := fo.SortToShards() - output := make(map[uint64]*ImportRequest, len(sharded)) - for shard, shardOp := range sharded { - shardReq := *ir - shardReq.ColumnKeys = nil - shardReq.RowKeys = nil - shardReq.Shard = shard - shardReq.ColumnIDs = shardOp.RecordIDs - shardReq.RowIDs = shardOp.Values - shardReq.Timestamps = shardOp.Signed - output[shard] = &shardReq + diffMask := uint64(0) + prev := ir.ColumnIDs[0] + for _, r := range ir.ColumnIDs[1:] { + diffMask |= r ^ prev + prev = r } + bitsRemaining := bits.Len64(diffMask) + if bitsRemaining <= shardwidth.Exponent { + shard := ir.ColumnIDs[0] >> shardwidth.Exponent + ir.Shard = shard + ir.ColumnKeys = nil + ir.RowKeys = nil + return map[uint64]*ImportRequest{shard: ir} + } + output := make(map[uint64]*ImportRequest) + sortToShardsInto(ir, bitsRemaining-8, output) return output } +// sortToShardsInto puts the shards it finds into the given map, so that +// as we split off buckets, they can be inserted into the same map. +func sortToShardsInto(ir *ImportRequest, shift int, into map[uint64]*ImportRequest) { + if shift < shardwidth.Exponent { + shift = shardwidth.Exponent + } + nextShift := shift - 8 + if nextShift < shardwidth.Exponent { + nextShift = shardwidth.Exponent + } + // count things that belong in each of the 256 buckets + var buckets [256]int + var starts [256]int + + // compute the buckets ourselves + for _, r := range ir.ColumnIDs { + b := (r >> shift) & 0xFF + buckets[b]++ + } + total := 0 + // compute starting points of each bucket, converting the + // bucket counts into ends + for i := range buckets { + starts[i] = total + total += buckets[i] + buckets[i] = total + } + // starts[n] is the index of the first thing that should + // go in that bucket, buckets[n] is the index of the first + // thing that shouldn't + var bucketOp ImportRequest = *ir + bucketOp.ColumnKeys = nil + bucketOp.RowKeys = nil + origStarts := make([]int, len(starts)) + copy(origStarts, starts[:]) + for bucket, start := range origStarts { + end := buckets[bucket] + if end <= start { + continue + } + for j := start; j < end; j++ { + want := int((ir.ColumnIDs[j] >> shift) & 0xFF) + for want != bucket { + // move this to the beginning of the + // bucket it wants to be in, swapping + // the thing there here + dst := starts[want] + ir.ColumnIDs[j], ir.ColumnIDs[dst] = ir.ColumnIDs[dst], ir.ColumnIDs[j] + if ir.RowIDs != nil { + ir.RowIDs[j], ir.RowIDs[dst] = ir.RowIDs[dst], ir.RowIDs[j] + } + if ir.Timestamps != nil { + ir.Timestamps[j], ir.Timestamps[dst] = ir.Timestamps[dst], ir.Timestamps[j] + } + starts[want]++ + want = int((ir.ColumnIDs[j] >> shift) & 0xFF) + } + } + // If shift == shardwidth.Exponent, then this is a completed + // shard and can go into the sharded output. otherwise, we + // can subdivide it. + bucketOp.ColumnIDs = ir.ColumnIDs[start:end] + if ir.RowIDs != nil { + bucketOp.RowIDs = ir.RowIDs[start:end] + } + if ir.Timestamps != nil { + bucketOp.Timestamps = ir.Timestamps[start:end] + } + if shift == shardwidth.Exponent { + x := bucketOp + shard := ir.ColumnIDs[start] >> shardwidth.Exponent + x.Shard = shard + into[shard] = &x + } else { + sortToShardsInto(&bucketOp, nextShift, into) + } + } +} + // ValidateWithTimestamp ensures that the payload of the request is valid. func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error { if (ir.IndexCreatedAt != 0 && ir.IndexCreatedAt != indexCreatedAt) || diff --git a/handler_test.go b/handler_test.go new file mode 100644 index 000000000..7e102d303 --- /dev/null +++ b/handler_test.go @@ -0,0 +1,28 @@ +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa_test + +import ( + "math/rand" + "testing" + + pilosa "github.com/molecula/featurebase/v3" +) + +func TestSortToShards(t *testing.T) { + var ir pilosa.ImportRequest + expected := make(map[uint64][]uint64) + const n = 50 + rng := rand.New(rand.NewSource(3)) + for i := 0; i < n; i++ { + x := uint64(rng.Intn(8 * pilosa.ShardWidth)) + shard := x / pilosa.ShardWidth + expected[shard] = append(expected[shard], x) + ir.ColumnIDs = append(ir.ColumnIDs, x) + } + out := ir.SortToShards() + for shard, values := range out { + if len(values.ColumnIDs) != len(expected[shard]) { + t.Fatalf("shard %d: expected values %d, got values %d", shard, expected[shard], values.ColumnIDs) + } + } +} diff --git a/http_handler.go b/http_handler.go index f26a3074b..5579bd0ee 100644 --- a/http_handler.go +++ b/http_handler.go @@ -33,7 +33,6 @@ import ( "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/authz" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/monitor" "github.com/molecula/featurebase/v3/pql" @@ -584,10 +583,7 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/index/{index}/shards", handler.chkAuthZ(handler.handleGetIndexAvailableShards, authz.Read)).Methods("GET").Name("GetIndexAvailableShards") router.HandleFunc("/internal/nodes", handler.chkAuthN(handler.handleGetNodes)).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.chkAuthN(handler.handleGetShardsMax)).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/ingest/{index}", handler.chkAuthZ(handler.handlePostIngestData, authz.Write)).Methods("POST").Name("PostIngestData") - router.HandleFunc("/internal/ingest/{index}/node", handler.chkAuthZ(handler.handlePostIngestNode, authz.Write)).Methods("POST").Name("PostIngestNode") - router.HandleFunc("/internal/schema", handler.chkAuthZ(handler.handleIngestSchema, authz.Admin)).Methods("POST").Name("PostIngestSchema") router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.chkAuthZ(handler.handleFindIndexKeys, authz.Admin)).Methods("POST").Name("FindIndexKeys") router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.chkAuthZ(handler.handleCreateIndexKeys, authz.Admin)).Methods("POST").Name("CreateIndexKeys") router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.chkAuthZ(handler.handlePostTranslateIndexDB, authz.Admin)).Methods("POST").Name("PostTranslateIndexDB") @@ -2087,40 +2083,6 @@ func (h *Handler) handlePatchField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -// handlePostIngestData handles JSON ingest data that may need key -// translation, for the entire cluster. -func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { - if !validHeaderAcceptJSON(r.Header) { - http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) - return - } - - indexName, ok := mux.Vars(r)["index"] - if !ok { - http.Error(w, "index name is required", http.StatusBadRequest) - return - } - - qcx := h.api.Txf().NewQcx() - err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) - if err != nil { - qcx.Abort() - switch e := err.(type) { - case RedirectError: - http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) - return - } - } - err = qcx.Finish() - if err != nil { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - return - } - - resp := successResponse{h: h, Name: indexName} - resp.write(w, err) -} - type ingestSpec struct { IndexName string `json:"index-name"` IndexAction string `json:"index-action"` @@ -2182,79 +2144,6 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { return opt } -func (h *Handler) handleIngestSchema(w http.ResponseWriter, r *http.Request) { - if !validHeaderAcceptJSON(r.Header) { - http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) - return - } - - resp := successResponse{h: h} - - dec := json.NewDecoder(r.Body) - dec.DisallowUnknownFields() - schema := ingestSpec{} - // if a key in cleanupIndexes points to a 0-length slice, the - // entire index should be cleaned; otherwise, only the named - // fields within that index should be cleaned. - cleanupIndexes := map[string][]string{} - var schemaErr error - defer func() { - // we set schemaErr in any case where we need to do cleanup - if schemaErr != nil { - for index, fields := range cleanupIndexes { - if len(fields) == 0 { - err := h.api.DeleteIndex(r.Context(), index) - if err != nil { - h.logger.Printf("deleting index %q after schema err: %v", index, err) - } - } else { - for _, field := range fields { - err := h.api.DeleteField(r.Context(), index, field) - if err != nil { - h.logger.Printf("deleting field %q from index %q after schema err: %v", field, index, err) - } - } - } - } - } - }() - for dec.More() { - err := dec.Decode(&schema) - if err != nil { - resp.write(w, err) - return - } - index, fields, err := h.api.ApplyOneIngestSchema(r.Context(), &schema) - if err != nil { - switch e := err.(type) { - case RedirectError: - http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) - return - default: - // if a previous schema created things, clean them up... - schemaErr = err - resp.write(w, err) - return - } - } - // we only have one slot to report these, sorry. - resp.Name = index.Name() - resp.CreatedAt = index.CreatedAt() - cleanupIndexes[index.Name()] = fields - } - // if we got here, we have a cleanupIndexes which we want to return, - // so we want to do that *instead* of the successResponse we'd be - // using otherwise (ironically, to indicate an error) - var mapBody []byte - var err error - if mapBody, err = json.Marshal(cleanupIndexes); err != nil { - resp.write(w, err) - } - if _, err = w.Write(mapBody); err != nil { - h.logger.Printf("error trying to write response: %v", err) - } -} - type postFieldRequest struct { Options fieldOptions `json:"options"` } @@ -3600,52 +3489,6 @@ func (h *Handler) handlePostShardImportRoaring(w http.ResponseWriter, r *http.Re } } -// handlePostIngestNode is the internal endpoint taking already-translated -// ingest operations, sorted by shard, for a single node. -func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if error, code := validateProtobufHeader(r); error != "" { - http.Error(w, error, code) - return - } - - ctx := r.Context() - - // Read entire body. - span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body") - body, err := readBody(r) - span.LogKV("bodySize", len(body)) - span.Finish() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - req := &ingest.ShardedRequest{} - span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") - err = h.serializer.Unmarshal(body, req) - span.Finish() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - urlVars := mux.Vars(r) - indexName := urlVars["index"] - - qcx := h.api.Txf().NewQcx() - err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req) - if err == nil { - err = qcx.Finish() - if err != nil { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - } - } else { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) - qcx.Abort() - } -} - func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { diff --git a/http_handler_test.go b/http_handler_test.go index 928a80363..81bdc34e6 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -345,110 +345,15 @@ func TestUpdateFieldNoStandardView(t *testing.T) { } } -func TestIngestSchemaHandler(t *testing.T) { - c := test.MustRunCluster(t, 3) - defer c.Close() - - schema := fmt.Sprintf(` -{ - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "idset", - "field-type": "id", - "field-options": { - "cache-type": "none" - } - }, - { - "field-name": "id", - "field-type": "id", - "field-options": { - "enforce-mutual-exclusion": true - } - }, - { - "field-name": "bool", - "field-type": "bool" - }, - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - }, - { - "field-name": "string", - "field-type": "string", - "field-options": { - "enforce-mutual-exclusion": true - } - }, - { - "field-name": "int", - "field-type": "int" - }, - { - "field-name": "decimal", - "field-type": "decimal", - "field-options": { - "scale": 2 - } - }, - { - "field-name": "timestamp", - "field-type": "timestamp", - "field-options": { - "epoch": "1996-12-19T16:39:57-08:00", - "unit": "µs" - } - }, - { - "field-name": "quantum", - "field-type": "string", - "field-options": { - "time-quantum": "YMDH" - } - } - ] -} -`, c) - m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) - } - // now, try again, expecting a failure: - resp = test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusConflict { - t.Errorf("invalid status: expected 409, got %d, body=%s", resp.StatusCode, resp.Body) - } -} - func TestPostFieldWithTTL(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - indexName := c.Idx("%s") - - schema := fmt.Sprintf(` - { - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields":[] - } - `, c) - m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + indexName := c.Idx("s") + _, err := c.GetNode(0).API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) } + m := c.GetPrimary() tests := []struct { name string @@ -543,27 +448,15 @@ func TestGetViewAndDelete(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - schema := fmt.Sprintf(` - { - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "test_view", - "field-type": "time", - "field-options": { - "time-quantum": "YMDH" - } - } - ] - } - `, c) m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + + _, err := m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = m.API.CreateField(context.Background(), c.Idx("s"), "test_view", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) + if err != nil { + t.Fatalf("creating field: %v", err) } // Send sample data @@ -658,30 +551,16 @@ func TestTranslationHandlers(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - - schema := fmt.Sprintf(` -{ - "index-name": "%s", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - } - ] -} -`, c) m := c.GetPrimary() - schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) - resp := test.Do(t, "POST", schemaURL, string(schema)) - if resp.StatusCode != http.StatusOK { - t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + _, err = m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) } + _, err = m.API.CreateField(context.Background(), c.Idx("s"), "stringset", pilosa.OptFieldTypeSet("ranked", 100000), pilosa.OptFieldKeys()) + if err != nil { + t.Fatalf("creating field: %v", err) + } + baseURLs := []string{ fmt.Sprintf("%s/internal/translate/index/%s/", m.URL(), c), fmt.Sprintf("%s/internal/translate/field/%s/stringset/", m.URL(), c), @@ -802,10 +681,18 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` defer c.Close() m := c.GetPrimary() - index := "allowed-networks-index" - keyedIndex := "allowed-networks-index-keyed" + index := c.Idx("s") + keyedIndex := c.Idx("k") field := "field1" + // This keyed index used to be created by the Post-Schema subtest, but that doesn't + // exist anymore, so we create it up here. We don't create the other one because it's + // supposed to get created by Post-Index. + _, err = m.API.CreateIndex(context.Background(), c.Idx("k"), pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + // needed for key translation nameBytes, err := json.Marshal([]string{"a", "b", "c"}) if err != nil { @@ -813,24 +700,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` } names := string(nameBytes) - schema := ` - { - "index-name": "allowed-networks-index-keyed", - "primary-key-type": "string", - "index-action": "create", - "fields": [ - { - "field-name": "stringset", - "field-type": "string", - "field-options": { - "cache-type": "ranked", - "cache-size": 100000 - } - } - ] - } - ` - IPTests := []struct { TestName string ClientIP string @@ -864,12 +733,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` url: fmt.Sprintf("%s/schema", m.URL()), body: "", }, - { - testName: "Post-Schema", - method: "POST", - url: fmt.Sprintf("%s/internal/schema", m.URL()), - body: schema, - }, { testName: "Get-Shards", method: "GET", diff --git a/ingest/codec.go b/ingest/codec.go deleted file mode 100644 index 45d17d0f9..000000000 --- a/ingest/codec.go +++ /dev/null @@ -1,1169 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "math" - "sort" - "strconv" - "time" - - "github.com/buger/jsonparser" - "github.com/pkg/errors" -) - -// Featurebase has the following field types as of this writing, we plan to -// support all of them but not all are implemented. -// -// Type Single Signed Timestamp -// set no no no -// time no no yes -// mutex yes no no -// int yes yes no -// decimal yes yes no -// timestamp yes yes no - -// KeyTranslator is a thing that can translate strings to IDs, and also -// IDs back to strings. The ID->string conversion is used only to render -// a request back to JSON, which in turn is only used in testing. The -// functions optionally take an existing map which they then augment. -type KeyTranslator interface { - TranslateKeys(keys ...string) (map[string]uint64, error) - TranslateIDs(ids ...uint64) (map[uint64]string, error) -} - -// Codec is a single-use parser which decodes data into columnar vectors. -type Codec interface { - AddSetField(name string, keys KeyTranslator) error - AddTimeQuantumField(name string, keys KeyTranslator) error - AddMutexField(name string, keys KeyTranslator) error - AddBoolField(name string) error - AddIntField(name string, keys KeyTranslator) error - AddDecimalField(name string, scale int64) error - AddTimestampField(name string, scale string, epoch int64) error - - // Parse data from a reader into the vectors. - // This must only be called once on a codec. - Parse(io.Reader) (*Request, error) -} - -type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error - -// jsonEncFn encodes a value, or range of values, according to a given -// fieldCodec's translation rules. key values, if present, will be used to -// replace whichever values could have been provided as keys. so for instance, -// with an int field which is keyed, values are actually of type int64, but -// if strings are present, those are used. for a time quantum field, the time -// is set from the signed field, and the value is replaced by the key if -// keys are provided. -type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error - -// fieldCodec represents something that can encode and decode a -// particular field. Its methods are not reentrant, it uses internal -// buffers. -type fieldCodec struct { - valueKeys *StringTable - currentOp *FieldOperation - decode jsonDecFn - encode jsonEncFn - // For decimal: Decimal digits of precision. So for instance, with - // scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as - // 120. - scaleUnit int64 - // For timestamp: we use timeUnit to determine the scale at which - // to store a timestamp. For example if the timeUnit is milliseconds - // we store the number of milliseconds from the given epoch. - timeUnit string - scale int64 - epoch int64 // used only by Timestamp fields - scratch []uint64 // reusable scratch space for sets of values - keys KeyTranslator - buf []byte // scratch space for format operations. - fieldType FieldType -} - -// JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps. -type JSONCodec struct { - fieldTypes map[string]FieldType - recKeys *StringTable - fields map[string]*fieldCodec - keys KeyTranslator - currentOp *Operation -} - -// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which -// can be used to write to its buffer. Both types are just embedded because -// they have non-overlapping APIs. Please don't look at my horrible face. -// -// This has some internal objects it can use to stash things -// so that it can pass &foo to enc.Encode() without needing to heap-allocate -// something for the interface conversion, and a buffer it can use for -// converting numbers or times. -// -// It also has an internal error buffer. In fact, in many cases, errors -// are so far as I can tell absolutely impossible -- bytes.Buffer specifically -// promises never to yield an error, and nothing seems to hint that -// enc.Encode can error on integers, strings, booleans, or arrays. So we -// have dozens of error checks that we can't cause to happen even with -// malformed inputs... so we eat those errors, don't require them to be -// checked externally, and return them when done if anyone checks. -type jsonBuffer struct { - *bytes.Buffer - enc *json.Encoder - buf [48]byte // scratch space for using strconv.AppendInt, etc - uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64 - strbuf []string // dummy buffer so that we don't have to alloc to print []string - str string // and again, "so we don't have to alloc a copy" - err error -} - -// Encode removes the stray newlines added by json.Encoder. -func (j *jsonBuffer) Encode(v interface{}) { - err := j.enc.Encode(v) - if err == nil { - j.Truncate(j.Len() - 1) - } else { - j.err = err - } -} - -// EncodeInt uses AppendInt into a static buffer to reduce allocs. -func (j *jsonBuffer) EncodeInt(i int64) { - rep := strconv.AppendInt(j.buf[:0], i, 10) - _, _ = j.Write(rep) -} - -// EncodeUint uses AppendUint into a static buffer to reduce allocs. -func (j *jsonBuffer) EncodeUint(u uint64) { - rep := strconv.AppendUint(j.buf[:0], u, 10) - _, _ = j.Write(rep) -} - -// EncodeUints uses a static copy of a []uint64 -- the slice, not its -// contents -- in already-allocated memory so the interface conversion -// doesn't have to do that. -func (j *jsonBuffer) EncodeUints(u []uint64) { - j.uintbuf = u - j.Encode(&j.uintbuf) - j.strbuf = nil -} - -// EncodeStrings uses a static copy of a []string -- the slice, not its -// contents -- in already-allocated memory so the interface conversion -// doesn't have to do that. -func (j *jsonBuffer) EncodeStrings(s []string) { - j.strbuf = s - j.Encode(&j.strbuf) - j.strbuf = nil -} - -// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs, -// while also surrounding the value with quotes. -func (j *jsonBuffer) EncodeQuotedUint(u uint64) { - j.buf[0] = '"' - rep := strconv.AppendUint(j.buf[1:1], u, 10) - j.buf[len(rep)+1] = '"' - _, _ = j.Write(j.buf[:len(rep)+2]) -} - -// EncodeString appends the JSON encoding of a string. This exists -// because otherwise runtime allocates a heap-allocated copy of the -// string to live inside an interface{} for the duration of a function -// call... -func (j *jsonBuffer) EncodeString(s string) { - j.str = s - j.Encode(&j.str) - j.str = "" -} - -// EncodeTime exists because time's MarshalJSON allocates a new -// buffer every time it gets called, resulting in a full 13% of -// all the allocations produced in a test run, plus another 13% -// or so of them which were for the copies of the time objects -// made to stuff them into an interface{}. Eww. -func (j *jsonBuffer) EncodeTime(t time.Time) { - j.buf[0] = '"' - rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano) - j.buf[len(rep)+1] = '"' - _, _ = j.Write(j.buf[:len(rep)+2]) -} - -// EncodeBool writes a literal representation directly to reduce allocs. -func (j *jsonBuffer) EncodeBool(b bool) { - if b { - _, _ = j.WriteString("true") - } else { - _, _ = j.WriteString("false") - } -} - -func (j *jsonBuffer) Err() error { - return j.err -} - -func newJSONBuffer(data []byte) *jsonBuffer { - e := &jsonBuffer{Buffer: bytes.NewBuffer(data)} - e.enc = json.NewEncoder(e.Buffer) - e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON - return e -} - -var _ Codec = &JSONCodec{} - -func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) { - j := &JSONCodec{ - fields: map[string]*fieldCodec{}, - fieldTypes: map[string]FieldType{}, - } - if keys != nil { - j.recKeys = NewStringTable() - j.keys = keys - } - return j, nil -} - -func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeTimeQuantum, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue - fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeSet, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeSetValue - fieldCodec.encode = fieldCodec.EncodeSetValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeInt, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeIntValue - fieldCodec.encode = fieldCodec.EncodeIntValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeMutex, - keys: keys, - } - fieldCodec.decode = fieldCodec.DecodeMutexValue - fieldCodec.encode = fieldCodec.EncodeMutexValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) AddBoolField(name string) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeBool, - } - fieldCodec.decode = fieldCodec.DecodeBoolValue - fieldCodec.encode = fieldCodec.EncodeBoolValue - return codec.addField(name, fieldCodec) -} - -// TimestampField is used to store seconds since unix epoch. The numeric values -// stored are adjusted based on the given time.Duration; for instance, if the -// time scale is time.Second, then the second after the epoch is stored as 1, -// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch -// passed to this function should be the offset from the Unix epoch to the -// desired epoch, in the same scale. (So if the scale is milliseconds, -// it should be the Unix timestamp in seconds, times 1000.) -func (codec *JSONCodec) AddTimestampField(name string, timeScale string, epoch int64) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeTimeStamp, - timeUnit: timeScale, - epoch: epoch, - } - fieldCodec.decode = fieldCodec.DecodeTimeValue - fieldCodec.encode = fieldCodec.EncodeTimeValue - return codec.addField(name, fieldCodec) -} - -// AddDecimalField adds a decimal field, which is stored as integer values -// with a scale offset, but parsed as floating point values. For instance, -// with decimalScale=2, `0.01` would store the value 1. -func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error { - fieldCodec := &fieldCodec{ - fieldType: FieldTypeDecimal, - scale: decimalScale, - scaleUnit: int64(math.Pow(10, float64(decimalScale))), - } - fieldCodec.decode = fieldCodec.DecodeDecimalValue - fieldCodec.encode = fieldCodec.EncodeDecimalValue - return codec.addField(name, fieldCodec) -} - -func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error { - if _, ok := codec.fields[name]; ok { - return fmt.Errorf("duplicate field %q", name) - } - if fieldCodec.keys != nil { - fieldCodec.valueKeys = NewStringTable() - } - codec.fields[name] = fieldCodec - codec.fieldTypes[name] = fieldCodec.fieldType - return nil -} - -// decodeSetOrValue decodes a value which might be either an array of values or a -// single value, where values might be string keys or bare numbers, calling cb for -// each value it finds. -func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { - switch dataType { - case jsonparser.Array: - _, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - var id uint64 - switch dataType { - case jsonparser.String: - id, err = j.valueKeys.ID(value) - if err != nil { - return - } - err = cb(id) - case jsonparser.Number: - if j.valueKeys != nil { - err = errors.New("expecting key, got numeric value") - return - } - id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64) - if err != nil { - return - } - err = cb(id) - default: - err = fmt.Errorf("expecting value or array, got %v", dataType) - } - }) - if err != nil { - return err - } - if arrayErr != nil { - return arrayErr - } - case jsonparser.String: - id, err := j.valueKeys.ID(data) - if err != nil { - return err - } - return cb(id) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting key, got numeric value") - } - id, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - return cb(id) - default: - return fmt.Errorf("expecting value or array, got %v", dataType) - } - return err -} - -// DecodeSetValue decodes a set of unsigned values from the provided data into the -// associated currentOp. -func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { - return j.decodeSetOrValue(dataType, data, func(id uint64) error { - j.currentOp.AddPair(recID, id) - return nil - }) -} - -func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 1 || len(values) == 1 { - // simplify: just hand this off to a single-value case - return j.EncodeMutexValue(dst, values, signed, keys) - } - appendKeysJSON(dst, values, keys) - return nil -} - -// DecodeIntValue decodes a single signed value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String: - value, err := j.valueKeys.IntID(data) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, value) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting string key, got numeric value") - } - value, err := strconv.ParseInt(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, value) - default: - if j.valueKeys != nil { - return fmt.Errorf("expecting string key, got %v", dataType) - } else { - return fmt.Errorf("expecting integer value, got %v", dataType) - } - } - return nil -} - -func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 0 { - if len(signed) == 0 { - return errors.New("encodeIntValue: need a value") - } - dst.EncodeInt(signed[0]) - return nil - } - dst.EncodeString(keys[0]) - return nil -} - -// DecodeMutexValue decodes a single unsigned value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String: - value, err := j.valueKeys.ID(data) - if err != nil { - return err - } - j.currentOp.AddPair(recID, value) - case jsonparser.Number: - if j.valueKeys != nil { - return errors.New("expecting string key, got numeric value") - } - value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) - if err != nil { - return err - } - j.currentOp.AddPair(recID, value) - default: - if j.valueKeys != nil { - return fmt.Errorf("expecting string key, got %v", dataType) - } else { - return fmt.Errorf("expecting integer value, got %v", dataType) - } - } - return nil -} - -func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(keys) == 0 { - if len(values) == 0 { - return errors.New("encodeMutexValue: need a value") - } - dst.EncodeUint(values[0]) - return nil - } - dst.EncodeString(keys[0]) - return nil -} - -// DecodeBoolValue decodes a single true/false value from the provided data -// into the associated currentOp. -func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { - value := uint64(0) - switch typ { - case jsonparser.String: - if string(data) != "true" && string(data) != "false" { - return fmt.Errorf("expecting boolean, got %q", data) - } - // if it's exactly "true" or "false" let's be forgiving - fallthrough - case jsonparser.Boolean: - if data[0] == 't' { - value = 1 - } - case jsonparser.Number: - v, err := jsonparser.GetInt(data) - if err != nil { - return err - } - if v == 1 { - value = 1 - } else if v != 0 { - return errors.New("boolean should be true/false/0/1") - } - default: - return errors.New("boolean should be true/false/0/1") - } - j.currentOp.AddPair(recID, value) - return nil -} - -func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(values) == 0 { - return errors.New("encoding boolean value, but none provided") - } - var x bool - if values[0] != 0 { - x = true - } - dst.EncodeBool(x) - return nil -} - -// DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the -// provided data into the associated currentOp. -func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { - j.scratch = j.scratch[:0] - stamp := time.Unix(0, 0).UTC() - err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { - switch string(key) { - case "time": - switch dataType { - case jsonparser.String: - stamp, err = time.Parse(time.RFC3339, pretendByteIsString(value)) - case jsonparser.Number: - var unix int64 - unix, err = strconv.ParseInt(pretendByteIsString(value), 10, 64) - stamp = time.Unix(unix, 0) - default: - return fmt.Errorf("expecting time, got %q", value) - } - case "values": - err = j.decodeSetOrValue(dataType, value, func(id uint64) error { - j.scratch = append(j.scratch, id) - return nil - }) - } - return err - }) - if err != nil { - return err - } - if len(j.scratch) > 0 { - defer func() { - // mark these as consumed so if we get called - // again, and don't see a "values" key, we don't reuse them. - j.scratch = j.scratch[:0] - }() - unix := stamp.UnixNano() - for _, value := range j.scratch { - j.currentOp.AddStampedPair(recID, value, unix) - } - } - return nil -} - -// Our format does not allow a record to have multiple values set with -// different timestamps at the same time. We use the first timestamp for -// the whole set. -func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(values) == 0 { - return errors.New("encoding time quantum value: no value provided") - } - dst.WriteString(`{"values":`) - appendKeysJSON(dst, values, keys) - // a zero timestamp is idiomatic for no-time-provided, and i sort of - // hate that, but here we are. - if len(signed) > 0 && signed[0] != 0 { - _, _ = dst.WriteString(`,"time":`) - dst.EncodeTime(time.Unix(0, signed[0]).UTC()) - } - _, _ = dst.WriteString(`}`) - return nil -} - -// DecodeTimeValue will eventually work but right now it doesn't actually. -func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { - var stamp time.Time - switch dataType { - case jsonparser.String: - stamp, err = time.Parse(time.RFC3339Nano, pretendByteIsString(data)) - if err != nil { - return fmt.Errorf("parsing timestamp: %w", err) - } - j.currentOp.AddSignedPair(recID, TimestampToVal(j.timeUnit, stamp)-j.epoch) - case jsonparser.Number: - // We could in theory convert this to a time, then convert it - // back, by multiplying by scaleUnit, then dividing. Or... not. - i64, err := strconv.ParseInt(pretendByteIsString(data), 10, 64) - if err != nil { - return fmt.Errorf("parsing numeric timestamp: %w", err) - } - j.currentOp.AddSignedPair(recID, i64) - default: - return fmt.Errorf("expecting time, got %s", dataType) - } - return nil -} - -func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - if len(signed) == 0 { - return errors.New("encoding time value: no value provided") - } - t, err := ValToTimestamp(j.timeUnit, signed[0]+j.epoch) - if err != nil { - return errors.Wrap(err, "translating value to timestamp") - } - dst.EncodeTime(t) - return nil -} - -// DecodeDecimalValue will eventually work but right now it doesn't actually. -func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { - switch dataType { - case jsonparser.String, jsonparser.Number: - value, err := jsonparser.GetFloat(data) - if err != nil { - return err - } - j.currentOp.AddSignedPair(recID, int64(value*float64(j.scaleUnit))) - default: - return fmt.Errorf("expecting floating-point value, got %v", dataType) - } - - return nil -} - -func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { - j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10) - scale := int(j.scale) - if len(j.buf) > scale { - j.buf = append(j.buf, '.') - // shove the last scaleUnit values over - copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:]) - j.buf[len(j.buf)-scale-1] = '.' - } - _, _ = dst.Write(j.buf) - return nil -} - -// FieldTypes gives a mapping of fields to their basic types used by this -// codec. -func (codec *JSONCodec) FieldTypes() map[string]FieldType { - return codec.fieldTypes -} - -// ParseKeyedRecords parses the records it finds. Note that, when you're doing -// a Write op, this will update ClearRecordIDs automatically as it goes, -// even though record_ids isn't specified in the JSON for that case. -func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { - seen := make(map[uint64]struct{}) - return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - id, err := codec.recKeys.ID(key) - if err != nil { - return err - } - if _, ok := seen[id]; ok { - return fmt.Errorf("key %q duplicated in input", key) - } - seen[id] = struct{}{} - if codec.currentOp.OpType == OpWrite { - codec.currentOp.ClearRecordIDs = append(codec.currentOp.ClearRecordIDs, id) - } - return jsonparser.ObjectEach(value, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - fieldCodec, ok := codec.fields[string(key)] - if !ok { - return errFieldNotFound{string(key)} - } - err := fieldCodec.decode(id, dataType, value) - if err != nil { - return fmt.Errorf("parsing value for field %q: %v", key, err) - } - return nil - }) - }) -} - -func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) { - op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq} - codec.currentOp = op - err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { - switch string(key) { - case "action": - op.OpType, err = ParseOpType(string(value)) - if err != nil { - return fmt.Errorf("unknown action type %q", value) - } - case "records": - for name, fieldCodec := range codec.fields { - fieldOp := &FieldOperation{} - op.FieldOps[name] = fieldOp - // cache this op so we don't have to do the lookups every time - fieldCodec.currentOp = fieldOp - } - err = codec.ParseKeyedRecords(value) - if err != nil { - return fmt.Errorf("parsing records: %v", err) - } - case "record_ids": - var id uint64 - var idErr error - _, err = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - id, idErr = codec.recKeys.ID(value) - if idErr != nil { - return - } - op.ClearRecordIDs = append(op.ClearRecordIDs, id) - - }) - // if we got an error converting an ID, error out with it here. - // we can't stop the ArrayEach early, though? - if idErr != nil { - return idErr - } - if err != nil { - return err - } - case "fields": - _, err = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - op.ClearFields = append(op.ClearFields, string(value)) - }) - if err != nil { - return err - } - default: - return fmt.Errorf("unknown operation field %q", key) - } - return nil - }) - if err != nil { - return nil, err - } - if op.OpType == OpNone { - return nil, fmt.Errorf("action not specified") - } - return op, err -} - -// Parse reads a request, but does not sort the results at all or divide -// them into shards. -func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - return codec.ParseBytes(data) -} - -// ParseBytes reads a request from a slice of bytes. We need to use a -// byte slice because jsonparser's model is fundamentally built around -// being able to random-access the slice and return slices of it, so -// it can't really admit functional streaming. If we need streaming, the -// streaming needs to be at a higher level. -func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { - var ops []*Operation - var lastErr error - var seq int - _, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { - switch dataType { - case jsonparser.Object: - op, err := codec.ParseOperation(value, seq) - seq++ - if err != nil { - lastErr = fmt.Errorf("parsing operation: %v", err) - return - } - ops = append(ops, op) - default: - lastErr = fmt.Errorf("expected operation, found %s", dataType) - } - }) - if lastErr != nil { - return nil, lastErr - } - if err != nil { - return nil, err - } - // and now, key translation! - var keyMap []uint64 - if codec.keys != nil { - keyMap, err = codec.recKeys.MakeIDMap(codec.keys) - if err != nil { - return nil, fmt.Errorf("trying to find record key mapping: %w", err) - } - } - req = &Request{} - valueMaps := map[string]func(*FieldOperation) error{} - for name, fieldCodec := range codec.fields { - // make closure survive iteration - fieldCodec := fieldCodec - if fieldCodec.keys != nil { - fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys) - if err != nil { - return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err) - } - if fieldCodec.fieldType == FieldTypeInt { - valueMaps[name] = func(fo *FieldOperation) error { - return translateSigned(fieldMap, fo.Signed) - } - } else { - valueMaps[name] = func(fo *FieldOperation) error { - return translateUnsigned(fieldMap, fo.Values) - } - } - } - } - for _, op := range ops { - // For Clear and Write, we need to translate/sort our record ID - // list. - if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { - if keyMap != nil { - if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil { - return nil, fmt.Errorf("mapping record keys for clear op: %w", err) - } - } - } - // For clear/delete, that's all we need to do; there's no meaningful fieldops under them. - if op.OpType == OpClear || op.OpType == OpDelete { - continue - } - for field, fieldOp := range op.FieldOps { - if len(fieldOp.RecordIDs) == 0 { - delete(op.FieldOps, field) - continue - } - if keyMap != nil { - if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil { - return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err) - } - } - if fieldTranslate, ok := valueMaps[field]; ok { - if err = fieldTranslate(fieldOp); err != nil { - return nil, fmt.Errorf("mapping values for op on %q: %w", field, err) - } - } - // Sort by column keys, for now. - // Write op will also want to clear every field we saw. - if op.OpType == OpWrite { - op.ClearFields = append(op.ClearFields, field) - } - } - } - req.Ops = ops - return req, nil -} - -// AppendBytes appends bytes which we would expect to produce the same -// request, using this codec. It does not try to recreate FieldTypes, which -// would be handled by the codec anyway. It's written as an Append so you -// can reuse a buffer. -func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) { - if req == nil || len(req.Ops) == 0 { - return append(data, "[]"...), nil - } - dst := newJSONBuffer(data) - // we ignore the FieldTypes part of the Request, which is just there to - // let the request's ByShard use the correct sorting routines, which is - // itself sort of awful. The codec will insert it again when parsing the - // bytes. - _, _ = dst.WriteString(`,`) - for _, op := range req.Ops { - // EncodeJSON needs to have access to this codec's field data - // and key translators. - err = op.EncodeJSON(dst, codec) - if err != nil { - return nil, err - } - _, _ = dst.WriteString(`,`) - } - // delete the last comma - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`]`) - // there's very few ways an error could occur, possibly none, but - // just in case lots of internal operations stashed an error if one - // happened, so we'll return anything they came up with. - return dst.Bytes(), dst.Err() -} - -// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed -// list of quoted strings if there's a key translator, or integer values -// if the KeyTranslator is nil, comma-separated. It can error because a -// translator can error. -func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { - return appendIDsJSON(dst, values, keys) -} - -func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) { - if len(values) == 0 && len(keys) == 0 { - _, _ = dst.WriteString("[]") - return - } - if len(keys) != 0 { - dst.EncodeStrings(keys) - } else { - dst.EncodeUints(values) - } -} - -func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { - if len(values) == 0 { - _, _ = dst.WriteString("[]") - return nil - } - - if keys == nil { - dst.EncodeUints(values) - return nil - } - _, _ = dst.WriteString(`[`) - translated, err := keys.TranslateIDs(values...) - if err != nil { - return err - } - for _, v := range values { - dst.EncodeString(translated[v]) - _, _ = dst.WriteString(`,`) - } - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`]`) - return nil -} - -// RequestByShard makes up for the fact that we don't want to stash the -// field type data in the request, but we need it to actually do the by-shard. -func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) { - return req.ByShard(codec.fieldTypes) -} - -// EncodeJSON encodes an operation as JSON using a provided buffer. -// It uses the provided codec where necessary to help with key -// translation. -func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) { - // We're not trying to guarantee that what we produce makes sense or is - // identical to what produced us, just to write our current state out. - if o == nil { - _, _ = dst.WriteString(`{}`) - return nil - } - _, _ = dst.WriteString(`{"action":`) - dst.EncodeString(o.OpType.String()) - // it is intentional that o.Seq isn't encoded here; it makes no sense - // to allow an op to specify its seq in JSON. - if o.OpType != OpWrite { - // for Write, ClearRecordIDs and ClearFields were computed - // from the records being written, and aren't actually part of the data. - if len(o.ClearRecordIDs) != 0 { - _, _ = dst.WriteString(`,"record_ids":`) - err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys) - if err != nil { - return err - } - } - if len(o.ClearFields) != 0 { - _, _ = dst.WriteString(`,"fields":`) - dst.EncodeStrings(o.ClearFields) - } - } - if len(o.FieldOps) == 0 { - _, _ = dst.WriteString(`}`) - return - } - // collect all the fields that have a non-empty set of records. we can - // just ignore the others. - fieldNames := make([]string, 0, len(o.FieldOps)) - for field, op := range o.FieldOps { - if op != nil && len(op.RecordIDs) != 0 { - fieldNames = append(fieldNames, field) - } - } - // nevermind then - if len(fieldNames) == 0 { - _, _ = dst.WriteString(`}`) - return nil - } - _, _ = dst.WriteString(`,"records":{`) - - // now we have to invert the logic, creating records from - // fields with corresponding ops. uh-oh. - fieldOps := make([]*FieldOperation, len(o.FieldOps)) - fieldCodecs := make([]*fieldCodec, len(o.FieldOps)) - fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys - indexes := make([]int, len(o.FieldOps)) - sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] }) - next := ^uint64(0) - var idKeys map[uint64]string - if codec.keys != nil { - idKeys = make(map[uint64]string) - } - for i, field := range fieldNames { - // populate a parallel slice of field ops so we don't have - // to do map lookups for every single piece of data - op := o.FieldOps[field] - fieldOps[i] = op - fc := codec.fields[field] - if fc == nil { - return fmt.Errorf("unknown field: %q", field) - } - fieldCodecs[i] = fc - if codec.keys != nil { - // we'll build a list of keys we need for any records - for _, v := range op.RecordIDs { - idKeys[v] = "" - } - } - id := op.RecordIDs[0] - if id < next { - next = id - } - if fc.keys != nil { - thisFieldKeys := make([]string, len(op.RecordIDs)) - if fc.fieldType == FieldTypeInt { - u := make([]uint64, len(op.Signed)) - for i := range op.Signed { - u[i] = uint64(op.Signed[i]) - } - valueKeys, err := fc.keys.TranslateIDs(u...) - if err != nil { - return err - } - for j, v := range op.Signed { - thisFieldKeys[j] = valueKeys[uint64(v)] - } - } else { - valueKeys, err := fc.keys.TranslateIDs(op.Values...) - if err != nil { - return err - } - for j, v := range op.Values { - thisFieldKeys[j] = valueKeys[v] - } - } - fieldKeys[i] = thisFieldKeys - } - } - // ... no fieldops actually have any entries. therefore no record will - // have any fields set, therefore no records exist... - if next == ^uint64(0) { - _, _ = dst.WriteString(`}}`) - return nil - } - if codec.keys != nil { - idList := make([]uint64, 0, len(idKeys)) - for k := range idKeys { - idList = append(idList, k) - } - idKeys, err = codec.keys.TranslateIDs(idList...) - if err != nil { - return err - } - } - - // next is the ID of a record which exists for at least one field. - // we'll recompute it every loop. - for next != ^uint64(0) { - if codec.keys != nil { - dst.EncodeString(idKeys[next]) - } else { - dst.EncodeQuotedUint(next) - } - _, _ = dst.WriteString(`:{`) - current := next - next = ^uint64(0) - for i, field := range fieldNames { - op := fieldOps[i] - idx := indexes[i] - if idx >= len(op.RecordIDs) { - continue - } - id := op.RecordIDs[idx] - if id == current { - var j int - // count ahead to first index which is either outside the list - // or a different id - for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ { - } - - // field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i])) - // print this one, and advance this index to next position - dst.EncodeString(field) - _, _ = dst.WriteString(`:`) - var values []uint64 - var signed []int64 - var keys []string - if len(op.Values) >= j { - values = op.Values[idx:j] - } - if len(op.Signed) >= j { - signed = op.Signed[idx:j] - } - if len(fieldKeys[i]) >= j { - keys = fieldKeys[i][idx:j] - } - err = fieldCodecs[i].encode(dst, values, signed, keys) - if err != nil { - return err - } - _, _ = dst.WriteString(`,`) - indexes[i] = j - // and we'll fall through to the id < next check, so we - // just set id here. - if indexes[i] < len(op.RecordIDs) { - id = op.RecordIDs[indexes[i]] - } else { - id = ^uint64(0) - } - } - if id < next { - next = id - } - } - // we should always have a trailing comma after any entry, and - // if there were no entries we shouldn't have had a list... - dst.Truncate(dst.Len() - 1) - _, _ = dst.WriteString(`},`) - } - dst.Truncate(dst.Len() - 1) - // close both the records list, and the whole object. - _, _ = dst.WriteString(`}}`) - // In theory, there's no way for most of these to produce errors, - // but just in case, we'll check for an error every operation or so. - return dst.Err() -} - -type errFieldNotFound struct { - field string -} - -func (err errFieldNotFound) Error() string { - return fmt.Sprintf("field not found: %q", err.field) -} - -// TimestampToVal takes a time unit and a time.Time and converts it to an integer value -func TimestampToVal(unit string, ts time.Time) int64 { - switch unit { - case "s": - return ts.Unix() - case "ms": - return ts.UnixMilli() - case "us": - return ts.UnixMicro() - case "ns": - return ts.UnixNano() - } - return 0 - -} - -// ValToTimestamp takes a timeunit and an integer value and converts it to time.Time -func ValToTimestamp(unit string, val int64) (time.Time, error) { - switch unit { - case "s": - return time.Unix(val, 0).UTC(), nil - case "ms": - return time.UnixMilli(val).UTC(), nil - case "us", "μs": - return time.UnixMicro(val).UTC(), nil - case "ns": - return time.Unix(0, val).UTC(), nil - default: - return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit) - } -} diff --git a/ingest/codec_test.go b/ingest/codec_test.go deleted file mode 100644 index 838328981..000000000 --- a/ingest/codec_test.go +++ /dev/null @@ -1,978 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/molecula/featurebase/v3/shardwidth" -) - -func TestStableTranslator(t *testing.T) { - tr := newStableTranslator() - m1, err := tr.TranslateKeys("a", "b") - if err != nil { - t.Fatalf("translation error on initial keys: %v", err) - } - m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6) - if err != nil { - t.Fatalf("translation error on reverse lookup: %v", err) - } - m3, err := tr.TranslateKeys("a", "k-6") - if err != nil { - t.Fatalf("translation error on new keys: %v", err) - } - for k, v := range m3 { - if m2[v] != k { - t.Fatalf("expected round trip to equate %q and %d", k, v) - } - } -} - -func TestMakeCodec(t *testing.T) { - codec, _ := NewJSONCodec(nil) - err := codec.AddSetField("set", nil) - if err != nil { - t.Fatalf("unexpected error creating field: %v", err) - } - err = codec.AddSetField("set", nil) - if err == nil { - t.Fatalf("expected error creating duplicate field, didn't get it") - } -} - -func TestEncode(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - codecs := []*JSONCodec{codec} - - // redo all of that, only on a keyed translator - codec, _ = NewJSONCodec(newStableTranslator()) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - codecs = append(codecs, codec) - - encodeTests := []*Request{ - { - Ops: []*Operation{ - { - OpType: OpWrite, - ClearRecordIDs: []uint64{0, 1, 2, 3, 5}, - ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"}, - FieldOps: map[string]*FieldOperation{ - "int": { - RecordIDs: []uint64{0, 1}, - Signed: []int64{1, -3}, - }, - "intkeys": { - RecordIDs: []uint64{0, 1}, - Signed: []int64{1, 1}, - }, - "set": { - RecordIDs: []uint64{0, 1, 2, 2}, - Values: []uint64{1, 1, 0, 1}, - }, - "setkeys": { - RecordIDs: []uint64{0, 1, 2, 2}, - Values: []uint64{1, 1, 0, 1}, - }, - "mutex": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{1, 2}, - }, - "mutexkeys": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{1, 2}, - }, - "tq": { - RecordIDs: []uint64{5, 5}, - Values: []uint64{8, 9}, - Signed: []int64{1234567890e9, 1234567890e9}, - }, - "tqkeys": { - RecordIDs: []uint64{3, 3}, - Values: []uint64{2, 4}, - Signed: []int64{1234567890e9, 1234567890e9}, - }, - "ts": { - RecordIDs: []uint64{0}, - Signed: []int64{1}, - }, - "bool": { - RecordIDs: []uint64{0, 1}, - Values: []uint64{0, 1}, - }, - "dec": { - RecordIDs: []uint64{0, 1, 2}, - Signed: []int64{123, -123, 0}, - }, - }, - }, - { - OpType: OpClear, - Seq: 1, - ClearRecordIDs: []uint64{6}, - ClearFields: []string{"tq"}, - }, - }, - }, - { - // this one needs to get filled in programmatically; see below - Ops: []*Operation{ - { - OpType: OpSet, - FieldOps: map[string]*FieldOperation{}, - }, - }, - }, - { - Ops: []*Operation{ - { - OpType: OpRemove, - FieldOps: map[string]*FieldOperation{ - "set": {}, - }, - }, - }, - }, - } - // and now we populate encodeTests[1] with a larger pool of data - const dataSize = 5000 - shardCount := uint64(600) // shards we want to target - passes := uint64(0) - recordIDs := make([]uint64, dataSize) - values := make([]uint64, dataSize) - timeStamps := make([]int64, dataSize) - signedValues := make([]int64, dataSize) - for i := uint64(0); i < dataSize; i++ { - if (i % shardCount) == 0 { - passes++ - } - recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes - values[i] = (i % 4) - timeStamps[i] = int64(1234567890e9 + (i * 100e9)) - signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys - } - // ensure record IDs are sorted, because other stuff might rely on this - sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) - op := encodeTests[1].Ops[0] - op.FieldOps["tq"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: append([]uint64{}, values...), - Signed: append([]int64{}, timeStamps...), - } - op.FieldOps["tqkeys"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: values, - Signed: timeStamps, - } - op.FieldOps["int"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Signed: append([]int64{}, signedValues...), - } - op.FieldOps["intkeys"] = &FieldOperation{ - RecordIDs: recordIDs, - Signed: signedValues, - } - // for sets, we want to shuffle things into fewer shards, and ensure - // non-duplication of values within each record, but also have lots - // of duplication of record IDs in the low shards - recordIDs = make([]uint64, dataSize) - values = make([]uint64, dataSize) - valuesPerRecord := uint64(5) - recordsPerShard := dataSize / valuesPerRecord / 30 - if recordsPerShard < 1 { - recordsPerShard = 1 - } - shard := uint64(0) - nextID := uint64(0) - nextValue := uint64(0) - for i := uint64(0); i < dataSize; i++ { - recordIDs[i] = nextID - values[i] = nextValue + (i % valuesPerRecord) - nextValue++ - if nextValue == valuesPerRecord { - nextValue = 0 - nextID++ - if nextID%(1< 1 { - valuesPerRecord-- - } - } - } - } - sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) - op.FieldOps["set"] = &FieldOperation{ - RecordIDs: append([]uint64{}, recordIDs...), - Values: append([]uint64{}, values...), - } - op.FieldOps["setkeys"] = &FieldOperation{ - RecordIDs: recordIDs, - Values: values, - } - var buf []byte - for i, tc := range encodeTests { - for _, c := range codecs { - data, err := c.AppendBytes(tc, buf[:0]) - if err != nil { - t.Fatalf("encode test %d: error encoding: %v", i, err) - } - // t.Logf("data:\n%s", data) - req, err := c.ParseBytes(data) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: error parsing: %v", i, err) - } - err = req.Compare(tc) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) - } - data, err = c.AppendBytes(req, buf[:0]) - if err != nil { - t.Fatalf("encode test %d: error encoding: %v", i, err) - } - // t.Logf("data:\n%s", data) - req2, err := c.ParseBytes(data) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: error parsing: %v", i, err) - } - err = req2.Compare(tc) - if err != nil { - t.Logf("encode test %d: data:\n%s", i, data) - t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) - } - } - } -} - -func TestCodecErrors(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - - testCases := []struct { - name string - json []byte - error string - }{ - { - name: "no action", - json: []byte(`[{"records":{"0":{"set":[0]}}}]`), - error: "action not specified", - }, - { - name: "unknown action", - json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`), - error: "unknown action", - }, - { - name: "unknown field", - json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`), - error: "field not found", - }, - { - name: "unknown operation field", - json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`), - error: "unknown operation field", - }, - { - name: "expected operation", - json: []byte(`[true]`), - error: "expected operation", - }, - { - name: "expecting key", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`), - error: "expecting key", - }, - { - name: "invalid int for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`), - error: "boolean should be", - }, - { - name: "invalid number for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`), - error: "looks like Number", - }, - { - name: "invalid string for bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`), - error: "expecting boolean", - }, - { - name: "nonsense bool", - json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`), - error: "boolean should be", - }, - { - name: "expecting numeric value", - json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting value", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`), - error: "expecting value", - }, - { - name: "expecting array-key", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`), - error: "expecting key", - }, - { - name: "expecting array-value", - json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`), - error: "expecting value", - }, - { - name: "expecting numeric array-value", - json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`), - error: "invalid syntax", - }, - { - name: "expecting numeric value", - json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting int key", - json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`), - error: "expecting string key", - }, - { - name: "expecting int value", - json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`), - error: "expecting integer value", - }, - { - name: "expecting string array-value", - json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`), - error: "expecting string key", - }, - { - name: "expecting numeric mutex value", - json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`), - error: "invalid syntax", - }, - { - name: "expecting mutex key", - json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`), - error: "expecting string key", - }, - { - name: "expecting mutex value", - json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`), - error: "expecting integer value", - }, - { - name: "expecting mutex string value", - json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`), - error: "expecting string key", - }, - { - name: "time quantum invalid time", - json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`), - error: "expecting time", - }, - { - name: "time stamp invalid integer", - json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`), - error: "parsing numeric time", - }, - { - name: "time stamp invalid string", - json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`), - error: "parsing time", - }, - { - name: "time stamp invalid type", - json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`), - error: "expecting time", - }, - { - name: "invalid decimal", - json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`), - error: "expecting floating", - }, - { - name: "duplicate record", - json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`), - error: "duplicated in input", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - req, err := codec.ParseBytes(tc.json) - if err == nil { - req.Dump(t.Logf) - t.Fatalf("expected error like %q, got request instead", tc.error) - } else { - msg := err.Error() - if !strings.Contains(msg, tc.error) { - t.Fatalf("expected error like %q, got %q", tc.error, msg) - } - } - }) - } -} - -func TestSimpleCodec(t *testing.T) { - codec, _ := NewJSONCodec(nil) - _ = codec.AddSetField("set", nil) - _ = codec.AddSetField("setkeys", newStableTranslator()) - _ = codec.AddMutexField("mutex", nil) - _ = codec.AddMutexField("mutexkeys", newStableTranslator()) - _ = codec.AddTimeQuantumField("tq", nil) - _ = codec.AddIntField("int", nil) - _ = codec.AddIntField("intkeys", newStableTranslator()) - epoch, err := time.Parse("2006-01-02", "2020-01-01") - if err != nil { - t.Fatalf("can't parse sample epoch time: %v", err) - } - _ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000) - _ = codec.AddDecimalField("dec", 2) - _ = codec.AddBoolField("bool") - var nextShard = uint64(1<> shardwidth.Exponent: f} - } - output := make(ShardedFieldOperation) - sortToShardsInto(f, bitsRemaining-8, output) - return output -} - -// sortToShardsInto puts the shards it finds into the given map, so that -// as we split off buckets, they can be inserted into the same map. -func sortToShardsInto(f *FieldOperation, shift int, into ShardedFieldOperation) { - if shift < shardwidth.Exponent { - shift = shardwidth.Exponent - } - nextShift := shift - 8 - if nextShift < shardwidth.Exponent { - nextShift = shardwidth.Exponent - } - // count things that belong in each of the 256 buckets - var buckets [256]int - var starts [256]int - - // compute the buckets ourselves - for _, r := range f.RecordIDs { - b := (r >> shift) & 0xFF - buckets[b]++ - } - total := 0 - // compute starting points of each bucket, converting the - // bucket counts into ends - for i := range buckets { - starts[i] = total - total += buckets[i] - buckets[i] = total - } - // starts[n] is the index of the first thing that should - // go in that bucket, buckets[n] is the index of the first - // thing that shouldn't - var bucketOp FieldOperation - for bucket, start := range starts { - end := buckets[bucket] - if end <= start { - continue - } - for j := start; j < end; j++ { - want := int((f.RecordIDs[j] >> shift) & 0xFF) - for want != bucket { - // move this to the beginning of the - // bucket it wants to be in, swapping - // the thing there here - dst := starts[want] - f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j] - if f.Values != nil { - f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j] - } - if f.Signed != nil { - f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j] - } - starts[want]++ - want = int((f.RecordIDs[j] >> shift) & 0xFF) - } - } - // If shift == shardwidth.Exponent, then this is a completed - // shard and can go into the sharded output. otherwise, we - // can subdivide it. - bucketOp.RecordIDs = f.RecordIDs[start:end] - if f.Values != nil { - bucketOp.Values = f.Values[start:end] - } - if f.Signed != nil { - bucketOp.Signed = f.Signed[start:end] - } - if shift == shardwidth.Exponent { - x := bucketOp - into[f.RecordIDs[start]>>shardwidth.Exponent] = &x - } else { - sortToShardsInto(&bucketOp, nextShift, into) - } - } -} - -const shardMask = ((uint64(1) << shardwidth.Exponent) - 1) - -// SortByValues sorts the operation by values first, then by record -// ID within each value. This is the best ordering for set/mutex fields, -// where we'll want to generate positions in that order. For these -// purposes, a time quantum or bool counts as a kind of a set. -func (f *FieldOperation) SortByValues() { - keys := make([]uint64, len(f.RecordIDs)) - for i, v := range f.RecordIDs { - keys[i] = (f.Values[i] << shardwidth.Exponent) | (v & shardMask) - } - f.SortByKeys(keys) -} - -// SortByRecords sorts the operation by record ID, and not by value at -// all. This makes the most sense for int fields and the like. -func (f *FieldOperation) SortByRecords() { - f.SortByKeys(f.RecordIDs) -} - -// SortByKeys reorganizes the record IDs and values of f according to the -// corresponding members of keys. -func (f *FieldOperation) SortByKeys(keys []uint64) { - if len(f.RecordIDs) < 2 { - return - } - diffMask := uint64(0) - prev := keys[0] - for _, r := range keys[1:] { - diffMask |= r ^ prev - prev = r - } - bitsRemaining := bits.Len64(diffMask) - sortPartialByKeys(f, keys, bitsRemaining-8) -} - -// simpleSort sorts a FieldOperation by external keys, or record IDs. It's a -// horribly naive bubble sort because N is small and a more complex algorithm -// doesn't help as much as you'd hope. This beats using stdlib sort by about -// a factor of two for those small N, for larger N we're using the radix sort -// that calls this. -// -// External keys exist only when we are sorting by value, which is to say, -// when we're using row-oriented formats (set, mutex, time quantum). -// For int/decimal/timestamp fields, we're sorting by record only. -// So, if keys is the same as f.RecordIDs, we're looking at an int field -// or equivalent, so Signed exists and Values doesn't exist. -// Otherwise, we might be looking at a time quantum field (both exist) -// or set/mutex (only Values exist). -func simpleSort(f *FieldOperation, keys []uint64) { - // keys might actually just point to record IDs, in which case, we don't - // want to shuffle the corresponding RecordIDs too, because that would just - // reverse our swaps. If they're different, we actually need to swap them - // both. - if &keys[0] != &f.RecordIDs[0] { - // sorting by record IDs - if f.Values != nil && f.Signed != nil { // time quantum field - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - - } - } - } else if f.Values != nil { // set/mutex/bool - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - } - } - } else if f.Signed != nil { // can't-happen, we think - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else { // can't-happen, we think - for i := 1; i < len(keys); i++ { - for j := i; j > 0 && keys[j-1] > keys[j]; j-- { - keys[j-1], keys[j] = keys[j], keys[j-1] - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - } - } - } - } else { - if f.Values == nil && f.Signed != nil { - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else if f.Values != nil && f.Signed != nil { // can't happen, we think - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else if f.Values != nil { // only happens during testing - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] - } - } - } else { // should definitely not happen - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - } - } - } - } -} - -func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) { - if shift < 0 { - shift = 0 - } - externalKeys := &f.RecordIDs[0] != &keys[0] - nextShift := shift - 8 - if nextShift < 0 { - nextShift = 0 - } - // count things that belong in each of the 256 buckets - var buckets [256]int - var starts [256]int - // compute the buckets ourselves - for _, r := range keys { - b := (r >> shift) & 0xFF - buckets[b]++ - } - total := 0 - // compute starting points of each bucket, converting the - // bucket counts into ends - for i := range buckets { - starts[i] = total - total += buckets[i] - buckets[i] = total - } - // starts[n] is the index of the first thing that should - // go in that bucket, buckets[n] is the index of the first - // thing that shouldn't - // var newbuckets [256]int - var bucketOp FieldOperation - for bucket, start := range starts { - end := buckets[bucket] - if end <= start { - continue - } - for j := start; j < end; j++ { - want := int((keys[j] >> shift) & 0xFF) - for want != bucket { - // move this to the beginning of the - // bucket it wants to be in, swapping - // the thing there here - dst := starts[want] - keys[j], keys[dst] = keys[dst], keys[j] - // we do this to allow you to just pass in the records as keys - if externalKeys { - f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j] - } - if f.Values != nil { - f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j] - } - if f.Signed != nil { - f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j] - } - starts[want]++ - want = int((keys[j] >> shift) & 0xFF) - } - } - // If shift == shardwidth.Exponent, then this is a completed - // shard and can go into the sharded output. otherwise, we - // can subdivide it. - if shift > 0 { - bucketOp.RecordIDs = f.RecordIDs[start:end] - if f.Values != nil { - bucketOp.Values = f.Values[start:end] - } - if f.Signed != nil { - bucketOp.Signed = f.Signed[start:end] - } - // if there's not very many, sort naively instead - if end-start > 32 { - sortPartialByKeys(&bucketOp, keys[start:end], nextShift) - } else { - simpleSort(&bucketOp, keys[start:end]) - } - } - } -} - -// AddPair adds a record ID/value pair where the value is unsigned, as -// when used with set/mutex/time quantum fields. -func (f *FieldOperation) AddPair(rec uint64, value uint64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Values = append(f.Values, value) -} - -// AddSignedPair adds a record ID/value pair where the value is signed, -// as when used with int/decimal/timestamp fields. -func (f *FieldOperation) AddSignedPair(rec uint64, value int64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Signed = append(f.Signed, value) -} - -// AddStampedPair adds a record/value pair plus a time, which is just -// a Unix time in seconds. (Note, no scaling here; timestamp fields are -// scaled int fields, this is for time quantums.) -func (f *FieldOperation) AddStampedPair(rec uint64, value uint64, stamp int64) { - f.RecordIDs = append(f.RecordIDs, rec) - f.Values = append(f.Values, value) - f.Signed = append(f.Signed, stamp) -} - -// Compare returns a diagnostic if the field operations do not seem -// equivalent. -func (got *FieldOperation) Compare(expected *FieldOperation) error { - if got == nil { - if expected == nil { - return nil - } - // We don't worry about non-empty Values or Signed here, because in theory - // RecordIDs are the Source of Truth as to what's in the op. - if len(expected.RecordIDs) == 0 { - return nil - } - return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs)) - } - if expected == nil { - if len(got.RecordIDs) == 0 { - return nil - } - return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs)) - } - if len(got.RecordIDs) != len(expected.RecordIDs) { - return fmt.Errorf("record counts differ: expected %d, got %d", len(expected.RecordIDs), len(got.RecordIDs)) - } - for i, v1 := range got.RecordIDs { - v2 := expected.RecordIDs[i] - if v1 != v2 { - return fmt.Errorf("record id %d differs: expected %d, got %d", i, v2, v1) - } - } - if len(got.Values) != len(expected.Values) { - return fmt.Errorf("value counts differ: expected %d, got %d", len(expected.Values), len(got.Values)) - } - for i, v1 := range got.Values { - v2 := expected.Values[i] - if v1 != v2 { - return fmt.Errorf("value %d differs: expected %d, got %d", i, v2, v1) - } - } - if len(got.Signed) != len(expected.Signed) { - return fmt.Errorf("signed value counts differ: expected %d, got %d", len(expected.Signed), len(got.Signed)) - } - for i, v1 := range got.Signed { - v2 := expected.Signed[i] - if v1 != v2 { - return fmt.Errorf("signed value %d differs: expected %d, got %d", i, v2, v1) - } - } - return nil -} - -// ShardOperations is a set of Operations associated with a specific shard. -type ShardOperations struct { - Shard uint64 - Ops []*Operation -} - -// Request is a complete ingest request, which may be any combination -// of operations, which may apply to multiple shards. -type Request struct { - Ops []*Operation -} - -// ShardedRequest is an ingest request, split up into individual per-shard -// operations. -type ShardedRequest struct { - Ops map[uint64][]*Operation -} - -// ByShard converts a request into the same request, only sharded. -func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) { - if len(r.Ops) == 0 { - return &ShardedRequest{Ops: nil}, nil - } - req := make(map[uint64][]*Operation) - shards := make(map[uint64]*Operation) - // we're getting per-field things, which we want to divide per-shard, - // and return to per-shard sets of per-field things, so we're inverting - // the structure. - for _, op := range r.Ops { - // for clear and write ops, we also need to split up the - // ClearRecords values, which may be distinct from the set of - // records for any given field. For Write ops, we'll then end - // up adding in field values for some fields. - if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { - sharded := ShardIDs(op.ClearRecordIDs) - for shard, data := range sharded { - shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} - } - } - for field, fieldOp := range op.FieldOps { - sharded := fieldOp.ByShard() - sorter := fieldTypeSorts[fields[field]] - if sorter == nil { - sorter = (*FieldOperation).SortByRecords - } - for shard, data := range sharded { - sorter(data) - shardOp, ok := shards[shard] - if !ok { - if op.OpType == OpWrite { - return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard) - } - shardOp = &Operation{OpType: op.OpType, Seq: op.Seq} - shards[shard] = shardOp - shardOp.FieldOps = map[string]*FieldOperation{field: data} - } else { - shardOp.FieldOps[field] = data - } - } - } - for shard, shardOp := range shards { - req[shard] = append(req[shard], shardOp) - } - for k := range shards { - delete(shards, k) - } - - } - return &ShardedRequest{Ops: req}, nil -} - -// merge combines the components of a sharded request back into a single -// unsharded request, processing shards in numerical order. -func (s *ShardedRequest) merge() *Request { - req := &Request{} - if s == nil || len(s.Ops) == 0 { - return req - } - shards := make([]uint64, 0, len(s.Ops)) - for shard := range s.Ops { - shards = append(shards, shard) - } - sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] }) - for _, shard := range shards { - ops := s.Ops[shard] - for _, op := range ops { - var _ *Operation - if op.Seq >= len(req.Ops) { - // Pad out with nil *Operations to the required length - req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...) - } - if req.Ops[op.Seq] == nil { - req.Ops[op.Seq] = op.clone() - continue - } - req.Ops[op.Seq].merge(op) - } - } - return req -} - -func (r *Request) Dump(logf func(string, ...interface{})) { - logf("req: %#v", r) - for _, op := range r.Ops { - logf("op: %#v", op) - if len(op.ClearRecordIDs) > 0 { - if len(op.ClearRecordIDs) > 8 { - logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8) - } else { - logf(" clearRecordIDs: %d", op.ClearRecordIDs) - } - } - if len(op.ClearFields) > 0 { - if len(op.ClearFields) > 8 { - logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8) - } else { - logf(" clearFields: %s", op.ClearFields) - } - } - for field, fieldOp := range op.FieldOps { - if fieldOp != nil { - logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed)) - if len(fieldOp.RecordIDs) > 0 { - if len(fieldOp.RecordIDs) > 8 { - logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8) - } else { - logf(" records %d", fieldOp.RecordIDs) - } - } - } else { - logf(" field %q: nil op", field) - } - } - } -} - -func (r *Request) Compare(other *Request) error { - if other == nil { - if r != nil && len(r.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - if r == nil { - if other != nil && len(other.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - ops := r.Ops - ops2 := other.Ops - if len(ops2) != len(ops) { - return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2)) - } - for i, op := range ops { - if err := op.Compare(ops2[i]); err != nil { - return fmt.Errorf("op %d: %v", i, err) - } - } - return nil -} - -// Compare checks whether two ShardedRequest objects represent the same -// data. Empty shards shouldn't have entries in the map in the first place, -// so we don't accept a nil or 0-length slice of ops as equal to the -// shard key not existing, but we do accept nil or empty requests as -// equal to each other. -func (s *ShardedRequest) Compare(other *ShardedRequest) error { - if other == nil { - if s != nil && len(s.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - if s == nil { - if other != nil && len(other.Ops) != 0 { - return errors.New("non-empty sharded request can't equal empty/nil sharded request") - } - // empty and nil are allowed - return nil - } - for shard, ops := range s.Ops { - ops2, ok := other.Ops[shard] - if !ok { - return fmt.Errorf("shard %d missing in other", shard) - } - if len(ops2) != len(ops) { - return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2)) - } - for i, op := range ops { - if err := op.Compare(ops2[i]); err != nil { - return fmt.Errorf("shard %d, op %d: %v", shard, i, err) - } - } - } - if len(other.Ops) != len(s.Ops) { - for shard := range other.Ops { - if _, ok := s.Ops[shard]; !ok { - return fmt.Errorf("shard %d missing in self", shard) - } - } - } - return nil -} diff --git a/ingest/op_test.go b/ingest/op_test.go deleted file mode 100644 index 31e1c56ff..000000000 --- a/ingest/op_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "math/rand" - "testing" - - "github.com/molecula/featurebase/v3/shardwidth" -) - -type opShardingTestCase struct { - name string - input *Request - output *ShardedRequest -} - -var opShardingTestCases = []opShardingTestCase{ - { - name: "sample", - input: &Request{ - Ops: []*Operation{ - { - OpType: OpSet, - FieldOps: map[string]*FieldOperation{ - "shard0": { - RecordIDs: []uint64{0, 1}, - }, - "shard0-1": { - RecordIDs: []uint64{ - 0, - 1 << shardwidth.Exponent, - }, - }, - "shard1": { - RecordIDs: []uint64{ - 1 << shardwidth.Exponent, - 1< 1 { - valuesPerRecord-- - } - } - } - } - op.FieldOps["set"] = &FieldOperation{ - RecordIDs: recordIDs, - Values: values, - } - fieldTypes := codec.FieldTypes() - sharded, err := req.ByShard(fieldTypes) - if err != nil { - t.Errorf("sharding: unexpected error %v", err) - } - merged := sharded.merge() - if err := req.Compare(merged); err != nil { - t.Fatalf("merge comparison: %v", err) - } -} - -func TestFancySharding(t *testing.T) { - const shardLimit = 700 - const recordCount = 5000 - grr := rand.New(rand.NewSource(0)) - for i := 0; i < 100; i++ { - f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} - shards := make([]int, shardLimit) - for j := range f.RecordIDs { - v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent)) - f.RecordIDs[j] = v - f.Values[j] = uint64(grr.Int63n(8)) - shards[v>>shardwidth.Exponent]++ - } - - sharded := f.SortToShards() - for shard, data := range sharded { - if len(data.RecordIDs) != shards[shard] { - t.Errorf("shard %d: expected %d items, got %d", shard, shards[shard], len(data.RecordIDs)) - } - for _, v := range data.RecordIDs { - if (v >> shardwidth.Exponent) != shard { - t.Errorf("shard %d: got %x, which should be in %d", shard, v, v>>shardwidth.Exponent) - } - } - // expect sorted-ness - data.SortByRecords() - prev := data.RecordIDs[0] - for i, next := range data.RecordIDs[1:] { - if next < prev { - t.Errorf("index %d: prev %d, next %d", i+1, prev, next) - } - prev = next - } - data.SortByValues() - prevV, prevRec := data.Values[0], data.RecordIDs[0] - for i, nextRec := range data.RecordIDs[1:] { - nextV := data.Values[i+1] - if nextV < prevV { - t.Errorf("index %d: prev value %d, next value %d", i+1, prevV, nextV) - } - if nextV == prevV { - if nextRec < prevRec { - t.Errorf("index %d, value %d: prev rec %d, next rec %d", i+1, nextV, prevRec, nextRec) - } - } - prevV = nextV - prevRec = nextRec - } - } - } -} diff --git a/ingest/shard.go b/ingest/shard.go deleted file mode 100644 index 788623bdb..000000000 --- a/ingest/shard.go +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest diff --git a/ingest/sort.go b/ingest/sort.go deleted file mode 100644 index 7020c841f..000000000 --- a/ingest/sort.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -// "math/bits" - -// HERE THERE BE DRAGONS - -// This is some sorting logic Nia was experimenting with, which we aren't currently -// using, but which beat stdlib sort by a factor-of-several on at least some test -// data, so we aren't deleting it just yet. - -// groupIDPairsByShard destructively groups ID pairs by shard. -// The returned slices reference the original pairs slice. -// func groupIDPairsByShard(pairs []IDPair) map[uint64][]IDPair { -// if len(pairs) == 0 { -// return nil -// } -// -// // Sort pairs by shard (in-place radix sort). -// // This may also change the order of pairs within a shard, but that should not matter. -// for { -// // Find the highest bit which needs to be sorted. -// var diffMask uint64 -// prev := pairs[0].RecordID -// for _, v := range pairs[1:] { -// if v.RecordID < prev { -// diffMask |= v.RecordID ^ prev -// } -// -// prev = v.RecordID -// } -// diffLen := bits.Len64(diffMask) -// if diffLen <= shardwidth.Exponent { -// // The pairs are sorted by shard. -// break -// } -// -// // Select a right bit shift index such that the highest unsorted bit moves to the 128's place. -// shift := uint(diffLen) - 8 -// -// // Create a mask that can be used to group values by sorted bits. -// sortedMask := ^uint64(0) << bits.Len64(diffMask) -// -// for i := 0; i < len(pairs); { -// // Select a group of pairs to sort. -// // While doing so, count the pairs within each bucket. -// j := i -// var buckets [256]struct { -// start, end uint -// } -// for group := pairs[i].RecordID & sortedMask; i < len(pairs) && pairs[i].RecordID&sortedMask == group; i++ { -// buckets[uint8(pairs[i].RecordID>>uint64(shift))].end++ -// } -// group := pairs[j:i] -// -// // Assign indices within the group to the buckets. -// { -// var start uint -// for i := range buckets { -// bucket := &buckets[i] -// bucket.start = start -// bucket.end += start -// start = bucket.end -// } -// } -// -// // Split the group into the buckets. -// for i, b := range buckets { -// // There is no need to update the state of the current bucket - we will never reference it again after this. -// i := uint8(i) -// for j := b.start; j < b.end; j++ { -// // This inner loop may run quite a few times for the first few buckets, but no element will be moved more than twice per byte. -// for uint8(group[j].RecordID>>shift) != i { -// // This pair is in the wrong bucket. -// // Swap it into the correct bucket. -// dstBucket := &buckets[uint8(group[j].RecordID>>shift)] -// k := dstBucket.start -// dstBucket.start++ -// group[j], group[k] = group[k], group[j] -// } -// } -// } -// } -// } -// -// // Split the pairs by shard. -// shards := make(map[uint64][]IDPair) -// for i := 0; i < len(pairs); { -// // Select the shard. -// shard := pairs[i].RecordID >> shardwidth.Exponent -// -// // Find all pairs in the shard. -// j := i -// incr := 1 -// for i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard { -// i += incr -// incr *= 2 -// } -// for ; incr > 0; incr /= 2 { -// if i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard { -// i += incr -// } -// } -// // that found us the last thing in this shard, so... -// i++ -// -// // Add the shard, referencing the original slice. -// // This sets the cap so that we dont accidentally overwrite other shards data. -// shards[shard] = pairs[j:i:i] -// } -// -// return shards -// } - -// pairsToZigZag converts a set of record-value pairs to Pilosa's zig-zag format. -// This assumes that all pairs are within the same shard. -// func pairsToZigZag(pairs []IDPair) []uint64 { -// const recordMask = (1 << shardwidth.Exponent) - 1 -// -// dst := make([]uint64, len(pairs)) -// for i, p := range pairs { -// dst[i] = (p.ID << shardwidth.Exponent) | (p.RecordID & recordMask) -// } -// -// return dst -// } - -// radixSort64 sorts the data with radix-sort. -// The "shift" is the highest differing bit position, rounded down to a multiple of 8. -// If there are duplicates, this may change the duplicate count for some values. -// func radixSort64(data []uint64, shift uint) { -// if len(data) < 2 { -// return -// } -// if shift <= 8 { -// // The data falls into a 16-bit span, so the remaining digits can be sorted simultaneously with a bitmask. -// maskSort(data) -// return -// } -// -// // Count the values within each bucket. -// var buckets [256]struct { -// start, end uint -// } -// for _, v := range data { -// buckets[uint8(v>>shift)].end++ -// } -// -// // Assign indices within the group to the buckets. -// { -// var start uint -// for i := range buckets { -// bucket := &buckets[i] -// bucket.start = start -// bucket.end += start -// start = bucket.end -// } -// } -// -// // Split the data into the buckets. -// var start uint -// for i, b := range buckets { -// // Replace misplaced values until the contents of the bucket all have the correct digit. -// i := uint8(i) -// for j := b.start; j < b.end; j++ { -// // This inner loop may run quite a few times for the first few buckets, but it will never run more than once-per-element-per-byte. -// for uint8(data[j]>>shift) != i { -// // This pair is in the wrong bucket. -// // Swap it into the correct bucket. -// dstBucket := &buckets[uint8(data[j]>>shift)] -// k := dstBucket.start -// dstBucket.start++ -// data[j], data[k] = data[k], data[j] -// } -// } -// -// // Sort the contents of the bucket. -// data := data[start:b.end] -// switch { -// case len(data) < 64: -// // Use insertion-sort because the data is too small for a more complex algorithm to be efficient. -// for i := 0; i < len(data); i++ { -// for j := i; j > 0 && data[j-1] > data[j]; j-- { -// data[j-1], data[j] = data[j], data[j-1] -// } -// } -// -// default: -// // Sort the next byte recursively. -// radixSort64(data, shift-8) -// } -// start = b.end -// } -// } - -// maskSort sorts integers using a bitmask. -// The values must all fall within one 16-bit span. -// If there are duplicates, this may change the duplicate count for some values. -// func maskSort(data []uint64) { -// if len(data) < 2 { -// return -// } -// -// base := data[0] &^ ((1 << 16) - 1) -// -// // Dump everything into the mask. -// var mask [(1 << 16) / 64]uint64 -// for _, v := range data { -// mask[uint16(v)/64] |= 1 << (v % 64) -// } -// -// // Scan through the set bits in the mask. -// k := 0 -// for i, w := range mask { -// for w != 0 { -// j := bits.TrailingZeros64(w) -// w &^= 1 << j -// data[k] = 64*uint64(i) + uint64(j) + base -// k++ -// } -// } -// -// if k < len(data) { -// // Copy the ending value to fill up the rest of the space. -// // This happens once for each duplicate value. -// endVal := data[k-1] -// for k < len(data) { -// data[k] = endVal -// k++ -// } -// } -// } diff --git a/ingest/sort_test.go b/ingest/sort_test.go deleted file mode 100644 index 017ffa60f..000000000 --- a/ingest/sort_test.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "testing" -) - -const sampleSize = 1000000 - -var sampleSortingData = createSampleFieldData() - -func createSampleFieldData() *FieldOperation { - fo := &FieldOperation{} - fo.RecordIDs = make([]uint64, sampleSize) - fo.Values = make([]uint64, sampleSize) - fo.Signed = make([]int64, sampleSize) - for i := range fo.RecordIDs { - fo.RecordIDs[i] = (uint64(i) * 63 * 3456789) % 50000000 - fo.Values[i] = (uint64(i) * 6) % 8 - fo.Signed[i] = ((int64(i) * 17) % 15) - 8 - } - return fo -} - -func benchmarkOneSort(b *testing.B, fo *FieldOperation) { - for i := 0; i < b.N; i++ { - b.StopTimer() - sortable := fo.clone() - b.StartTimer() - _ = sortable.SortToShards() - } -} - -func BenchmarkSortFieldOp(b *testing.B) { - b.Run("full", func(b *testing.B) { - f2 := *sampleSortingData - benchmarkOneSort(b, &f2) - }) - b.Run("nosign", func(b *testing.B) { - f2 := *sampleSortingData - f2.Signed = nil - benchmarkOneSort(b, &f2) - }) - b.Run("signonly", func(b *testing.B) { - f2 := *sampleSortingData - f2.Values = nil - benchmarkOneSort(b, &f2) - }) -} - -// func BenchmarkSort64(b *testing.B) { -// gen := func(n, width uint64) func() []uint64 { -// var data []uint64 -// var once sync.Once -// return func() []uint64 { -// once.Do(func() { -// data = make([]uint64, n) -// var rng rand.PCGSource -// rng.Seed(9001) -// for i := range data { -// data[i] = rng.Uint64() % width -// } -// }) -// -// return data -// } -// } -// -// algos := []struct { -// name string -// maxn uint64 -// fn func([]uint64) []uint64 -// }{ -// { -// name: "stdlib", -// maxn: 1024 * 1024 * 1024, -// fn: stdSort, -// }, -// { -// name: "heap", -// maxn: 1024 * 1024 * 1024, -// fn: heapSort, -// }, -// { -// name: "radix-insertion-mask", -// maxn: 1024 * 1024 * 1024, -// fn: dedupSort64, -// }, -// } -// -// widths := []struct { -// name string -// width uint64 -// }{ -// {"64", 64}, -// {"1K", 1024}, -// {"64K", 64 * 1024}, -// {"1M", 1024 * 1024}, -// {"16M", 16 * 1024 * 1024}, -// {"128M", 128 * 1024 * 1024}, -// {"1B", 1024 * 1024 * 1024}, -// } -// -// counts := []struct { -// name string -// n uint64 -// }{ -// {"64", 64}, -// {"1K", 1024}, -// {"64K", 64 * 1024}, -// {"1M", 1024 * 1024}, -// {"4M", 4 * 1024 * 1024}, -// {"16M", 16 * 1024 * 1024}, -// {"64M", 64 * 1024 * 1024}, -// {"256M", 256 * 1024 * 1024}, -// } -// -// for _, width := range widths { -// width := width -// b.Run(width.name, func(b *testing.B) { -// for _, count := range counts { -// if count.n > width.width { -// continue -// } -// -// count := count -// b.Run(count.name, func(b *testing.B) { -// datasrc := gen(count.n, width.width) -// for _, alg := range algos { -// if count.n > alg.maxn { -// continue -// } -// -// alg := alg -// b.Run(alg.name, func(b *testing.B) { -// data := datasrc() -// buf := make([]uint64, len(data)) -// b.SetBytes(8 * int64(len(buf))) -// -// b.StopTimer() -// b.ResetTimer() -// -// for i := 0; i < b.N; i++ { -// copy(buf, data) -// b.StartTimer() -// alg.fn(buf) -// b.StopTimer() -// } -// }) -// } -// }) -// } -// }) -// } -// } -// -// func heapSort(data []uint64) []uint64 { -// for i, v := range data { -// for i > 0 && v > data[(i-1)/2] { -// data[i] = data[(i-1)/2] -// i = (i - 1) / 2 -// } -// data[i] = v -// } -// { -// heap := data -// for len(heap) > 1 { -// heap[0], heap[len(heap)-1] = heap[len(heap)-1], heap[0] -// heap = heap[:len(heap)-1] -// i := 0 -// for { -// max := i -// if r := 2*i + 1; r < len(heap) && heap[r] > heap[max] { -// max = r -// } -// if l := 2*i + 2; l < len(heap) && heap[l] > heap[max] { -// max = l -// } -// if max == i { -// break -// } -// -// heap[max], heap[i] = heap[i], heap[max] -// i = max -// } -// } -// } -// -// j := 1 -// prev := data[0] -// for _, v := range data[1:] { -// if v == prev { -// continue -// } -// -// data[j] = v -// prev = v -// } -// -// return data[:j] -// } -// -// func stdSort(data []uint64) []uint64 { -// sort.Slice(data, func(i, j int) bool { return data[i] < data[j] }) -// -// j := 1 -// prev := data[0] -// for _, v := range data[1:] { -// if v == prev { -// continue -// } -// -// data[j] = v -// prev = v -// } -// -// return data[:j] -// } diff --git a/ingest/translate_test.go b/ingest/translate_test.go deleted file mode 100644 index f84265cba..000000000 --- a/ingest/translate_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "testing" -) - -// stableTranslator implements a key translator that can be reused and -// will continue to give the same keys for the same values. Possibly -// surprisingly, it will invent new keys for IDs it is asked about but -// hasn't seen. This allows us to give a codec which would use keys on -// translation a request which contains arbitrary numbers, and request -// text that would parse into that request. -type stableTranslator struct { - in map[string]uint64 - out map[uint64]string - next uint64 -} - -func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - ret := make(map[string]uint64, len(keys)) - for _, key := range keys { - if existing, ok := s.in[key]; ok { - ret[key] = existing - continue - } - id := s.next - // but what if someone already translated that ID, so now it already - // exists? - if _, ok := s.out[id]; ok { - for k := range s.out { - if k > id { - id = k - } - } - // one larger than the largest we already have. this could - // wrap around, in which case, it's your own fault. - id++ - } - s.next = id + 1 - s.in[key] = id - s.out[id] = key - ret[key] = id - } - return ret, nil -} - -func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - ret := make(map[uint64]string, len(ids)) - for _, id := range ids { - if existing, ok := s.out[id]; ok { - ret[id] = existing - continue - } - key := fmt.Sprintf("k-%d", id) - s.in[key] = id - s.out[id] = key - ret[id] = key - if id >= s.next { - s.next = id + 1 - } - } - return ret, nil -} - -// newStableTranslator produces a translator which can translate forwards -// and backwards and invent new things if it needs to. Don't use this. -func newStableTranslator() *stableTranslator { - return &stableTranslator{ - in: make(map[string]uint64), - out: make(map[uint64]string), - } -} - -func TestTranslateReuse(t *testing.T) { - // the original stable-translator design had a flaw in that it - // assumed that each new ID would always come from a string translation, - // never from a key translation, and that they'd show up sequentially. - tr := newStableTranslator() - orig, err := tr.TranslateIDs(1, 2) - tr.next = 1 // intentionally break the translation logic for testing purposes - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - var s [6]string - stash := s[:0] - for _, v := range orig { - stash = append(stash, v) - } - for i := range s[len(orig):] { - stash = append(stash, fmt.Sprintf("key-%d", i)) - } - keys, err := tr.TranslateKeys(stash...) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ids := make([]uint64, 0, len(keys)) - for _, v := range keys { - ids = append(ids, v) - } - idMap, err := tr.TranslateIDs(ids...) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - for k, v := range keys { - if idMap[v] != k { - t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q", - k, v, v, idMap[v]) - } - } - for id, key := range idMap { - if keys[key] != id { - t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d", - id, key, key, keys[key]) - } - } - for id, key := range orig { - if keys[key] != id { - t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d", - id, key, key, keys[key]) - } - } -} diff --git a/ingest/update.go b/ingest/update.go deleted file mode 100644 index f36363ba7..000000000 --- a/ingest/update.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "github.com/molecula/featurebase/v3/roaring" -) - -// ShardUpdate is an update request for a shard. -type ShardUpdate struct { - // TODO: include schema version - - // Sets are the updates to set fields. - Sets map[string]SetMatrixUpdate - - // Mutexes are the updates to mutex fields. - Mutexes map[string]MutexMatrixUpdate - - // TimeTensors are the updates to time fields. - TimeTensors map[string]TimeTensorUpdate - - // Ints are the updates to int fields. - Ints map[string]IntUpdate -} - -// Convert the ID vector to a raw update that can be imported. -// All records are assumed to fall within a shard. -// func (vec IDVector) Convert() (MutexMatrixUpdate, error) { -// // Before converting the vector, do a sanity-check for duplicates. -// dedup := make(map[uint64]struct{}, len(vec.Updates)+len(vec.Clears)) -// for _, p := range vec.Updates { -// if _, ok := dedup[p.RecordID]; !ok { -// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates") -// } -// -// dedup[p.RecordID] = struct{}{} -// } -// for _, p := range vec.Clears { -// if _, ok := dedup[p]; !ok { -// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates") -// } -// -// dedup[p] = struct{}{} -// } -// -// // Convert the updates to a bitmap. -// updates, err := idPairsToBitmap(vec.Updates, false) -// if err != nil { -// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex updates") -// } -// -// // Convert the clears to a bitmap. -// clears, err := idListToBitmap(vec.Clears, false) -// if err != nil { -// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex clears") -// } -// -// // Realign clears bitmap to start of shard. -// if min, ok := clears.Min(); ok { -// min &^= (1 << shardwidth.Exponent) - 1 -// clears = clears.OffsetRange(-min, 0, 1< 0 { -// return SetMatrixUpdate{}, errors.New("set adds and removes overlap") -// } -// -// return SetMatrixUpdate{ -// Add: adds, -// Remove: removes, -// Clear: clears, -// }, nil -// } - -// SetMatrixUpdate is an encoded update request for a set-type (set/mutex) view. -type SetMatrixUpdate struct { - // Add is a bitmap to union the matrix against. - Add *roaring.Bitmap - - // Remove is a bitmap to difference out of the matrix. - // No values will be present in both add and remove. - // This bitmap will not include any records in the clear bitmap. - Remove *roaring.Bitmap - - // Clear is a set of records for which all values should be removed. - // This is applied before add operations. - Clear *roaring.Bitmap -} - -// func idPairsToBitmap(pairs []IDPair, allowDup bool) (*roaring.Bitmap, error) { -// return idListToBitmap(pairsToZigZag(pairs), allowDup) -// } -// -// func idListToBitmap(ids []uint64, allowDup bool) (*roaring.Bitmap, error) { -// vals := dedupSort64(ids) -// if len(vals) != len(ids) && !allowDup { -// return nil, errors.New("input contains duplicate values") -// } -// -// // TODO: make the roaring package actually handle this well -// return roaring.NewBitmap(vals...), nil -// } - -// Convert the int vector to a raw update that can be imported. -// All records are assumed to fall within a shard. -// func (vec IntVector) Convert() (IntUpdate, error) { -// // Convert the clears to a bitmap. -// clears, err := idListToBitmap(vec.Clears, false) -// if err != nil { -// return IntUpdate{}, errors.Wrap(err, "encoding int clears") -// } -// -// // Realign clears bitmap to start of shard. -// if min, ok := clears.Min(); ok { -// min &^= (1 << shardwidth.Exponent) - 1 -// clears = clears.OffsetRange(-min, 0, 1< 0 { -// return IntUpdate{}, errors.New("update duplicated with a clear") -// } -// -// return IntUpdate{ -// BSI: updates, -// Clear: clears, -// }, nil -// } - -// IntUpdate is an encoded update request for a BSI (int/timestamp/etc.) view. -type IntUpdate struct { - // BSI is a bitmap of new BSI data to overwrite existing values. - BSI *roaring.Bitmap - - // Clear is a set of records to assign null. - Clear *roaring.Bitmap -} diff --git a/ingest/vec.go b/ingest/vec.go deleted file mode 100644 index 94d6477f3..000000000 --- a/ingest/vec.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "fmt" - "reflect" - "strconv" - "unsafe" -) - -// StringTable is a mapping of strings to temporary IDs. -// All mapped-to IDs fall in the range [0, len). The zero value, -// a nil map, instead just parses the numbers. -// -// We keep the array of names in creation order because we want reproducibility; -// the first key we see is always key 0. Otherwise, the keys are created in -// arbitrary orders. -type StringTable struct { - names []string - values map[string]uint64 -} - -// NewStringTable just creates a string table with a non-nil map. -func NewStringTable() *StringTable { - return &StringTable{values: map[string]uint64{}} -} - -// unsafe is -func pretendByteIsString(data []byte) (result string) { - dH := (*reflect.SliceHeader)(unsafe.Pointer(&data)) - sH := (*reflect.StringHeader)(unsafe.Pointer(&result)) - sH.Data = dH.Data - sH.Len = dH.Len - return result -} - -// ID returns an ID associated to the string, adding it to the table if it is not already present, -// or parsing an integer if there's no table. -func (tbl *StringTable) ID(in []byte) (uint64, error) { - str := pretendByteIsString(in) - if tbl != nil { - id, ok := tbl.values[str] - if !ok { - id = uint64(len(tbl.values)) - tbl.values[str] = id - tbl.names = append(tbl.names, str) - } - return id, nil - } - return strconv.ParseUint(str, 10, 64) -} - -// SignedID returns an ID associated to the string, adding it to the table if it is not already present, -// or parsing an integer if there's no table. It yields signed values only. -func (tbl *StringTable) IntID(in []byte) (int64, error) { - str := pretendByteIsString(in) - if tbl != nil { - id, ok := tbl.values[str] - if !ok { - id = uint64(len(tbl.values)) - tbl.values[str] = id - tbl.names = append(tbl.names, str) - } - return int64(id), nil - } - return strconv.ParseInt(str, 10, 64) -} - -// MapForStringTable, given a string table mapping strings to consecutive -// integers and a translation function from strings to "real" keys, yields -// a translation/lookup slice. If it cannot translate all the keys, it -// returns an error. -func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) { - lookedUp, err := keys.TranslateKeys(tbl.names...) - if err != nil { - return nil, err - } - if len(lookedUp) != len(tbl.names) { - return nil, fmt.Errorf("missing keys: expected %d keys, got %d", len(tbl.values), len(lookedUp)) - } - out := make([]uint64, len(tbl.names)) - for i, v := range tbl.names { - out[i] = lookedUp[v] - } - return out, nil -} - -// translateSigned replaces values from 0 to len(mapping)-1 with the -// elements of mapping. It yields an error if any values aren't -// mapped. -func translateSigned(mapping []uint64, values []int64) error { - oops := 0 - for i, v := range values { - if v >= int64(len(mapping)) { - oops++ - } else { - values[i] = int64(mapping[v]) - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} - -// translateUnsigned replaces values from 0 to len(mapping)-1 with the -// elements of mapping. It yields an error if any values aren't -// mapped. -func translateUnsigned(mapping []uint64, values []uint64) error { - oops := 0 - for i, v := range values { - if v >= uint64(len(mapping)) { - oops++ - } else { - values[i] = mapping[v] - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} diff --git a/ingest/vec_test.go b/ingest/vec_test.go deleted file mode 100644 index 9b5947272..000000000 --- a/ingest/vec_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package ingest - -import ( - "errors" - "testing" -) - -type badTranslator struct{} - -func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - if len(keys) == 0 { - return nil, errors.New("no keys") - } - m := make(map[string]uint64) - skip := true - for i, k := range keys { - if skip { - skip = false - continue - } - m[k] = uint64(i) - } - out := make([]uint64, len(keys)-1) - for i := range out { - out[i] = uint64(i) - } - return nil, nil -} - -func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) { - return nil, nil -} - -func TestStringTableErrors(t *testing.T) { - tbl := NewStringTable() - btr := badTranslator{} - _, keyErr := tbl.MakeIDMap(btr) - if keyErr == nil { - t.Fatalf("expected error passed up from failed translate, didn't get it") - } - a1, err := tbl.ID([]byte("a")) - if err != nil { - t.Fatalf("getting translation for key: %v", err) - } - b1, err := tbl.ID([]byte("b")) - if err != nil { - t.Fatalf("getting translation for key: %v", err) - } - _, keyErr = tbl.MakeIDMap(btr) - if keyErr == nil { - t.Fatalf("expected error for short translate, didn't get it") - } - tr := newStableTranslator() - _, err = tr.TranslateKeys("c", "d") - if err != nil { - t.Fatalf("translating stray keys: %v", err) - } - m, err := tbl.MakeIDMap(tr) - if err != nil { - t.Fatalf("creating lookup: %v", err) - } - var y = []uint64{a1, b1} - err = translateUnsigned(m, y) - if err != nil { - t.Fatalf("unexpected unsigned translation error: %v", err) - } - trResults, err := tr.TranslateKeys("a", "b") - if err != nil { - t.Fatalf("unexpected translation error: %v", err) - } - if y[0] != trResults["a"] { - t.Fatalf("expected %d, got %d", trResults["a"], y[0]) - } - if y[1] != trResults["b"] { - t.Fatalf("expected %d, got %d", trResults["b"], y[1]) - } - y[0] = a1 - y[1] = (a1 + b1 + 1) // assumed not to be any of them - err = translateUnsigned(m, y) - if err == nil { - t.Fatalf("no error from translating invalid table") - } - z := []int64{int64(a1), int64(b1)} - err = translateSigned(m, z) - if err != nil { - t.Fatalf("unexpected unsigned translation error: %v", err) - } - if uint64(z[0]) != trResults["a"] { - t.Fatalf("expected %d, got %d", trResults["a"], z[0]) - } - if uint64(z[1]) != trResults["b"] { - t.Fatalf("expected %d, got %d", trResults["b"], z[1]) - } - z[0] = int64(a1) - z[1] = int64(a1 + b1 + 1) // assumed not to be any of them - err = translateSigned(m, z) - if err == nil { - t.Fatalf("no error from translating invalid table") - } -} diff --git a/ingest_test.go b/ingest_test.go deleted file mode 100644 index 341ab9d8c..000000000 --- a/ingest_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package pilosa_test - -import ( - "bytes" - "context" - "encoding/json" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/ingest" - "github.com/molecula/featurebase/v3/test" - "github.com/pkg/errors" -) - -// For ingest API testing, we want to do tests which have a known -// schema, no existing data before we start, and perform ingests and -// then do queries. -// -// A good starting point for this would be a fairly simple file -// divided into sections which are just the JSON text of the data -// we want to be working with, or the PQL queries we want to run, -// or their expected results. -// -// So, roughly like this: -// -// schema: -// { -// "index-name": "example", -// "primary-key-type": "string", -// "fields": [ -// { -// "field-name": "set", -// "field-type": "id", -// "field-options": { "cache-type": "none" } -// } -// ] -// } -// ingest: -// [ -// { -// "action": "set", -// "records": { -// "1": { -// "set": [ 2 ], -// } -// } -// } -// ] -// queries: -// Row(set=2): -// [1] -// -// Additionally, the names "schema-error" and "ingest-error" are taken to -// represent a schema, or data set, which is expected to produce an error. -// For instance: -// -// ingest-error: -// [ { "action": puppy } -// -// In this case, it would be considered a test failure if an ingest request -// did NOT fail. -// Lines starting with # - -// ingestSchemaPartial represents the only part of a schema we need to -// know about in order to undo its creation of a schema for use in a -// test case. -type ingestSchemaPartial struct { - IndexName string `json:"index-name"` -} - -type ingestActionKind int - -const ( - ingestActionNone = ingestActionKind(iota) - ingestActionSchema - ingestActionIngest - ingestActionSchemaError - ingestActionIngestError - ingestActionQueries -) - -var ingestActionKinds = map[string]ingestActionKind{ - "schema": ingestActionSchema, - "ingest": ingestActionIngest, - "schema-error": ingestActionSchemaError, - "ingest-error": ingestActionIngestError, - "queries": ingestActionQueries, -} -var ingestActionKindNames = map[ingestActionKind]string{} - -type testCaseAction struct { - kind ingestActionKind - comment []byte - lineStart int - lineEnd int - data []byte -} - -type liner struct { - data []byte - at int - start int - remaining []byte - line []byte -} - -func newLiner(data []byte) *liner { - return &liner{data: data, at: 0, remaining: data} -} - -func (l *liner) next() bool { - if len(l.remaining) == 0 { - return false - } - foundNL := true - nextNL := bytes.IndexByte(l.remaining, '\n') - if nextNL == -1 { - foundNL = false - nextNL = len(l.remaining) - } - l.line = l.remaining[:nextNL] - // move past the newline we found, if we found one - if foundNL { - nextNL++ - } - l.start, l.at = l.at, l.at+nextNL - l.remaining = l.data[l.at:] - return true -} - -func (l *liner) text() (line []byte, start int, end int) { - return l.line, l.start, l.at -} - -// parseExpectedResults handles something that looks like -// [1, 2, 3] or ["a", "b", "c"]. It does not handle things like -// quotes within strings, etcetera. -func parseExpectedResults(data []byte) (ints []uint64, keys []string, err error) { - if len(data) < 2 || data[0] != '[' || data[len(data)-1] != ']' { - return nil, nil, errors.New("expecting [] results") - } - words := bytes.Split(data[1:len(data)-1], []byte{','}) - if len(words) == 1 && len(words[0]) == 0 { - return nil, nil, nil - } - for _, word := range words { - word = bytes.TrimSpace(word) - if len(word) == 0 { - return nil, nil, errors.New("found empty word expecting result") - } - if word[0] == '"' { - keys = append(keys, string(word[1:len(word)-1])) - continue - } - v, err := strconv.ParseInt(string(word), 10, 64) - if err != nil { - return nil, nil, err - } - ints = append(ints, uint64(v)) - } - if len(ints) > 0 && len(keys) > 0 { - return nil, nil, errors.New("mixed integers and strings are invalid") - } - return ints, keys, err -} - -func testQueries(t *testing.T, ctx context.Context, cmd *test.Command, index string, action testCaseAction) { - qcx := cmd.API.Txf().NewQcx() - defer func() { - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - l := newLiner(action.data) - for l.next() { - query, _, _ := l.text() - if !l.next() { - t.Fatalf("processing query list: no expected after %q", query) - } - expected, _, _ := l.text() - ints, keys, err := parseExpectedResults(expected) - if err != nil { - t.Fatalf("processing query list: invalid expected results %q", expected) - } - t.Logf("expecting %q -> %s", query, expected) - res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: string(query)}) - if err != nil { - t.Errorf("query: %v", err) - } - if len(res.Results) != 1 { - t.Fatalf("expected one result per query, got %d results", len(res.Results)) - } - var row *pilosa.Row - var ok bool - if row, ok = res.Results[0].(*pilosa.Row); !ok { - t.Fatalf("expected results to be a row") - } - if ints != nil { - cols := row.Columns() - if len(cols) != len(ints) { - t.Fatalf("wrong number of values, expected %d, got %d", len(ints), len(cols)) - } - for i := range cols { - if ints[i] != cols[i] { - t.Fatalf("result %d: expected %d, got %d", i, ints[i], cols[i]) - } - } - } - // key return value is unpredictable, so... - if keys != nil { - seen := make(map[string]struct{}) - for _, k := range keys { - seen[k] = struct{}{} - } - for _, k := range row.Keys { - if _, ok := seen[k]; !ok { - t.Fatalf("unexpected result key %q", k) - } - delete(seen, k) - } - for k := range seen { - t.Fatalf("expected result to contain %q, but did not get it", k) - } - } - } -} - -// testOneIngestTestcase runs a set of actions, then cleans up after itself -func testOneIngestTestcase(t *testing.T, ctx context.Context, cmd *test.Command, tcpath string) { - data, err := os.ReadFile(tcpath) - if err != nil { - t.Fatalf("reading %q: %v", tcpath, err) - } - var actions []testCaseAction - var action testCaseAction - l := newLiner(data) - var line []byte - var start, lineStart, lineEnd int - lineCount := 0 - for l.next() { - line, lineStart, lineEnd = l.text() - lineCount++ - if colon := bytes.IndexByte(line, ':'); colon != -1 { - if kind, ok := ingestActionKinds[string(line[:colon])]; ok { - if action.kind != ingestActionNone { - action.data = data[start:lineStart] - actions = append(actions, action) - action.lineEnd = lineCount - 1 - } else { - if lineStart != 0 { - t.Logf("warning: %d bytes with no action type before first action", lineStart) - } - } - start = lineEnd - action.data = nil - action.kind = kind - if line[len(line)-1] == ':' { - action.comment = line[:len(line)-1] - } else { - action.comment = line - } - action.lineStart = lineCount - } - } - } - if action.kind != ingestActionNone { - action.lineEnd = lineCount - 1 - action.data = data[start:] - actions = append(actions, action) - } - var mostRecentIndex string - seenIndexes := map[string]struct{}{} - - cli := cmd.Client() - created := map[string][]string{} - defer func() { - t.Logf("deleting created indexes/fields:") - for k, v := range created { - if len(v) == 0 { - t.Logf(" index: %q", k) - if err := cmd.API.DeleteIndex(ctx, k); err != nil { - t.Errorf("deleting index %q: %v", k, err) - } - } else { - t.Logf(" fields in %q: %q", k, v) - for _, field := range v { - if err := cmd.API.DeleteField(ctx, k, field); err != nil { - t.Errorf("deleting field %q from %q: %v", field, k, err) - } - } - } - } - }() - - noticeCreation := func(newlyCreated map[string][]string) { - for k, v := range newlyCreated { - if len(v) == 0 { - if existing, ok := created[k]; ok { - if len(existing) > 0 { - t.Fatalf("creation reports index %q newly created, but we created fields %q in it previously", - k, existing) - } - } - // create an empty list, indicating that the whole index is - // believed nil - created[k] = nil - continue - } - if existing, ok := created[k]; ok { - if len(existing) == 0 { - // we'll delete this index anyway, don't need to delete fields in it - continue - } - created[k] = append(existing, v...) - continue - } - created[k] = v - } - } - - for _, action := range actions { - t.Logf("%s, lines %d-%d", action.comment, action.lineStart, action.lineEnd) - switch action.kind { - case ingestActionSchema: - var scratch ingestSchemaPartial - err = json.Unmarshal(action.data, &scratch) - if err != nil { - t.Fatalf("couldn't parse schema data: %v", err) - } - if scratch.IndexName == "" { - t.Fatalf("test case must provide an index name") - } - // stash the string from the schema, because we - // might need it later - mostRecentIndex = scratch.IndexName - seenIndexes[mostRecentIndex] = struct{}{} - var newlyCreated map[string][]string - newlyCreated, err = cli.IngestSchema(ctx, nil, action.data) - if err != nil { - t.Fatalf("executing schema: %v", err) - } - noticeCreation(newlyCreated) - case ingestActionSchemaError: - var scratch ingestSchemaPartial - err = json.Unmarshal(action.data, &scratch) - if err != nil { - t.Logf("got expected error from schema: %v", err) - break - } - var newlyCreated map[string][]string - newlyCreated, err = cli.IngestSchema(ctx, nil, action.data) - if err != nil { - t.Logf("got expected error from schema: %v", err) - break - } - noticeCreation(newlyCreated) - t.Fatalf("expected error from schema, didn't get it") - case ingestActionIngest: - func() { - qcx := cmd.API.Txf().NewQcx() - var err error - defer func() { - if err == nil { - qcx.Abort() - return - } - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data)) - if err != nil { - t.Fatalf("importing data: %v", err) - } - }() - case ingestActionIngestError: - func() { - qcx := cmd.API.Txf().NewQcx() - var err error - defer func() { - if err == nil { - qcx.Abort() - return - } - if err := qcx.Finish(); err != nil { - t.Fatalf("finishing qcx: %v", err) - } - }() - err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data)) - if err != nil { - t.Logf("got expected error from ingest: %v", err) - return - } - t.Fatalf("expected error from ingest, didn't get it") - }() - case ingestActionQueries: - testQueries(t, ctx, cmd, mostRecentIndex, action) - } - } - -} - -// TestIngestTestcases reads sample test cases from a test data directory -// and evaluates them. -func TestIngestTestcases(t *testing.T) { - _ = &ingest.Operation{} - var testcases []string - if len(ingestActionKindNames) == 0 { - for k, v := range ingestActionKinds { - ingestActionKindNames[v] = k - } - } - err := filepath.Walk("ingest_testdata", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if strings.HasSuffix(path, ".tc") { - testcases = append(testcases, path) - } - return nil - }) - if err != nil { - t.Fatalf("looking for test cases: %v", err) - } - if len(testcases) == 0 { - t.Fatalf("no ingest test cases found") - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := test.MustRunCluster(t, 3) - defer c.Close() - - coord := c.GetPrimary() - - for _, tc := range testcases { - t.Run(strings.TrimSuffix(tc, ".tc"), func(t *testing.T) { - testOneIngestTestcase(t, ctx, coord, tc) - }) - } -} diff --git a/ingest_testdata/bool.tc b/ingest_testdata/bool.tc deleted file mode 100644 index a5957cb15..000000000 --- a/ingest_testdata/bool.tc +++ /dev/null @@ -1,17 +0,0 @@ -schema: -{ - "index-name": "example", - "index-action": "create", - "primary-key-type": "uint", - "fields": [ - { - "field-name": "tf", - "field-type": "bool" - } - ] -} -ingest: -[{"action": "write", "records": {"2": { "tf": true }}}] -queries: -Row(tf=true) -[2] diff --git a/ingest_testdata/expect_errors.tc b/ingest_testdata/expect_errors.tc deleted file mode 100644 index 707bfcdba..000000000 --- a/ingest_testdata/expect_errors.tc +++ /dev/null @@ -1,101 +0,0 @@ -schema-error: fail to create field -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "cookie", - "field-type": "id", - "field-options": { "cache-type": "none" } - }, - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "nun" } - } - ] -} -schema: confirm index was not created because field failed -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: can't recreate index -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema: can add field to existing index -{ - "index-name": "examplekeys", - "index-action": "ensure", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: second field failing in existing index -{ - "index-name": "examplekeys", - "index-action": "ensure", - "primary-key-type": "string", - "fields": [ - { - "field-name": "addokay", - "field-type": "id", - "field-options": { "cache-type": "none" } - }, - { - "field-name": "addfail", - "field-type": "id", - "field-options": { "cache-type": "nun" } - } - ] -} -schema: verify that fields we think exist do exist -{ - "index-name": "examplekeys", - "index-action": "require", - "primary-key-type": "string", - "fields": [ - { - "field-name": "newfield", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -schema-error: successful field deleted anyway because second field failed -{ - "index-name": "examplekeys", - "index-action": "require", - "primary-key-type": "string", - "fields": [ - { - "field-name": "addokay", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} diff --git a/ingest_testdata/keyed.tc b/ingest_testdata/keyed.tc deleted file mode 100644 index 8f7d5c4b6..000000000 --- a/ingest_testdata/keyed.tc +++ /dev/null @@ -1,32 +0,0 @@ -schema: -{ - "index-name": "examplekeys", - "index-action": "create", - "primary-key-type": "string", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "a": { - "set": [ 2 ], - }, - "b": { - "set": 3, - } - } - } -] -queries: -Row(set=2) -["a"] -Union(Row(set=3),Row(set=2)) -["a","b"] diff --git a/ingest_testdata/sample.tc b/ingest_testdata/sample.tc deleted file mode 100644 index 764965025..000000000 --- a/ingest_testdata/sample.tc +++ /dev/null @@ -1,99 +0,0 @@ -schema: -{ - "index-name": "example", - "index-action": "create", - "primary-key-type": "uint", - "fields": [ - { - "field-name": "set", - "field-type": "id", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "1": { - "set": [ 2 ], - }, - "2": { - "set": 3, - } - } - } -] -queries: -Row(set=2) -[1] -Union(Row(set=3),Row(set=2)) -[1,2] -schema-error: -{ - "index-name": "example", - "primary-key-type": "uint", - "index-action": "require", - "fields": [ - { - "field-name": "setkey", - "field-type": "string", - "field-options": { "cache-type": "none" } - } - ] -} -schema: -{ - "index-name": "example", - "primary-key-type": "uint", - "index-action": "ensure", - "fields": [ - { - "field-name": "setkey", - "field-type": "string", - "field-options": { "cache-type": "none" } - } - ] -} -ingest: -[ - { - "action": "set", - "records": { - "1": { - "setkey": [ "a" ], - }, - "2": { - "setkey": "b", - } - } - } -] -queries: -Row(setkey="a") -[1] -ingest-error: -[ - { - "action": "setkeys", - "records": { - "a": { - "setkey": [ "a" ], - }, - "b": { - "setkey": "b", - } - } - } -] -ingest: -[ - { - "action": "delete", - "record_ids": [ 1 ] - } -] -queries: -Row(setkey="a") -[] diff --git a/internal_client.go b/internal_client.go index 8d7666292..9620c4a75 100644 --- a/internal_client.go +++ b/internal_client.go @@ -21,7 +21,6 @@ import ( "github.com/hashicorp/go-retryablehttp" "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/tracing" @@ -392,124 +391,6 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return rsp.Indexes, nil } -// IngestSchema uses the new schema ingest endpoint. It returns a -// map from index names to fields created within them; note that if the -// entire index was created, the list of fields is empty. The intended -// usage is cleaning up after creating the indexes, so if you create the -// index, you don't need to delete the fields, but if you created fields -// within an existing index, you should delete those fields but not the -// whole index. -func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []byte) (created map[string][]string, err error) { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path("/internal/schema") - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - buf, err = io.ReadAll(resp.Body) - if resp.StatusCode != 200 { - if err != nil { - return nil, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status) - } - var msg string - // try to decode a JSON response - var sr successResponse - qr := &QueryResponse{} - if err = json.Unmarshal(buf, &sr); err == nil { - msg = sr.Error.Error() - } else if err := c.serializer.Unmarshal(buf, qr); err == nil { - msg = qr.Err.Error() - } else { - msg = string(buf) - } - return nil, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg) - } - // this is the err from io.ReadAll, but in the case where resp.StatusCode - // was 2xx, so we don't have a bad status. - if err != nil { - return nil, errors.Wrapf(err, "error reading response body") - } - if err = json.Unmarshal(buf, &created); err != nil { - return nil, errors.Wrapf(err, "error interpreting response body") - } - return created, nil -} - -// IngestOperations uses the new ingest endpoint for ingest data -func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, indexName string, buf []byte) error { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path(fmt.Sprintf("/internal/ingest/%s", indexName)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return errors.Errorf("unexpected status code: %s", resp.Status) - } - return nil -} - -// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data -func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { - if uri == nil { - uri = c.defaultURI - } - u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName)) - - buf, err := c.serializer.Marshal(ireq) - if err != nil { - return errors.Wrap(err, "marshalling") - } - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - AddAuthToken(ctx, &req.Header) - - resp, err := c.executeRequest(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return errors.Errorf("unexpected status code: %s", resp.Status) - } - return nil -} - // MutexCheck uses the mutex-check endpoint to request mutex collision data // from a single node. It produces per-shard results, and does not translate // them. diff --git a/translate.go b/translate.go index 7de4863b4..64d3f6bd8 100644 --- a/translate.go +++ b/translate.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/molecula/featurebase/v3/disco" - "github.com/molecula/featurebase/v3/ingest" "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" ) @@ -89,39 +88,6 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul Delete(records *roaring.Bitmap) (Commitor, error) } -// This implements ingest's key translator interface, which differs -// slightly because we want to be able to do fast lookups on arbitrary -// IDs which are not necessarily contiguous small values, so the []string -// from TranslateIDs isn't a good fit. -type ingestKeyTranslator struct { - store TranslateStore -} - -var _ ingest.KeyTranslator = &ingestKeyTranslator{} - -func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - return i.store.CreateKeys(keys...) -} - -func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - keys, err := i.store.TranslateIDs(ids) - if err != nil { - return nil, err - } - if len(keys) != len(ids) { - return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) - } - out := make(map[uint64]string, len(keys)) - for i, id := range ids { - out[id] = keys[i] - } - return out, nil -} - -func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator { - return &ingestKeyTranslator{store: s} -} - // TranslatorSummary is returned, for example from the boltdb string key translators, // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. type TranslatorSummary struct {