From 9f271467fb1c73a0d83d50e28439e59413b40c53 Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 10 Sep 2021 12:10:22 -0500 Subject: [PATCH 1/8] ingest cluster support We add endpoints and protobuf encode/decode to allow for sending sharded requests over the wire in protobuf, so we can take our sharded data and send it to other nodes if needed. This is a squash of >15 other commits, so a bit of history is relevant: The Request type had FieldTypes in it because the field type information was needed for sharding because sorting requires that information. We change this around to make the external sharding operation require the field types, and curry that through the codec -- the codec is needed to tell the request how it shards. (This is because the correct sorting order varies by field type.) Requests (and ShardedRequests) no longer have that table in them. And then we hit a nasty bug in production and RCA showed that our testing wasn't good enough and we need to be more careful, and I discovered that test coverage in this package was around 70%. So, the other big thing here is coverage testing; in order to make coverage testing viable and programmatically testable, we have added the ability to render requests *back* to JSON. This is not a great idea, but it does allow us to do a lot of sanity-checking and verify that the encodings we're using are consistent and correct. This, plus some specific tests of decoding specific flawed inputs, has caught a number of issues. Which are now fixed! A lot of internal API surface got slightly changed, in ways that make it simpler to work with. For instance, the (*FieldOperation).TranslateUnsigned function doesn't really need to exist; we can just have a non-method translate function for unsigned and for signed, and use them based on field type. The stable translation hack used for testing had a bug that could allow it to end up producing incorrect results if you asked it to translate an ID first rather than exclusively asking it to translate strings first, this has been corrected. (This is a bug fix in code that was added partway through creating this, but is tricky enough to mention its own comment.) Test coverage is now just over 90%, and a lot of what's left is error-check returns that may well be actually unreachable unless, say, the documentation for encoding/json is full of lies. Which it probably is. --- api.go | 94 +++- client.go | 6 + cluster.go | 34 ++ encoding/proto/proto.go | 112 ++++- http/client.go | 33 ++ http/handler.go | 63 ++- ingest/codec.go | 746 +++++++++++++++++++++++++---- ingest/codec_test.go | 990 +++++++++++++++++++++++++++++++++------ ingest/op.go | 345 +++++++++----- ingest/op_test.go | 116 ++++- ingest/translate.go | 71 +++ ingest/translate_test.go | 70 +++ ingest/vec.go | 57 ++- ingest/vec_test.go | 114 +++++ ingest_test.go | 11 +- translate.go | 34 ++ 16 files changed, 2495 insertions(+), 401 deletions(-) create mode 100644 ingest/translate_test.go create mode 100644 ingest/vec_test.go diff --git a/api.go b/api.go index ce8ffda33..95e2199bf 100644 --- a/api.go +++ b/api.go @@ -1826,6 +1826,37 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } +// helper function: do the apply stuff for a known index with known fields +func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { + eg, ctx := errgroup.WithContext(ctx) + for shard, ops := range req.Ops { + // loop variable shadow capture is the go equivalent of man door hook hand + shard, ops := shard, ops + eg.Go(func() error { + return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) + }) + } + return eg.Wait() +} + +// IngestNodeOperations handles protobuf-formatted data which does not need +// key translation and is applicable to this specific node. +func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { + index := api.holder.Index(indexName) + if index == nil { + api.server.logger.Errorf("ingest: no such index %q", indexName) + return newNotFoundError(ErrIndexNotFound, indexName) + } + fields := index.Fields() + knownFields := map[string]*Field{} + for _, field := range fields { + knownFields[field.name] = field + } + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) +} + +// IngestOperations handles JSON-formatted data which may need key translation +// and may be for any or all nodes. func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") defer span.Finish() @@ -1841,34 +1872,32 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return newNotFoundError(ErrIndexNotFound, indexName) } fields := index.Fields() - var lookup ingest.KeyLookupFunc + var indexKeys ingest.KeyTranslator if index.Keys() { - lookup = func(keys ...string) (map[string]uint64, error) { - return api.cluster.createIndexKeys(ctx, indexName, keys...) - } + indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) } - codec, err := ingest.NewJSONCodec(lookup) + codec, err := ingest.NewJSONCodec(indexKeys) if err != nil { return errors.Wrap(err, "creating JSON codec") } knownFields := map[string]*Field{} for _, field := range fields { - var lookup ingest.KeyLookupFunc + var keys ingest.KeyTranslator if field.usesKeys { - lookup = field.translateStore.CreateKeys + keys = newIngestKeyTranslatorFromStore(field.translateStore) } knownFields[field.name] = field switch field.Type() { case "set": - if err = codec.AddSetField(field.name, lookup); err != nil { + if err = codec.AddSetField(field.name, keys); err != nil { return fmt.Errorf("adding set field to codec: %w", err) } case "time": - if err = codec.AddTimeQuantumField(field.name, lookup); err != nil { + if err = codec.AddTimeQuantumField(field.name, keys); err != nil { return fmt.Errorf("adding time quantum field to codec: %w", err) } case "mutex": - if err = codec.AddMutexField(field.name, lookup); err != nil { + if err = codec.AddMutexField(field.name, keys); err != nil { return fmt.Errorf("adding mutex field to codec: %w", err) } case "bool": @@ -1876,7 +1905,7 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return fmt.Errorf("adding bool field to codec: %w", err) } case "int": - if err = codec.AddIntField(field.name, lookup); err != nil { + if err = codec.AddIntField(field.name, keys); err != nil { return fmt.Errorf("adding int field to codec: %w", err) } case "decimal": @@ -1896,17 +1925,44 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string if err != nil { return errors.Wrap(err, "parsing input data") } - sharded, err := req.ByShard() + sharded, err := codec.RequestByShard(req) if err != nil { return errors.Wrap(err, "sharding input data") } - eg, ctx := errgroup.WithContext(ctx) + // now that we have this, let's assign the shards to nodes + snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) + // oh hey an easy case: we're presumably the only node + if len(snap.Nodes) == 1 { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + } + // split up by fields in some way + byNode := make(map[string]*ingest.ShardedRequest) for shard, ops := range sharded.Ops { - // loop variable shadow capture is the go equivalent of man door hook hand - shard, ops := shard, ops - eg.Go(func() error { - return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) - }) + nodes := snap.ShardNodes(indexName, shard) + forThisShard := byNode[nodes[0].ID] + if forThisShard == nil { + byNode[nodes[0].ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} + continue + } + forThisShard.Ops[shard] = ops + } + eg, ctx := errgroup.WithContext(ctx) + for _, node := range snap.Nodes { + node := node + sharded := byNode[node.ID] + // Sometimes, there's nothing for a specific node. + if sharded == nil { + continue + } + if node.ID == api.NodeID() { + eg.Go(func() error { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + }) + } else { + eg.Go(func() error { + return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) + }) + } } return eg.Wait() } @@ -3090,6 +3146,7 @@ const ( apiIDReset apiPartitionNodes apiIngestOperations + apiIngestNodeOperations apiMutexCheck ) @@ -3159,5 +3216,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIDReset: {}, apiPartitionNodes: {}, apiIngestOperations: {}, + apiIngestNodeOperations: {}, apiMutexCheck: {}, } diff --git a/client.go b/client.go index 62f16e240..8726c440a 100644 --- a/client.go +++ b/client.go @@ -19,6 +19,7 @@ import ( "io" "time" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" ) @@ -81,6 +82,7 @@ type InternalClient interface { ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) + IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) @@ -218,6 +220,10 @@ func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, return nil, nil } +func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + return nil +} + func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index 6419f3681..29c4c2164 100644 --- a/cluster.go +++ b/cluster.go @@ -24,6 +24,7 @@ import ( "time" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/topology" @@ -1566,6 +1567,39 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } +// This implements ingest's key translator interface on a cluster/index +// pair. +type clusterKeyTranslator struct { + ctx context.Context // we're created within a request context and need to pass that to cluster ops + c *cluster + indexName string +} + +var _ ingest.KeyTranslator = &clusterKeyTranslator{} + +func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.c.createIndexKeys(i.ctx, i.indexName, keys...) +} + +func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator { + return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName} +} + // TODO: remove this when it is no longer used func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keys := make([]string, 0, len(keySet)) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 69a456904..d765a8aff 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -21,6 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" @@ -324,7 +325,18 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeResizeAbortMessage(msg, mt) return nil - + case *ingest.ShardedRequest: + msg := &pb.ShardedIngestRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshalling ShardedRequest") + } + req, err := s.decodeShardedIngestRequest(msg) + if err != nil { + return err + } + *mt = *req + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -398,6 +410,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeResizeNodeMessage(mt) case *pilosa.ResizeAbortMessage: return s.encodeResizeAbortMessage(mt) + case *ingest.ShardedRequest: + return s.encodeShardedIngestRequest(mt) } return nil } @@ -931,6 +945,48 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr return &pb.TransactionStats{} } +func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest { + if req == nil || len(req.Ops) == 0 { + return &pb.ShardedIngestRequest{} + } + out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))} + for shard, ops := range req.Ops { + out.Ops[shard] = s.encodeShardIngestOperations(ops) + } + return out +} + +func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { + out := &pb.ShardIngestOperations{} + if len(ops) == 0 { + return out + } + for _, op := range ops { + if op == nil { + continue + } + out.Ops = append(out.Ops, s.encodeShardIngestOperation(op)) + } + return out +} + +func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation { + out := &pb.ShardIngestOperation{ + OpType: op.OpType.String(), + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + out.FieldOps[k] = &pb.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out +} + func (s Serializer) decodeResizeInstruction(ri *pb.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID m.Node = &topology.Node{} @@ -1829,3 +1885,57 @@ func decodeResizeNodeMessage(pb *pb.ResizeNodeMessage, m *pilosa.ResizeNodeMessa func decodeResizeAbortMessage(pb *pb.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) { } + +func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) { + if req == nil || len(req.Ops) == 0 { + return &ingest.ShardedRequest{}, nil + } + out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))} + for shard, ops := range req.Ops { + var err error + out.Ops[shard], err = s.decodeShardIngestOperations(ops) + if err != nil { + return nil, err + } + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) { + out := []*ingest.Operation{} + if len(ops.Ops) == 0 { + return out, nil + } + for _, op := range ops.Ops { + if op == nil { + continue + } + decoded, err := s.decodeShardIngestOperation(op) + if err != nil { + return nil, err + } + out = append(out, decoded) + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) { + opType, err := ingest.ParseOpType(op.OpType) + if err != nil { + return nil, err + } + out := &ingest.Operation{ + OpType: opType, + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + out.FieldOps[k] = &ingest.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out, nil +} diff --git a/http/client.go b/http/client.go index fb079218f..1d41a114f 100644 --- a/http/client.go +++ b/http/client.go @@ -32,6 +32,7 @@ import ( pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -284,6 +285,38 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in return nil } +// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data +func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + if uri == nil { + uri = c.defaultURI + } + u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName)) + + buf, err := c.serializer.Marshal(ireq) + if err != nil { + return errors.Wrap(err, "marshalling") + } + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("unexpected status code: %s", resp.Status) + } + return nil +} + // MutexCheck uses the mutex-check endpoint to request mutex collision data // from a single node. It produces per-shard results, and does not translate // them. diff --git a/http/handler.go b/http/handler.go index ac5a4dd68..d2eef2e5e 100644 --- a/http/handler.go +++ b/http/handler.go @@ -42,6 +42,7 @@ import ( "github.com/gorilla/mux" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/rbf" @@ -432,7 +433,9 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/ingest/{index}", handler.handleIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode") + router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema") router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys") router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys") @@ -1337,7 +1340,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { +// handlePostIngestData handles JSON ingest data that may need key +// translation, for the entire cluster. +func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return @@ -1351,6 +1356,14 @@ func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { qcx := h.api.Txf().NewQcx() err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + } + } else { + qcx.Abort() + } resp := successResponse{h: h, Name: indexName} resp.write(w, err) @@ -2908,6 +2921,52 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } } +// handlePostIngestNode is the internal endpoint taking already-translated +// ingest operations, sorted by shard, for a single node. +func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + ctx := r.Context() + + // Read entire body. + span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + body, err := readBody(r) + span.LogKV("bodySize", len(body)) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + req := &ingest.ShardedRequest{} + span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") + err = proto.DefaultSerializer.Unmarshal(body, req) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + urlVars := mux.Vars(r) + indexName := urlVars["index"] + + qcx := h.api.Txf().NewQcx() + err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + } + } else { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + qcx.Abort() + } +} + func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { diff --git a/ingest/codec.go b/ingest/codec.go index c966e3f1c..91b63000f 100644 --- a/ingest/codec.go +++ b/ingest/codec.go @@ -15,10 +15,13 @@ package ingest import ( + "bytes" + "encoding/json" "fmt" "io" "io/ioutil" "math" + "sort" "strconv" "time" @@ -37,15 +40,22 @@ import ( // decimal yes yes no // timestamp yes yes no -type KeyLookupFunc func(...string) (map[string]uint64, error) +// KeyTranslator is a thing that can translate strings to IDs, and also +// IDs back to strings. The ID->string conversion is used only to render +// a request back to JSON, which in turn is only used in testing. The +// functions optionally take an existing map which they then augment. +type KeyTranslator interface { + TranslateKeys(keys ...string) (map[string]uint64, error) + TranslateIDs(ids ...uint64) (map[uint64]string, error) +} // Codec is a single-use parser which decodes data into columnar vectors. type Codec interface { - AddSetField(name string, lookup KeyLookupFunc) error - AddTimeQuantumField(name string, lookup KeyLookupFunc) error - AddMutexField(name string, lookup KeyLookupFunc) error + AddSetField(name string, keys KeyTranslator) error + AddTimeQuantumField(name string, keys KeyTranslator) error + AddMutexField(name string, keys KeyTranslator) error AddBoolField(name string) error - AddIntField(name string, lookup KeyLookupFunc) error + AddIntField(name string, keys KeyTranslator) error AddDecimalField(name string, scale int64) error AddTimestampField(name string, scale time.Duration, epoch int64) error @@ -56,138 +66,299 @@ type Codec interface { type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error -// applyTranslationFn is a function which applies key lookups to the values -// of an operation, meaning it needs to know whether it's applying them -// to the signed or unsigned values. -type applyTranslationFn func(*FieldOperation, []uint64) error +// jsonEncFn encodes a value, or range of values, according to a given +// fieldCodec's translation rules. key values, if present, will be used to +// replace whichever values could have been provided as keys. so for instance, +// with an int field which is keyed, values are actually of type int64, but +// if strings are present, those are used. for a time quantum field, the time +// is set from the signed field, and the value is replaced by the key if +// keys are provided. +type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error -type jsonFieldCodec struct { +// fieldCodec represents something that can encode and decode a +// particular field. Its methods are not reentrant, it uses internal +// buffers. +type fieldCodec struct { valueKeys *StringTable currentOp *FieldOperation decode jsonDecFn - translate applyTranslationFn + encode jsonEncFn // For timestamp: scale-in-nanoseconds; for instance, if scaleUnit is // 1,000,000,000, we are storing numbers-of-seconds since the Unix epoch. // The actual value recorded in BSI will be offset by the field's // epoch, but we don't need to know that. // For decimal: Decimal digits of precision. So for instance, with - // scaleUnit 2, "1" is stored as 100 and "1.2" is stored as 120. + // scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as + // 120. scaleUnit int64 + scale int64 epoch int64 // used only by Timestamp fields scratch []uint64 // reusable scratch space for sets of values - lookup KeyLookupFunc + keys KeyTranslator + buf []byte // scratch space for format operations. fieldType FieldType } // JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps. type JSONCodec struct { - recKeys *StringTable - fields map[string]*jsonFieldCodec - keyLookup KeyLookupFunc - currentOp *Operation + fieldTypes map[string]FieldType + recKeys *StringTable + fields map[string]*fieldCodec + keys KeyTranslator + currentOp *Operation +} + +// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which +// can be used to write to its buffer. Both types are just embedded because +// they have non-overlapping APIs. Please don't look at my horrible face. +// +// This has some internal objects it can use to stash things +// so that it can pass &foo to enc.Encode() without needing to heap-allocate +// something for the interface conversion, and a buffer it can use for +// converting numbers or times. +// +// It also has an internal error buffer. In fact, in many cases, errors +// are so far as I can tell absolutely impossible -- bytes.Buffer specifically +// promises never to yield an error, and nothing seems to hint that +// enc.Encode can error on integers, strings, booleans, or arrays. So we +// have dozens of error checks that we can't cause to happen even with +// malformed inputs... so we eat those errors, don't require them to be +// checked externally, and return them when done if anyone checks. +type jsonBuffer struct { + *bytes.Buffer + enc *json.Encoder + buf [48]byte // scratch space for using strconv.AppendInt, etc + uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64 + strbuf []string // dummy buffer so that we don't have to alloc to print []string + str string // and again, "so we don't have to alloc a copy" + err error +} + +// Encode removes the stray newlines added by json.Encoder. +func (j *jsonBuffer) Encode(v interface{}) { + err := j.enc.Encode(v) + if err == nil { + j.Truncate(j.Len() - 1) + } else { + j.err = err + } +} + +// EncodeInt uses AppendInt into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeInt(i int64) { + rep := strconv.AppendInt(j.buf[:0], i, 10) + _, _ = j.Write(rep) +} + +// EncodeUint uses AppendUint into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeUint(u uint64) { + rep := strconv.AppendUint(j.buf[:0], u, 10) + _, _ = j.Write(rep) +} + +// EncodeUints uses a static copy of a []uint64 -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeUints(u []uint64) { + j.uintbuf = u + j.Encode(&j.uintbuf) + j.strbuf = nil +} + +// EncodeStrings uses a static copy of a []string -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeStrings(s []string) { + j.strbuf = s + j.Encode(&j.strbuf) + j.strbuf = nil +} + +// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs, +// while also surrounding the value with quotes. +func (j *jsonBuffer) EncodeQuotedUint(u uint64) { + j.buf[0] = '"' + rep := strconv.AppendUint(j.buf[1:1], u, 10) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeString appends the JSON encoding of a string. This exists +// because otherwise runtime allocates a heap-allocated copy of the +// string to live inside an interface{} for the duration of a function +// call... +func (j *jsonBuffer) EncodeString(s string) { + j.str = s + j.Encode(&j.str) + j.str = "" +} + +// EncodeTime exists because time's MarshalJSON allocates a new +// buffer every time it gets called, resulting in a full 13% of +// all the allocations produced in a test run, plus another 13% +// or so of them which were for the copies of the time objects +// made to stuff them into an interface{}. Eww. +func (j *jsonBuffer) EncodeTime(t time.Time) { + j.buf[0] = '"' + rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeBool writes a literal representation directly to reduce allocs. +func (j *jsonBuffer) EncodeBool(b bool) { + if b { + _, _ = j.WriteString("true") + } else { + _, _ = j.WriteString("false") + } +} + +func (j *jsonBuffer) Err() error { + return j.err +} + +func newJSONBuffer(data []byte) *jsonBuffer { + e := &jsonBuffer{Buffer: bytes.NewBuffer(data)} + e.enc = json.NewEncoder(e.Buffer) + e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON + return e } var _ Codec = &JSONCodec{} -func NewJSONCodec(lookup KeyLookupFunc) (*JSONCodec, error) { - j := &JSONCodec{fields: map[string]*jsonFieldCodec{}} - if lookup != nil { +func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) { + j := &JSONCodec{ + fields: map[string]*fieldCodec{}, + fieldTypes: map[string]FieldType{}, + } + if keys != nil { j.recKeys = NewStringTable() - j.keyLookup = lookup + j.keys = keys } return j, nil } -func (codec *JSONCodec) AddTimeQuantumField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeQuantum, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeTimeQuantum, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddSetField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeSet, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeSetValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeSet, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeSetValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddIntField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeInt, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeIntValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateSigned - } - return codec.addField(name, FieldTypeInt, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeIntValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddMutexField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} - fieldCodec.decode = fieldCodec.DecodeMutexValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned +func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeMutex, + keys: keys, } - return codec.addField(name, FieldTypeMutex, fieldCodec, lookup) + fieldCodec.decode = fieldCodec.DecodeMutexValue + fieldCodec.encode = fieldCodec.EncodeMutexValue + return codec.addField(name, fieldCodec) } func (codec *JSONCodec) AddBoolField(name string) error { - fieldCodec := &jsonFieldCodec{} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeBool, + } fieldCodec.decode = fieldCodec.DecodeBoolValue - return codec.addField(name, FieldTypeBool, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeBoolValue + return codec.addField(name, fieldCodec) } // TimestampField is used to store seconds since unix epoch. The numeric values // stored are adjusted based on the given time.Duration; for instance, if the // time scale is time.Second, then the second after the epoch is stored as 1, -// if it's time.Millisecond, then it's stored as 1000, etcetera. +// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch +// passed to this function should be the offset from the Unix epoch to the +// desired epoch, in the same scale. (So if the scale is milliseconds, +// it should be the Unix timestamp in seconds, times 1000.) func (codec *JSONCodec) AddTimestampField(name string, timeScale time.Duration, epoch int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(timeScale), epoch: epoch} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeStamp, + scaleUnit: int64(timeScale), + epoch: epoch, + } fieldCodec.decode = fieldCodec.DecodeTimeValue - return codec.addField(name, FieldTypeTimeStamp, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeTimeValue + return codec.addField(name, fieldCodec) } // AddDecimalField adds a decimal field, which is stored as integer values // with a scale offset, but parsed as floating point values. For instance, // with decimalScale=2, `0.01` would store the value 1. func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(math.Pow10(int(decimalScale)))} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeDecimal, + scale: decimalScale, + scaleUnit: int64(math.Pow(10, float64(decimalScale))), + } fieldCodec.decode = fieldCodec.DecodeDecimalValue - return codec.addField(name, FieldTypeDecimal, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeDecimalValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) addField(name string, fieldType FieldType, fieldCodec *jsonFieldCodec, lookup KeyLookupFunc) error { +func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error { if _, ok := codec.fields[name]; ok { return fmt.Errorf("duplicate field %q", name) } - if lookup != nil { + if fieldCodec.keys != nil { fieldCodec.valueKeys = NewStringTable() - fieldCodec.lookup = lookup } - fieldCodec.fieldType = fieldType codec.fields[name] = fieldCodec + codec.fieldTypes[name] = fieldCodec.fieldType return nil } // decodeSetOrValue decodes a value which might be either an array of values or a // single value, where values might be string keys or bare numbers, calling cb for // each value it finds. -func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { +func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { switch dataType { case jsonparser.Array: _, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - id, idErr := j.valueKeys.ID(value) - if idErr != nil { - err = idErr - return - } - // stash an error if we got one - valueErr := cb(id) - if valueErr != nil { - err = valueErr + var id uint64 + switch dataType { + case jsonparser.String: + id, err = j.valueKeys.ID(value) + if err != nil { + return + } + err = cb(id) + case jsonparser.Number: + if j.valueKeys != nil { + err = errors.New("expecting key, got numeric value") + return + } + id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64) + if err != nil { + return + } + err = cb(id) + default: + err = fmt.Errorf("expecting value or array, got %v", dataType) } }) if err != nil { @@ -212,23 +383,32 @@ func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data [] } return cb(id) default: - return fmt.Errorf("expecting array, got %v", dataType) + return fmt.Errorf("expecting value or array, got %v", dataType) } return err } // DecodeSetValue decodes a set of unsigned values from the provided data into the // associated currentOp. -func (j *jsonFieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { return j.decodeSetOrValue(dataType, data, func(id uint64) error { j.currentOp.AddPair(recID, id) return nil }) } +func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 1 || len(values) == 1 { + // simplify: just hand this off to a single-value case + return j.EncodeMutexValue(dst, values, signed, keys) + } + appendKeysJSON(dst, values, keys) + return nil +} + // DecodeIntValue decodes a single signed value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String: value, err := j.valueKeys.IntID(data) @@ -255,25 +435,62 @@ func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueT return nil } +func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(signed) == 0 { + return errors.New("encodeIntValue: need a value") + } + dst.EncodeInt(signed[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeMutexValue decodes a single unsigned value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { - case jsonparser.Number, jsonparser.String: - id, err := j.valueKeys.ID(data) + case jsonparser.String: + value, err := j.valueKeys.ID(data) if err != nil { return err } - j.currentOp.AddPair(recID, id) + j.currentOp.AddPair(recID, value) + case jsonparser.Number: + if j.valueKeys != nil { + return errors.New("expecting string key, got numeric value") + } + value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) + if err != nil { + return err + } + j.currentOp.AddPair(recID, value) default: - return fmt.Errorf("expecting integer value, got %v", dataType) + if j.valueKeys != nil { + return fmt.Errorf("expecting string key, got %v", dataType) + } else { + return fmt.Errorf("expecting integer value, got %v", dataType) + } } return nil } +func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(values) == 0 { + return errors.New("encodeMutexValue: need a value") + } + dst.EncodeUint(values[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeBoolValue decodes a single true/false value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { value := uint64(0) switch typ { case jsonparser.String: @@ -303,9 +520,21 @@ func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, return nil } +func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding boolean value, but none provided") + } + var x bool + if values[0] != 0 { + x = true + } + dst.EncodeBool(x) + return nil +} + // DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the // provided data into the associated currentOp. -func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { j.scratch = j.scratch[:0] stamp := time.Unix(0, 0).UTC() err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { @@ -346,8 +575,27 @@ func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.Val return nil } +// Our format does not allow a record to have multiple values set with +// different timestamps at the same time. We use the first timestamp for +// the whole set. +func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding time quantum value: no value provided") + } + dst.WriteString(`{"values":`) + appendKeysJSON(dst, values, keys) + // a zero timestamp is idiomatic for no-time-provided, and i sort of + // hate that, but here we are. + if len(signed) > 0 && signed[0] != 0 { + _, _ = dst.WriteString(`,"time":`) + dst.EncodeTime(time.Unix(0, signed[0]).UTC()) + } + _, _ = dst.WriteString(`}`) + return nil +} + // DecodeTimeValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { var stamp time.Time switch dataType { case jsonparser.String: @@ -364,12 +612,22 @@ func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.Value return fmt.Errorf("parsing numeric timestamp: %w", err) } j.currentOp.AddSignedPair(recID, i64) + default: + return fmt.Errorf("expecting time, got %s", dataType) } return nil } +func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(signed) == 0 { + return errors.New("encoding time value: no value provided") + } + dst.EncodeTime(time.Unix(0, (signed[0]+j.epoch)*j.scaleUnit).UTC()) + return nil +} + // DecodeDecimalValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String, jsonparser.Number: value, err := jsonparser.GetFloat(data) @@ -384,6 +642,28 @@ func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.Va return nil } +func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10) + scale := int(j.scale) + if len(j.buf) > scale { + j.buf = append(j.buf, '.') + // shove the last scaleUnit values over + copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:]) + j.buf[len(j.buf)-scale-1] = '.' + } + _, _ = dst.Write(j.buf) + return nil +} + +// FieldTypes gives a mapping of fields to their basic types used by this +// codec. +func (codec *JSONCodec) FieldTypes() map[string]FieldType { + return codec.fieldTypes +} + +// ParseKeyedRecords parses the records it finds. Note that, when you're doing +// a Write op, this will update ClearRecordIDs automatically as it goes, +// even though record_ids isn't specified in the JSON for that case. func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { seen := make(map[uint64]struct{}) return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { @@ -412,8 +692,8 @@ func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { }) } -func (codec *JSONCodec) ParseOperation(data []byte) (op *Operation, err error) { - op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields))} +func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) { + op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq} codec.currentOp = op err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { switch string(key) { @@ -491,10 +771,12 @@ func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { var ops []*Operation var lastErr error + var seq int _, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { switch dataType { case jsonparser.Object: - op, err := codec.ParseOperation(value) + op, err := codec.ParseOperation(value, seq) + seq++ if err != nil { lastErr = fmt.Errorf("parsing operation: %v", err) return @@ -512,34 +794,39 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { } // and now, key translation! var keyMap []uint64 - if codec.keyLookup != nil { - keyMap, err = MapForStringTable(codec.recKeys, codec.keyLookup) + if codec.keys != nil { + keyMap, err = codec.recKeys.MakeIDMap(codec.keys) if err != nil { return nil, fmt.Errorf("trying to find record key mapping: %w", err) } } - req = &Request{FieldTypes: make(map[string]FieldType, len(codec.fields))} + req = &Request{} valueMaps := map[string]func(*FieldOperation) error{} for name, fieldCodec := range codec.fields { // make closure survive iteration fieldCodec := fieldCodec - if fieldCodec.lookup != nil { - fieldMap, err := MapForStringTable(fieldCodec.valueKeys, fieldCodec.lookup) + if fieldCodec.keys != nil { + fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys) if err != nil { return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err) } - valueMaps[name] = func(fo *FieldOperation) error { - return fieldCodec.translate(fo, fieldMap) + if fieldCodec.fieldType == FieldTypeInt { + valueMaps[name] = func(fo *FieldOperation) error { + return translateSigned(fieldMap, fo.Signed) + } + } else { + valueMaps[name] = func(fo *FieldOperation) error { + return translateUnsigned(fieldMap, fo.Values) + } } } - req.FieldTypes[name] = fieldCodec.fieldType } for _, op := range ops { // For Clear and Write, we need to translate/sort our record ID // list. if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { if keyMap != nil { - if err = translateUnsignedSlice(op.ClearRecordIDs, keyMap); err != nil { + if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for clear op: %w", err) } } @@ -554,7 +841,7 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { continue } if keyMap != nil { - if err = fieldOp.TranslateKeys(keyMap); err != nil { + if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err) } } @@ -574,6 +861,283 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { return req, nil } +// AppendBytes appends bytes which we would expect to produce the same +// request, using this codec. It does not try to recreate FieldTypes, which +// would be handled by the codec anyway. It's written as an Append so you +// can reuse a buffer. +func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) { + if req == nil || len(req.Ops) == 0 { + return append(data, "[]"...), nil + } + dst := newJSONBuffer(data) + // we ignore the FieldTypes part of the Request, which is just there to + // let the request's ByShard use the correct sorting routines, which is + // itself sort of awful. The codec will insert it again when parsing the + // bytes. + _, _ = dst.WriteString(`,`) + for _, op := range req.Ops { + // EncodeJSON needs to have access to this codec's field data + // and key translators. + err = op.EncodeJSON(dst, codec) + if err != nil { + return nil, err + } + _, _ = dst.WriteString(`,`) + } + // delete the last comma + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + // there's very few ways an error could occur, possibly none, but + // just in case lots of internal operations stashed an error if one + // happened, so we'll return anything they came up with. + return dst.Bytes(), dst.Err() +} + +// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed +// list of quoted strings if there's a key translator, or integer values +// if the KeyTranslator is nil, comma-separated. It can error because a +// translator can error. +func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + return appendIDsJSON(dst, values, keys) +} + +func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) { + if len(values) == 0 && len(keys) == 0 { + _, _ = dst.WriteString("[]") + return + } + if len(keys) != 0 { + dst.EncodeStrings(keys) + } else { + dst.EncodeUints(values) + } +} + +func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + if len(values) == 0 { + _, _ = dst.WriteString("[]") + return nil + } + + if keys == nil { + dst.EncodeUints(values) + return nil + } + _, _ = dst.WriteString(`[`) + translated, err := keys.TranslateIDs(values...) + if err != nil { + return err + } + for _, v := range values { + dst.EncodeString(translated[v]) + _, _ = dst.WriteString(`,`) + } + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + return nil +} + +// RequestByShard makes up for the fact that we don't want to stash the +// field type data in the request, but we need it to actually do the by-shard. +func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) { + return req.ByShard(codec.fieldTypes) +} + +// EncodeJSON encodes an operation as JSON using a provided buffer. +// It uses the provided codec where necessary to help with key +// translation. +func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) { + // We're not trying to guarantee that what we produce makes sense or is + // identical to what produced us, just to write our current state out. + if o == nil { + _, _ = dst.WriteString(`{}`) + return nil + } + _, _ = dst.WriteString(`{"action":`) + dst.EncodeString(o.OpType.String()) + // it is intentional that o.Seq isn't encoded here; it makes no sense + // to allow an op to specify its seq in JSON. + if o.OpType != OpWrite { + // for Write, ClearRecordIDs and ClearFields were computed + // from the records being written, and aren't actually part of the data. + if len(o.ClearRecordIDs) != 0 { + _, _ = dst.WriteString(`,"record_ids":`) + err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys) + if err != nil { + return err + } + } + if len(o.ClearFields) != 0 { + _, _ = dst.WriteString(`,"fields":`) + dst.EncodeStrings(o.ClearFields) + } + } + if len(o.FieldOps) == 0 { + _, _ = dst.WriteString(`}`) + return + } + // collect all the fields that have a non-empty set of records. we can + // just ignore the others. + fieldNames := make([]string, 0, len(o.FieldOps)) + for field, op := range o.FieldOps { + if op != nil && len(op.RecordIDs) != 0 { + fieldNames = append(fieldNames, field) + } + } + // nevermind then + if len(fieldNames) == 0 { + _, _ = dst.WriteString(`}`) + return nil + } + _, _ = dst.WriteString(`,"records":{`) + + // now we have to invert the logic, creating records from + // fields with corresponding ops. uh-oh. + fieldOps := make([]*FieldOperation, len(o.FieldOps)) + fieldCodecs := make([]*fieldCodec, len(o.FieldOps)) + fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys + indexes := make([]int, len(o.FieldOps)) + sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] }) + next := ^uint64(0) + var idKeys map[uint64]string + if codec.keys != nil { + idKeys = make(map[uint64]string) + } + for i, field := range fieldNames { + // populate a parallel slice of field ops so we don't have + // to do map lookups for every single piece of data + op := o.FieldOps[field] + fieldOps[i] = op + fc := codec.fields[field] + if fc == nil { + return fmt.Errorf("unknown field: %q", field) + } + fieldCodecs[i] = fc + if codec.keys != nil { + // we'll build a list of keys we need for any records + for _, v := range op.RecordIDs { + idKeys[v] = "" + } + } + id := op.RecordIDs[0] + if id < next { + next = id + } + if fc.keys != nil { + thisFieldKeys := make([]string, len(op.RecordIDs)) + if fc.fieldType == FieldTypeInt { + u := make([]uint64, len(op.Signed)) + for i := range op.Signed { + u[i] = uint64(op.Signed[i]) + } + valueKeys, err := fc.keys.TranslateIDs(u...) + if err != nil { + return err + } + for j, v := range op.Signed { + thisFieldKeys[j] = valueKeys[uint64(v)] + } + } else { + valueKeys, err := fc.keys.TranslateIDs(op.Values...) + if err != nil { + return err + } + for j, v := range op.Values { + thisFieldKeys[j] = valueKeys[v] + } + } + fieldKeys[i] = thisFieldKeys + } + } + // ... no fieldops actually have any entries. therefore no record will + // have any fields set, therefore no records exist... + if next == ^uint64(0) { + _, _ = dst.WriteString(`}}`) + return nil + } + if codec.keys != nil { + idList := make([]uint64, 0, len(idKeys)) + for k := range idKeys { + idList = append(idList, k) + } + idKeys, err = codec.keys.TranslateIDs(idList...) + if err != nil { + return err + } + } + + // next is the ID of a record which exists for at least one field. + // we'll recompute it every loop. + for next != ^uint64(0) { + if codec.keys != nil { + dst.EncodeString(idKeys[next]) + } else { + dst.EncodeQuotedUint(next) + } + _, _ = dst.WriteString(`:{`) + current := next + next = ^uint64(0) + for i, field := range fieldNames { + op := fieldOps[i] + idx := indexes[i] + if idx >= len(op.RecordIDs) { + continue + } + id := op.RecordIDs[idx] + if id == current { + var j int + // count ahead to first index which is either outside the list + // or a different id + for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ { + } + // fmt.Printf("field %s encoding %d-%d (v %d, s %d, k %d)\n", + // field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i])) + // print this one, and advance this index to next position + dst.EncodeString(field) + _, _ = dst.WriteString(`:`) + var values []uint64 + var signed []int64 + var keys []string + if len(op.Values) >= j { + values = op.Values[idx:j] + } + if len(op.Signed) >= j { + signed = op.Signed[idx:j] + } + if len(fieldKeys[i]) >= j { + keys = fieldKeys[i][idx:j] + } + err = fieldCodecs[i].encode(dst, values, signed, keys) + if err != nil { + return err + } + _, _ = dst.WriteString(`,`) + indexes[i] = j + // and we'll fall through to the id < next check, so we + // just set id here. + if indexes[i] < len(op.RecordIDs) { + id = op.RecordIDs[indexes[i]] + } else { + id = ^uint64(0) + } + } + if id < next { + next = id + } + } + // we should always have a trailing comma after any entry, and + // if there were no entries we shouldn't have had a list... + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`},`) + } + dst.Truncate(dst.Len() - 1) + // close both the records list, and the whole object. + _, _ = dst.WriteString(`}}`) + // In theory, there's no way for most of these to produce errors, + // but just in case, we'll check for an error every operation or so. + return dst.Err() +} + type errFieldNotFound struct { field string } diff --git a/ingest/codec_test.go b/ingest/codec_test.go index 4490d3729..8605eef5a 100644 --- a/ingest/codec_test.go +++ b/ingest/codec_test.go @@ -16,36 +16,481 @@ package ingest import ( "fmt" + "sort" + "strings" "testing" "time" "github.com/molecula/featurebase/v2/shardwidth" ) -func unusableSampleTranslator(keys ...string) (map[string]uint64, error) { - out := make(map[string]uint64, len(keys)) - for _, key := range keys { - out[key] = uint64(len(out)) * 13 +func TestStableTranslator(t *testing.T) { + tr := newStableTranslator() + m1, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("translation error on initial keys: %v", err) + } + m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6) + if err != nil { + t.Fatalf("translation error on reverse lookup: %v", err) + } + m3, err := tr.TranslateKeys("a", "k-6") + if err != nil { + t.Fatalf("translation error on new keys: %v", err) + } + for k, v := range m3 { + if m2[v] != k { + t.Fatalf("expected round trip to equate %q and %d", k, v) + } } - return out, nil } -func TestSimpleCodec(t *testing.T) { - c, _ := NewJSONCodec(nil) - _ = c.AddSetField("set", nil) - _ = c.AddSetField("setkeys", unusableSampleTranslator) - _ = c.AddMutexField("mutex", nil) - _ = c.AddMutexField("mutexkeys", unusableSampleTranslator) - _ = c.AddTimeQuantumField("tq", nil) - _ = c.AddIntField("int", nil) - _ = c.AddIntField("intkeys", unusableSampleTranslator) +func TestMakeCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + err := codec.AddSetField("set", nil) + if err != nil { + t.Fatalf("unexpected error creating field: %v", err) + } + err = codec.AddSetField("set", nil) + if err == nil { + t.Fatalf("expected error creating duplicate field, didn't get it") + } +} + +func TestEncode(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) epoch, err := time.Parse("2006-01-02", "2020-01-01") if err != nil { t.Fatalf("can't parse sample epoch time: %v", err) } - _ = c.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) - _ = c.AddDecimalField("dec", 2) - _ = c.AddBoolField("bool") + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs := []*JSONCodec{codec} + + // redo all of that, only on a keyed translator + codec, _ = NewJSONCodec(newStableTranslator()) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs = append(codecs, codec) + + encodeTests := []*Request{ + { + Ops: []*Operation{ + { + OpType: OpWrite, + ClearRecordIDs: []uint64{0, 1, 2, 3, 5}, + ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"}, + FieldOps: map[string]*FieldOperation{ + "int": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, -3}, + }, + "intkeys": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, 1}, + }, + "set": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "setkeys": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "mutex": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "mutexkeys": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "tq": { + RecordIDs: []uint64{5, 5}, + Values: []uint64{8, 9}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "tqkeys": { + RecordIDs: []uint64{3, 3}, + Values: []uint64{2, 4}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "ts": { + RecordIDs: []uint64{0}, + Signed: []int64{1}, + }, + "bool": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{0, 1}, + }, + "dec": { + RecordIDs: []uint64{0, 1, 2}, + Signed: []int64{123, -123, 0}, + }, + }, + }, + { + OpType: OpClear, + Seq: 1, + ClearRecordIDs: []uint64{6}, + ClearFields: []string{"tq"}, + }, + }, + }, + { + // this one needs to get filled in programmatically; see below + Ops: []*Operation{ + { + OpType: OpSet, + FieldOps: map[string]*FieldOperation{}, + }, + }, + }, + { + Ops: []*Operation{ + { + OpType: OpRemove, + FieldOps: map[string]*FieldOperation{ + "set": {}, + }, + }, + }, + }, + } + // and now we populate encodeTests[1] with a larger pool of data + const dataSize = 5000 + shardCount := uint64(600) // shards we want to target + passes := uint64(0) + recordIDs := make([]uint64, dataSize) + values := make([]uint64, dataSize) + timeStamps := make([]int64, dataSize) + signedValues := make([]int64, dataSize) + for i := uint64(0); i < dataSize; i++ { + if (i % shardCount) == 0 { + passes++ + } + recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes + values[i] = (i % 4) + timeStamps[i] = int64(1234567890e9 + (i * 100e9)) + signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys + } + // ensure record IDs are sorted, because other stuff might rely on this + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op := encodeTests[1].Ops[0] + op.FieldOps["tq"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + Signed: append([]int64{}, timeStamps...), + } + op.FieldOps["tqkeys"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: values, + Signed: timeStamps, + } + op.FieldOps["int"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Signed: append([]int64{}, signedValues...), + } + op.FieldOps["intkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Signed: signedValues, + } + // for sets, we want to shuffle things into fewer shards, and ensure + // non-duplication of values within each record, but also have lots + // of duplication of record IDs in the low shards + recordIDs = make([]uint64, dataSize) + values = make([]uint64, dataSize) + valuesPerRecord := uint64(5) + recordsPerShard := dataSize / valuesPerRecord / 30 + if recordsPerShard < 1 { + recordsPerShard = 1 + } + shard := uint64(0) + nextID := uint64(0) + nextValue := uint64(0) + for i := uint64(0); i < dataSize; i++ { + recordIDs[i] = nextID + values[i] = nextValue + (i % valuesPerRecord) + nextValue++ + if nextValue == valuesPerRecord { + nextValue = 0 + nextID++ + if nextID%(1< 1 { + valuesPerRecord-- + } + } + } + } + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op.FieldOps["set"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + } + op.FieldOps["setkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + var buf []byte + for i, tc := range encodeTests { + for _, c := range codecs { + data, err := c.AppendBytes(tc, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + data, err = c.AppendBytes(req, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req2, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req2.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + } + } +} + +func TestCodecErrors(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + testCases := []struct { + name string + json []byte + error string + }{ + { + name: "no action", + json: []byte(`[{"records":{"0":{"set":[0]}}}]`), + error: "action not specified", + }, + { + name: "unknown action", + json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`), + error: "unknown action", + }, + { + name: "unknown field", + json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`), + error: "field not found", + }, + { + name: "unknown operation field", + json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`), + error: "unknown operation field", + }, + { + name: "expected operation", + json: []byte(`[true]`), + error: "expected operation", + }, + { + name: "expecting key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`), + error: "expecting key", + }, + { + name: "invalid int for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`), + error: "boolean should be", + }, + { + name: "invalid number for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`), + error: "looks like Number", + }, + { + name: "invalid string for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`), + error: "expecting boolean", + }, + { + name: "nonsense bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`), + error: "boolean should be", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`), + error: "expecting value", + }, + { + name: "expecting array-key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`), + error: "expecting key", + }, + { + name: "expecting array-value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`), + error: "expecting value", + }, + { + name: "expecting numeric array-value", + json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`), + error: "invalid syntax", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting int key", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting int value", + json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting string array-value", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "expecting numeric mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting mutex key", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting mutex string value", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "time quantum invalid time", + json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`), + error: "expecting time", + }, + { + name: "time stamp invalid integer", + json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`), + error: "parsing numeric time", + }, + { + name: "time stamp invalid string", + json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`), + error: "parsing time", + }, + { + name: "time stamp invalid type", + json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`), + error: "expecting time", + }, + { + name: "invalid decimal", + json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`), + error: "expecting floating", + }, + { + name: "duplicate record", + json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`), + error: "duplicated in input", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req, err := codec.ParseBytes(tc.json) + if err == nil { + req.Dump(t.Logf) + t.Fatalf("expected error like %q, got request instead", tc.error) + } else { + msg := err.Error() + if !strings.Contains(msg, tc.error) { + t.Fatalf("expected error like %q, got %q", tc.error, msg) + } + } + }) + } +} + +func TestSimpleCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") var nextShard = uint64(1< 0 { - subOp.Values = f.Values[prev:endIndex] - } - if len(f.Signed) > 0 { - subOp.Signed = f.Signed[prev:endIndex] - } - target[shard] = subOp - prev = endIndex +// clone makes a duplicate of the operation without shared storage +func (f *FieldOperation) clone() *FieldOperation { + f2 := &FieldOperation{ + RecordIDs: append([]uint64{}, f.RecordIDs...), + Values: append([]uint64{}, f.Values...), + Signed: append([]int64{}, f.Signed...), } + return f2 } func ShardIDs(ids []uint64) (out map[uint64][]uint64) { @@ -349,10 +396,22 @@ func (f *FieldOperation) SortByKeys(keys []uint64) { // doesn't help as much as you'd hope. This beats using stdlib sort by about // a factor of two for those small N, for larger N we're using the radix sort // that calls this. +// +// External keys exist only when we are sorting by value, which is to say, +// when we're using row-oriented formats (set, mutex, time quantum). +// For int/decimal/timestamp fields, we're sorting by record only. +// So, if keys is the same as f.RecordIDs, we're looking at an int field +// or equivalent, so Signed exists and Values doesn't exist. +// Otherwise, we might be looking at a time quantum field (both exist) +// or set/mutex (only Values exist). func simpleSort(f *FieldOperation, keys []uint64) { - if keys != nil { + // keys might actually just point to record IDs, in which case, we don't + // want to shuffle the corresponding RecordIDs too, because that would just + // reverse our swaps. If they're different, we actually need to swap them + // both. + if &keys[0] != &f.RecordIDs[0] { // sorting by record IDs - if f.Values != nil && f.Signed != nil { + if f.Values != nil && f.Signed != nil { // time quantum field for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -362,7 +421,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } - } else if f.Values != nil { + } else if f.Values != nil { // set/mutex/bool for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -371,7 +430,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { + } else if f.Signed != nil { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -379,7 +438,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else { + } else { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -388,7 +447,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } } else { - if f.Values != nil && f.Signed != nil { + if f.Values == nil && f.Signed != nil { + for i := 1; i < len(f.RecordIDs); i++ { + for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { + f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] + f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] + } + } + } else if f.Values != nil && f.Signed != nil { // can't happen, we think for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -396,22 +462,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else if f.Values != nil { + } else if f.Values != nil { // only happens during testing for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else { - // why do we only have record IDs? I don't know + } else { // should definitely not happen for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -493,12 +551,7 @@ func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) { if end-start > 32 { sortPartialByKeys(&bucketOp, keys[start:end], nextShift) } else { - // naive stdlib sort - if externalKeys { - simpleSort(&bucketOp, keys[start:end]) - } else { - simpleSort(&bucketOp, nil) - } + simpleSort(&bucketOp, keys[start:end]) } } } @@ -534,16 +587,15 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { if expected == nil { return nil } - if len(expected.RecordIDs) == 0 && len(expected.Values) == 0 && len(expected.Signed) == 0 { + // We don't worry about non-empty Values or Signed here, because in theory + // RecordIDs are the Source of Truth as to what's in the op. + if len(expected.RecordIDs) == 0 { return nil } return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs)) } if expected == nil { - if got == nil { - return nil - } - if len(got.RecordIDs) == 0 && len(got.Values) == 0 && len(got.Signed) == 0 { + if len(got.RecordIDs) == 0 { return nil } return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs)) @@ -578,52 +630,6 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { return nil } -func translateUnsignedSlice(target []uint64, mapping []uint64) (err error) { - oops := 0 - for i, v := range target { - if v >= uint64(len(mapping)) { - oops++ - } else { - target[i] = mapping[v] - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range keys when applying translation mapping", oops) - } - return nil -} - -// TranslateUnsigned translates keys according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateKeys(mapping []uint64) error { - return translateUnsignedSlice(op.RecordIDs, mapping) -} - -// TranslateUnsigned translates values according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateUnsigned(mapping []uint64) error { - return translateUnsignedSlice(op.Values, mapping) -} - -// TranslateSigned translates signed values according to the provided mapping. -// If we're using this, it's because we're in an integer-type field, which -// admits using keys for fields, so all key values are actually non-negative, -// but the field's type still requires values be expressed as signed ints. -func (op *FieldOperation) TranslateSigned(mapping []uint64) error { - oops := 0 - for i, v := range op.Signed { - if v >= int64(len(mapping)) { - oops++ - } else { - op.Signed[i] = int64(mapping[v]) - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} - // ShardOperations is a set of Operations associated with a specific shard. type ShardOperations struct { Shard uint64 @@ -633,19 +639,17 @@ type ShardOperations struct { // Request is a complete ingest request, which may be any combination // of operations, which may apply to multiple shards. type Request struct { - FieldTypes map[string]FieldType - Ops []*Operation + Ops []*Operation } // ShardedRequest is an ingest request, split up into individual per-shard // operations. type ShardedRequest struct { - FieldTypes map[string]FieldType - Ops map[uint64][]*Operation + Ops map[uint64][]*Operation } // ByShard converts a request into the same request, only sharded. -func (r *Request) ByShard() (*ShardedRequest, error) { +func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) { if len(r.Ops) == 0 { return &ShardedRequest{Ops: nil}, nil } @@ -662,12 +666,12 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { sharded := ShardIDs(op.ClearRecordIDs) for shard, data := range sharded { - shards[shard] = &Operation{OpType: op.OpType, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} + shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} } } for field, fieldOp := range op.FieldOps { sharded := fieldOp.ByShard() - sorter := fieldTypeSorts[r.FieldTypes[field]] + sorter := fieldTypeSorts[fields[field]] if sorter == nil { sorter = (*FieldOperation).SortByRecords } @@ -678,7 +682,7 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpWrite { return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard) } - shardOp = &Operation{OpType: op.OpType} + shardOp = &Operation{OpType: op.OpType, Seq: op.Seq} shards[shard] = shardOp shardOp.FieldOps = map[string]*FieldOperation{field: data} } else { @@ -697,18 +701,139 @@ func (r *Request) ByShard() (*ShardedRequest, error) { return &ShardedRequest{Ops: req}, nil } +// merge combines the components of a sharded request back into a single +// unsharded request, processing shards in numerical order. +func (s *ShardedRequest) Merge() *Request { + req := &Request{} + if s == nil || len(s.Ops) == 0 { + return req + } + shards := make([]uint64, 0, len(s.Ops)) + for shard := range s.Ops { + shards = append(shards, shard) + } + sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] }) + for _, shard := range shards { + ops := s.Ops[shard] + for _, op := range ops { + var _ *Operation + if op.Seq >= len(req.Ops) { + // Pad out with nil *Operations to the required length + req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...) + } + if req.Ops[op.Seq] == nil { + req.Ops[op.Seq] = op.clone() + continue + } + req.Ops[op.Seq].merge(op) + } + } + return req +} + func (r *Request) Dump(logf func(string, ...interface{})) { logf("req: %#v", r) for _, op := range r.Ops { logf("op: %#v", op) if len(op.ClearRecordIDs) > 0 { - logf(" clearRecordIDs: %d", op.ClearRecordIDs) + if len(op.ClearRecordIDs) > 8 { + logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8) + } else { + logf(" clearRecordIDs: %d", op.ClearRecordIDs) + } } if len(op.ClearFields) > 0 { - logf(" clearFields: %s", op.ClearFields) + if len(op.ClearFields) > 8 { + logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8) + } else { + logf(" clearFields: %s", op.ClearFields) + } } for field, fieldOp := range op.FieldOps { - logf(" field %q: %#v", field, fieldOp) + if fieldOp != nil { + logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed)) + if len(fieldOp.RecordIDs) > 0 { + if len(fieldOp.RecordIDs) > 8 { + logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8) + } else { + logf(" records %d", fieldOp.RecordIDs) + } + } + } else { + logf(" field %q: nil op", field) + } } } } + +func (r *Request) Compare(other *Request) error { + if other == nil { + if r != nil && len(r.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if r == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + ops := r.Ops + ops2 := other.Ops + if len(ops2) != len(ops) { + return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("op %d: %v", i, err) + } + } + return nil +} + +// Compare checks whether two ShardedRequest objects represent the same +// data. Empty shards shouldn't have entries in the map in the first place, +// so we don't accept a nil or 0-length slice of ops as equal to the +// shard key not existing, but we do accept nil or empty requests as +// equal to each other. +func (s *ShardedRequest) Compare(other *ShardedRequest) error { + if other == nil { + if s != nil && len(s.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if s == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + for shard, ops := range s.Ops { + ops2, ok := other.Ops[shard] + if !ok { + return fmt.Errorf("shard %d missing in other", shard) + } + if len(ops2) != len(ops) { + return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("shard %d, op %d: %v", shard, i, err) + } + } + } + if len(other.Ops) != len(s.Ops) { + for shard := range other.Ops { + if _, ok := s.Ops[shard]; !ok { + return fmt.Errorf("shard %d missing in self", shard) + } + } + } + return nil +} diff --git a/ingest/op_test.go b/ingest/op_test.go index b0b196e07..db3b5981e 100644 --- a/ingest/op_test.go +++ b/ingest/op_test.go @@ -16,7 +16,6 @@ package ingest_test import ( "math/rand" - "reflect" "testing" "github.com/molecula/featurebase/v2/ingest" @@ -25,14 +24,14 @@ import ( type opShardingTestCase struct { name string - input ingest.Request + input *ingest.Request output *ingest.ShardedRequest } var opShardingTestCases = []opShardingTestCase{ { name: "sample", - input: ingest.Request{ + input: &ingest.Request{ Ops: []*ingest.Operation{ { OpType: ingest.OpSet, @@ -56,6 +55,7 @@ var opShardingTestCases = []opShardingTestCase{ }, { OpType: ingest.OpRemove, + Seq: 1, FieldOps: map[string]*ingest.FieldOperation{ "shard0-2": { RecordIDs: []uint64{1, 2< 1 { + valuesPerRecord-- + } + } + } + } + op.FieldOps["set"] = &ingest.FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + fieldTypes := codec.FieldTypes() + sharded, err := req.ByShard(fieldTypes) + if err != nil { + t.Errorf("sharding: unexpected error %v", err) + } + merged := sharded.Merge() + if err := req.Compare(merged); err != nil { + t.Fatalf("merge comparison: %v", err) } } diff --git a/ingest/translate.go b/ingest/translate.go index 7ce7ea6dd..42f61e291 100644 --- a/ingest/translate.go +++ b/ingest/translate.go @@ -13,3 +13,74 @@ // limitations under the License. package ingest + +import ( + "fmt" +) + +// stableTranslator implements a key translator that can be reused and +// will continue to give the same keys for the same values. Possibly +// surprisingly, it will invent new keys for IDs it is asked about but +// hasn't seen. This allows us to give a codec which would use keys on +// translation a request which contains arbitrary numbers, and request +// text that would parse into that request. +type stableTranslator struct { + in map[string]uint64 + out map[uint64]string + next uint64 +} + +func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + ret := make(map[string]uint64, len(keys)) + for _, key := range keys { + if existing, ok := s.in[key]; ok { + ret[key] = existing + continue + } + id := s.next + // but what if someone already translated that ID, so now it already + // exists? + if _, ok := s.out[id]; ok { + for k := range s.out { + if k > id { + id = k + } + } + // one larger than the largest we already have. this could + // wrap around, in which case, it's your own fault. + id++ + } + s.next = id + 1 + s.in[key] = id + s.out[id] = key + ret[key] = id + } + return ret, nil +} + +func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + ret := make(map[uint64]string, len(ids)) + for _, id := range ids { + if existing, ok := s.out[id]; ok { + ret[id] = existing + continue + } + key := fmt.Sprintf("k-%d", id) + s.in[key] = id + s.out[id] = key + ret[id] = key + if id >= s.next { + s.next = id + 1 + } + } + return ret, nil +} + +// newStableTranslator produces a translator which can translate forwards +// and backwards and invent new things if it needs to. Don't use this. +func newStableTranslator() *stableTranslator { + return &stableTranslator{ + in: make(map[string]uint64), + out: make(map[uint64]string), + } +} diff --git a/ingest/translate_test.go b/ingest/translate_test.go new file mode 100644 index 000000000..78deefda7 --- /dev/null +++ b/ingest/translate_test.go @@ -0,0 +1,70 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "fmt" + "testing" +) + +func TestTranslateReuse(t *testing.T) { + // the original stable-translator design had a flaw in that it + // assumed that each new ID would always come from a string translation, + // never from a key translation, and that they'd show up sequentially. + tr := newStableTranslator() + orig, err := tr.TranslateIDs(1, 2) + tr.next = 1 // intentionally break the translation logic for testing purposes + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var s [6]string + stash := s[:0] + for _, v := range orig { + stash = append(stash, v) + } + for i := range s[len(orig):] { + stash = append(stash, fmt.Sprintf("key-%d", i)) + } + keys, err := tr.TranslateKeys(stash...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + ids := make([]uint64, 0, len(keys)) + for _, v := range keys { + ids = append(ids, v) + } + idMap, err := tr.TranslateIDs(ids...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for k, v := range keys { + if idMap[v] != k { + t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q", + k, v, v, idMap[v]) + } + } + for id, key := range idMap { + if keys[key] != id { + t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } + for id, key := range orig { + if keys[key] != id { + t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } +} diff --git a/ingest/vec.go b/ingest/vec.go index 082ff8575..14d62cbbc 100644 --- a/ingest/vec.go +++ b/ingest/vec.go @@ -19,8 +19,6 @@ import ( "reflect" "strconv" "unsafe" - - "github.com/pkg/errors" ) // StringTable is a mapping of strings to temporary IDs. @@ -85,11 +83,8 @@ func (tbl *StringTable) IntID(in []byte) (int64, error) { // integers and a translation function from strings to "real" keys, yields // a translation/lookup slice. If it cannot translate all the keys, it // returns an error. -// -// The lookup function corresponds to the FindKeys/CreateKeys methods of -// featurebase translators, by an AMAZING coincidence. -func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint64, error)) ([]uint64, error) { - lookedUp, err := lookup(tbl.names...) +func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) { + lookedUp, err := keys.TranslateKeys(tbl.names...) if err != nil { return nil, err } @@ -103,22 +98,38 @@ func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint return out, nil } -// TimeFormatForUnit returns the time transfer format (between the update encoder and the update applier) with appropriate resolution for a quantum unit. -func TimeFormatForUnit(unit rune) string { - switch unit { - case 'Y': - return "2006" - case 'M': - return "200601" - case 'D': - return "20060102" - case 'H': - return "2006010203" - default: - panic(errors.Errorf("invalid quantum unit: %q", unit)) +// translateSigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateSigned(mapping []uint64, values []int64) error { + oops := 0 + for i, v := range values { + if v >= int64(len(mapping)) { + oops++ + } else { + values[i] = int64(mapping[v]) + } } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil } -// TODO: bool - -// TODO: timestamp (just sugar on top of IntVector) +// translateUnsigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateUnsigned(mapping []uint64, values []uint64) error { + oops := 0 + for i, v := range values { + if v >= uint64(len(mapping)) { + oops++ + } else { + values[i] = mapping[v] + } + } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil +} diff --git a/ingest/vec_test.go b/ingest/vec_test.go new file mode 100644 index 000000000..45a103645 --- /dev/null +++ b/ingest/vec_test.go @@ -0,0 +1,114 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "errors" + "testing" +) + +type badTranslator struct{} + +func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + if len(keys) == 0 { + return nil, errors.New("no keys") + } + m := make(map[string]uint64) + skip := true + for i, k := range keys { + if skip { + skip = false + continue + } + m[k] = uint64(i) + } + out := make([]uint64, len(keys)-1) + for i := range out { + out[i] = uint64(i) + } + return nil, nil +} + +func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) { + return nil, nil +} + +func TestStringTableErrors(t *testing.T) { + tbl := NewStringTable() + btr := badTranslator{} + _, keyErr := tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error passed up from failed translate, didn't get it") + } + a1, err := tbl.ID([]byte("a")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + b1, err := tbl.ID([]byte("b")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + _, keyErr = tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error for short translate, didn't get it") + } + tr := newStableTranslator() + _, err = tr.TranslateKeys("c", "d") + if err != nil { + t.Fatalf("translating stray keys: %v", err) + } + m, err := tbl.MakeIDMap(tr) + if err != nil { + t.Fatalf("creating lookup: %v", err) + } + var y = []uint64{a1, b1} + err = translateUnsigned(m, y) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + trResults, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("unexpected translation error: %v", err) + } + if y[0] != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], y[0]) + } + if y[1] != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], y[1]) + } + y[0] = a1 + y[1] = (a1 + b1 + 1) // assumed not to be any of them + err = translateUnsigned(m, y) + if err == nil { + t.Fatalf("no error from translating invalid table") + } + z := []int64{int64(a1), int64(b1)} + err = translateSigned(m, z) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + if uint64(z[0]) != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], z[0]) + } + if uint64(z[1]) != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], z[1]) + } + z[0] = int64(a1) + z[1] = int64(a1 + b1 + 1) // assumed not to be any of them + err = translateSigned(m, z) + if err == nil { + t.Fatalf("no error from translating invalid table") + } +} diff --git a/ingest_test.go b/ingest_test.go index 649f5dd16..5cd4beac4 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -26,9 +26,7 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v2" - "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/ingest" - "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/test" "github.com/pkg/errors" ) @@ -448,14 +446,7 @@ func TestIngestTestcases(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c := test.MustRunCluster(t, 1, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("node0"), - pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - ) + c := test.MustRunCluster(t, 3) defer c.Close() coord := c.GetPrimary() diff --git a/translate.go b/translate.go index 738fc1686..f449d75be 100644 --- a/translate.go +++ b/translate.go @@ -23,6 +23,7 @@ import ( "sort" "sync" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/topology" "github.com/pkg/errors" ) @@ -98,6 +99,39 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul ReadFrom(io.Reader) (int64, error) } +// This implements ingest's key translator interface, which differs +// slightly because we want to be able to do fast lookups on arbitrary +// IDs which are not necessarily contiguous small values, so the []string +// from TranslateIDs isn't a good fit. +type ingestKeyTranslator struct { + store TranslateStore +} + +var _ ingest.KeyTranslator = &ingestKeyTranslator{} + +func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.store.CreateKeys(keys...) +} + +func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.store.TranslateIDs(ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator { + return &ingestKeyTranslator{store: s} +} + // TranslatorSummary is returned, for example from the boltdb string key translators, // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. type TranslatorSummary struct { From 610c4ed6cbd8b3d4d3cc2a0bcd4704151f8fc1fd Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 10 Sep 2021 12:25:43 -0500 Subject: [PATCH 2/8] introduce protobuf types for ingest ops We add a new protobuf type. Also, protoc changed slightly and remade some tests, in a way which should have no effects but makes the code *very* slightly cleaner. This introduces the first testing code in encoding/proto (whoops) so that scaffolding is a first draft; if you're looking at this code and the design is a problem go ahead and fix it. The purpose of this is to verify that we're actually covering all the branches in the ingest.ShardedRequest and pb.ShardedIngestRequest message conversions. (Except the top-level one for a nil request, which isn't checked by this.) The coverage report doesn't actually include coverage for the ingest code, though, so we haven't actually properly tested Compare. Baby steps! --- encoding/proto/proto.go | 3 + encoding/proto/proto_test.go | 149 +++ pb/private.pb.go | 1852 +++++++++++++++++++++++++++++----- pb/private.proto | 23 +- pb/public.pb.go | 170 +--- 5 files changed, 1813 insertions(+), 384 deletions(-) create mode 100644 encoding/proto/proto_test.go diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index d765a8aff..9a8a262a0 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -978,6 +978,9 @@ func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIn FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), } for k, v := range op.FieldOps { + if v == nil { + continue + } out.FieldOps[k] = &pb.FieldOperation{ RecordIDs: v.RecordIDs, Values: v.Values, diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go new file mode 100644 index 000000000..7ea635881 --- /dev/null +++ b/encoding/proto/proto_test.go @@ -0,0 +1,149 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proto + +import ( + "errors" + "reflect" + "testing" + + "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/ingest" +) + +func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) { + repr, err := s.Marshal(obj) + if err != nil { + if expectedMarshalErr == nil { + t.Fatalf("unexpected marshalling error %q", err.Error()) + } + if err.Error() != expectedMarshalErr.Error() { + t.Fatalf("expecting marshalling error %q, got %q", expectedMarshalErr.Error(), err.Error()) + } + } else { + if expectedMarshalErr != nil { + t.Fatalf("expected marshalling error %q, got no error", expectedMarshalErr.Error()) + } + } + + obj2 := reflect.New(reflect.TypeOf(obj).Elem()).Interface() + err = s.Unmarshal(repr, obj2) + if err != nil { + if expectedUnmarshalErr == nil { + t.Fatalf("unexpected unmarshalling error %q", err.Error()) + } + if err.Error() != expectedUnmarshalErr.Error() { + t.Fatalf("expecting unmarshalling error %q, got %q", expectedUnmarshalErr.Error(), err.Error()) + } + } else { + if expectedUnmarshalErr != nil { + t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error()) + } + } + switch real := obj.(type) { + case *ingest.ShardedRequest: + real2 := obj2.(*ingest.ShardedRequest) + err := real.Compare(real2) + if err != nil { + if expectedMismatchErr == nil { + t.Fatalf("unexpected compare error %q", err.Error()) + } + if err.Error() != expectedMismatchErr.Error() { + t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error()) + } + } else { + if expectedMismatchErr != nil { + t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error()) + } + } + default: + if !reflect.DeepEqual(obj, obj2) { + t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) + } + } +} + +type shardedIngestRequestTest struct { + req *ingest.ShardedRequest + err error +} + +var shardedIngestRequestTestcases = []shardedIngestRequestTest{ + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + ClearFields: []string{"clearField", "clearField2"}, + ClearRecordIDs: []uint64{1, 7, 9}, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + Signed: []int64{-9, 23, 17}, + }, + "writeValues": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + }, + "writeSigned": { + RecordIDs: []uint64{3, 6, 8}, + Signed: []int64{-9, 23, 17}, + }, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + }, + 2: {}, + 3: nil, + }, + }, + }, + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": {}, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + nil, + }, + }, + }, + err: errors.New("shard 1: expected 3 ops, got 2"), + }, +} + +func TestIngestRoundTrip(t *testing.T) { + for _, tc := range shardedIngestRequestTestcases { + t.Logf("next case") + testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err) + } +} diff --git a/pb/private.pb.go b/pb/private.pb.go index 54366c2cb..3a2807420 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -2479,6 +2479,234 @@ func (m *ResizeNodeMessage) GetAction() string { return "" } +type FieldOperation struct { + RecordIDs []uint64 `protobuf:"varint,1,rep,packed,name=RecordIDs,proto3" json:"RecordIDs,omitempty"` + Values []uint64 `protobuf:"varint,2,rep,packed,name=Values,proto3" json:"Values,omitempty"` + Signed []int64 `protobuf:"varint,3,rep,packed,name=Signed,proto3" json:"Signed,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldOperation) Reset() { *m = FieldOperation{} } +func (m *FieldOperation) String() string { return proto.CompactTextString(m) } +func (*FieldOperation) ProtoMessage() {} +func (*FieldOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{39} +} +func (m *FieldOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FieldOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FieldOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FieldOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOperation.Merge(m, src) +} +func (m *FieldOperation) XXX_Size() int { + return m.Size() +} +func (m *FieldOperation) XXX_DiscardUnknown() { + xxx_messageInfo_FieldOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldOperation proto.InternalMessageInfo + +func (m *FieldOperation) GetRecordIDs() []uint64 { + if m != nil { + return m.RecordIDs + } + return nil +} + +func (m *FieldOperation) GetValues() []uint64 { + if m != nil { + return m.Values + } + return nil +} + +func (m *FieldOperation) GetSigned() []int64 { + if m != nil { + return m.Signed + } + return nil +} + +type ShardIngestOperation struct { + OpType string `protobuf:"bytes,1,opt,name=OpType,proto3" json:"OpType,omitempty"` + ClearRecordIDs []uint64 `protobuf:"varint,2,rep,packed,name=ClearRecordIDs,proto3" json:"ClearRecordIDs,omitempty"` + ClearFields []string `protobuf:"bytes,3,rep,name=ClearFields,proto3" json:"ClearFields,omitempty"` + FieldOps map[string]*FieldOperation `protobuf:"bytes,4,rep,name=FieldOps,proto3" json:"FieldOps,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperation) Reset() { *m = ShardIngestOperation{} } +func (m *ShardIngestOperation) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperation) ProtoMessage() {} +func (*ShardIngestOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{40} +} +func (m *ShardIngestOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperation.Merge(m, src) +} +func (m *ShardIngestOperation) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperation) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperation proto.InternalMessageInfo + +func (m *ShardIngestOperation) GetOpType() string { + if m != nil { + return m.OpType + } + return "" +} + +func (m *ShardIngestOperation) GetClearRecordIDs() []uint64 { + if m != nil { + return m.ClearRecordIDs + } + return nil +} + +func (m *ShardIngestOperation) GetClearFields() []string { + if m != nil { + return m.ClearFields + } + return nil +} + +func (m *ShardIngestOperation) GetFieldOps() map[string]*FieldOperation { + if m != nil { + return m.FieldOps + } + return nil +} + +type ShardIngestOperations struct { + Ops []*ShardIngestOperation `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperations) Reset() { *m = ShardIngestOperations{} } +func (m *ShardIngestOperations) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperations) ProtoMessage() {} +func (*ShardIngestOperations) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{41} +} +func (m *ShardIngestOperations) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperations) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperations.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperations) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperations.Merge(m, src) +} +func (m *ShardIngestOperations) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperations) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperations.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperations proto.InternalMessageInfo + +func (m *ShardIngestOperations) GetOps() []*ShardIngestOperation { + if m != nil { + return m.Ops + } + return nil +} + +type ShardedIngestRequest struct { + Ops map[uint64]*ShardIngestOperations `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardedIngestRequest) Reset() { *m = ShardedIngestRequest{} } +func (m *ShardedIngestRequest) String() string { return proto.CompactTextString(m) } +func (*ShardedIngestRequest) ProtoMessage() {} +func (*ShardedIngestRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{42} +} +func (m *ShardedIngestRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardedIngestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardedIngestRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardedIngestRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardedIngestRequest.Merge(m, src) +} +func (m *ShardedIngestRequest) XXX_Size() int { + return m.Size() +} +func (m *ShardedIngestRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ShardedIngestRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardedIngestRequest proto.InternalMessageInfo + +func (m *ShardedIngestRequest) GetOps() map[uint64]*ShardIngestOperations { + if m != nil { + return m.Ops + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "pb.IndexMeta") proto.RegisterType((*FieldOptions)(nil), "pb.FieldOptions") @@ -2520,103 +2748,121 @@ func init() { proto.RegisterType((*TransactionStats)(nil), "pb.TransactionStats") proto.RegisterType((*ResizeAbortMessage)(nil), "pb.ResizeAbortMessage") proto.RegisterType((*ResizeNodeMessage)(nil), "pb.ResizeNodeMessage") + proto.RegisterType((*FieldOperation)(nil), "pb.FieldOperation") + proto.RegisterType((*ShardIngestOperation)(nil), "pb.ShardIngestOperation") + proto.RegisterMapType((map[string]*FieldOperation)(nil), "pb.ShardIngestOperation.FieldOpsEntry") + proto.RegisterType((*ShardIngestOperations)(nil), "pb.ShardIngestOperations") + proto.RegisterType((*ShardedIngestRequest)(nil), "pb.ShardedIngestRequest") + proto.RegisterMapType((map[uint64]*ShardIngestOperations)(nil), "pb.ShardedIngestRequest.OpsEntry") } func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1450 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xf7, 0xc3, 0xb1, 0x7d, 0x1c, 0x27, 0xce, 0x34, 0xea, 0xbb, 0xfd, 0x78, 0x23, 0x77, - 0x5e, 0x44, 0x43, 0x25, 0x22, 0x51, 0x2e, 0x8a, 0xe0, 0xa6, 0x49, 0x9c, 0x16, 0x53, 0xd2, 0x86, - 0x71, 0xda, 0x5b, 0x34, 0x5e, 0x8f, 0x9a, 0x55, 0xd6, 0xbb, 0x66, 0x3f, 0x52, 0xbb, 0x17, 0x48, - 0x20, 0x10, 0xfc, 0x04, 0x7e, 0x06, 0x37, 0xfc, 0x07, 0x6e, 0x90, 0xf8, 0x09, 0xa8, 0xfc, 0x11, - 0x34, 0x67, 0x66, 0x76, 0xd7, 0xae, 0x5b, 0x43, 0xc4, 0xdd, 0x9e, 0xe7, 0xcc, 0x9c, 0xf3, 0x9c, - 0x8f, 0x39, 0x33, 0x0b, 0xed, 0x49, 0x12, 0x5c, 0xf0, 0x4c, 0xec, 0x4d, 0x92, 0x38, 0x8b, 0x89, - 0x3d, 0x19, 0x5e, 0x5f, 0x9f, 0xe4, 0xc3, 0x30, 0xf0, 0x15, 0x42, 0x1f, 0x42, 0xb3, 0x1f, 0x8d, - 0xc4, 0xf4, 0x58, 0x64, 0x9c, 0x10, 0x70, 0x1f, 0x89, 0x59, 0xea, 0x39, 0x5d, 0x6b, 0xb7, 0xc1, - 0xf0, 0x9b, 0xbc, 0x0b, 0x1b, 0xa7, 0x09, 0xf7, 0xcf, 0x8f, 0xa6, 0x41, 0x9a, 0x89, 0xc8, 0x17, - 0x9e, 0x8b, 0xda, 0x05, 0x94, 0xfe, 0xec, 0xc0, 0xfa, 0x83, 0x40, 0x84, 0xa3, 0x27, 0x93, 0x2c, - 0x88, 0xa3, 0x54, 0x1a, 0x3b, 0x9d, 0x4d, 0x84, 0xd7, 0xe8, 0x5a, 0xbb, 0x4d, 0x86, 0xdf, 0xe4, - 0x26, 0x34, 0x0f, 0xb9, 0x7f, 0x26, 0x50, 0xe1, 0xa0, 0xa2, 0x04, 0x0a, 0xed, 0x20, 0x78, 0xa9, - 0xbc, 0xb4, 0x59, 0x09, 0x90, 0x2e, 0xb4, 0x4e, 0x83, 0xb1, 0xf8, 0x22, 0xe7, 0x51, 0x96, 0x8f, - 0xbd, 0x1a, 0xee, 0xae, 0x42, 0xe4, 0x2a, 0xac, 0x3d, 0x09, 0x47, 0xc7, 0x41, 0xe4, 0x35, 0xbb, - 0xd6, 0xae, 0xc3, 0xb4, 0x64, 0x70, 0x3e, 0xf5, 0xa0, 0xc4, 0xf9, 0xb4, 0x08, 0xb7, 0x35, 0x1f, - 0xee, 0xe3, 0x78, 0x90, 0xf1, 0x68, 0xc4, 0x93, 0xd1, 0xb3, 0x40, 0xbc, 0xf0, 0xd6, 0x55, 0xb8, - 0xf3, 0xa8, 0xdc, 0x7b, 0xc0, 0x53, 0xe1, 0xb5, 0xd1, 0x22, 0x7e, 0x93, 0xeb, 0xd0, 0x38, 0x08, - 0xb2, 0x9e, 0x98, 0x64, 0x67, 0xde, 0x46, 0xd7, 0xda, 0x75, 0x59, 0x21, 0x93, 0x6d, 0xa8, 0x0d, - 0x7c, 0x1e, 0x0a, 0x6f, 0x13, 0x37, 0x28, 0x81, 0x50, 0x58, 0x7f, 0x10, 0x27, 0x22, 0x78, 0x1e, - 0x61, 0x11, 0xbc, 0x0e, 0x06, 0x35, 0x87, 0x91, 0xff, 0x81, 0x23, 0x43, 0xda, 0xea, 0x5a, 0xbb, - 0xad, 0xbb, 0xad, 0xbd, 0xc9, 0x70, 0xaf, 0x27, 0xfc, 0x60, 0xcc, 0x43, 0x26, 0x71, 0x54, 0xf3, - 0xa9, 0x47, 0x96, 0xa9, 0xf9, 0x54, 0x72, 0x92, 0x29, 0x7a, 0x1a, 0x05, 0x99, 0x77, 0x05, 0xad, - 0x17, 0x32, 0xa5, 0xb0, 0xd1, 0x1f, 0x4f, 0xe2, 0x24, 0x63, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48, - 0x07, 0x9c, 0xa3, 0x24, 0xf1, 0x2c, 0x5c, 0x28, 0x3f, 0xe9, 0xd7, 0xd0, 0x39, 0x08, 0x63, 0xff, - 0xbc, 0xc7, 0x33, 0xce, 0xc4, 0x57, 0xb9, 0x48, 0x33, 0x19, 0x8b, 0xa2, 0xab, 0xd6, 0x29, 0x41, - 0xa2, 0x58, 0x7f, 0xcf, 0x56, 0x28, 0x0a, 0x32, 0x4f, 0x98, 0x45, 0x55, 0x2e, 0xfc, 0xc6, 0x5c, - 0x9c, 0xf1, 0x64, 0x84, 0x35, 0x76, 0x99, 0x12, 0x24, 0x8a, 0x9e, 0xb0, 0x2f, 0x5c, 0xa6, 0x04, - 0xda, 0x87, 0xad, 0x8a, 0x7f, 0x4d, 0xf3, 0x2a, 0xac, 0xb1, 0xf8, 0x45, 0xbf, 0x97, 0x7a, 0x56, - 0xd7, 0xd9, 0x75, 0x99, 0x96, 0xb0, 0x81, 0xe2, 0x30, 0x1f, 0x47, 0x52, 0x65, 0xa3, 0xaa, 0x04, - 0xe8, 0x35, 0xa8, 0x61, 0x37, 0xc9, 0x28, 0xcb, 0xbd, 0xf2, 0x93, 0x7e, 0x63, 0x41, 0xf3, 0x98, - 0x4f, 0x91, 0x48, 0x4a, 0xee, 0x41, 0xc3, 0xd4, 0x1a, 0x17, 0xb5, 0xee, 0xde, 0x90, 0x79, 0x2d, - 0x16, 0xec, 0x19, 0xed, 0x51, 0x94, 0x25, 0x33, 0x56, 0x2c, 0xbe, 0xfe, 0x09, 0xb4, 0xe7, 0x54, - 0xd2, 0xd3, 0xb9, 0x98, 0x99, 0x7c, 0x9e, 0x8b, 0x99, 0x8c, 0xf2, 0x82, 0x87, 0xb9, 0xc0, 0x2c, - 0xb9, 0x4c, 0x09, 0x1f, 0xdb, 0x1f, 0x59, 0xf4, 0x19, 0x90, 0xc3, 0x44, 0xf0, 0x4c, 0xa0, 0x93, - 0x63, 0x91, 0xa6, 0xfc, 0xb9, 0x58, 0x95, 0x6b, 0xa7, 0x9a, 0xeb, 0x22, 0xaf, 0x76, 0x25, 0xaf, - 0xf4, 0x0e, 0x90, 0x9e, 0x08, 0x45, 0x26, 0xf4, 0x39, 0x7f, 0x8b, 0x5d, 0x7a, 0x6e, 0x38, 0xac, - 0x5e, 0x4b, 0x6e, 0x81, 0x2b, 0x87, 0x06, 0x3a, 0x6b, 0xdd, 0x6d, 0xcb, 0x0c, 0x15, 0x93, 0x84, - 0xa1, 0x0a, 0xeb, 0x81, 0xe6, 0x46, 0xfb, 0x19, 0x52, 0x75, 0x58, 0x09, 0xd0, 0xef, 0x2c, 0xe3, - 0x0d, 0xe9, 0xff, 0xcd, 0x88, 0xe7, 0xba, 0xeb, 0x1d, 0xcd, 0xc1, 0x41, 0x0e, 0x1d, 0xc9, 0xa1, - 0x3a, 0x83, 0x96, 0xd1, 0x70, 0x17, 0x69, 0xdc, 0x37, 0xf9, 0xb9, 0x2c, 0x0b, 0xea, 0xc3, 0x0d, - 0x65, 0x61, 0xff, 0x82, 0x07, 0x21, 0x1f, 0x86, 0xff, 0xa8, 0x84, 0x73, 0x01, 0x79, 0x50, 0xc7, - 0xbd, 0xfd, 0x9e, 0x3e, 0x06, 0x46, 0xa4, 0x39, 0x94, 0x27, 0xea, 0x31, 0x1f, 0x0b, 0x6d, 0x0d, - 0xbf, 0x8b, 0x3c, 0xd8, 0x6f, 0xcd, 0xc3, 0x36, 0xd4, 0xe4, 0xf9, 0x93, 0xf3, 0xdd, 0x91, 0x2e, - 0x51, 0x58, 0x91, 0x9d, 0xf7, 0x61, 0x6d, 0xe0, 0x9f, 0x89, 0x31, 0x27, 0xff, 0x87, 0x3a, 0x32, - 0x17, 0xa9, 0x3e, 0x14, 0xcd, 0xa2, 0xe4, 0xcc, 0x68, 0xe8, 0xf7, 0x96, 0x0e, 0x76, 0x29, 0xcd, - 0x39, 0x57, 0xf6, 0x82, 0x2b, 0x72, 0x1b, 0xea, 0x9a, 0x2f, 0x4e, 0x8b, 0xd7, 0x7a, 0xca, 0x68, - 0xc9, 0x2d, 0x58, 0xc3, 0xe8, 0x52, 0xcf, 0x2d, 0x89, 0x20, 0xc2, 0xb4, 0x82, 0x1e, 0x81, 0xf3, - 0x94, 0xf5, 0xe5, 0xa0, 0x40, 0xf6, 0x86, 0x86, 0x96, 0x24, 0xb9, 0x4f, 0xe3, 0x34, 0xd3, 0xb9, - 0xc7, 0x6f, 0x89, 0x9d, 0xc4, 0x89, 0xea, 0xd3, 0x36, 0xc3, 0x6f, 0xfa, 0xa3, 0x05, 0xee, 0xe3, - 0x78, 0x24, 0xc8, 0x06, 0xd8, 0xfd, 0x9e, 0x36, 0x62, 0xf7, 0x7b, 0xe4, 0x1a, 0xda, 0xd7, 0xf9, - 0xae, 0x4b, 0xff, 0x4f, 0x59, 0x9f, 0xa1, 0xcf, 0x9b, 0xd0, 0xec, 0xa7, 0x27, 0x49, 0x30, 0xe6, - 0xc9, 0x4c, 0xdf, 0xa4, 0x25, 0x80, 0x67, 0x34, 0xe3, 0x99, 0xba, 0xdf, 0x9a, 0x4c, 0x09, 0xe4, - 0x16, 0xd4, 0x1f, 0xb2, 0x93, 0x43, 0x69, 0xb2, 0x36, 0x6f, 0xd2, 0xe0, 0xf4, 0x3e, 0x74, 0x24, - 0x13, 0x5c, 0x6f, 0x3a, 0xeb, 0x2a, 0xac, 0x49, 0xac, 0x60, 0xa6, 0xa5, 0xd2, 0x89, 0x5d, 0x71, - 0x42, 0x1f, 0x28, 0x0b, 0x47, 0x17, 0x22, 0xca, 0x2a, 0xbd, 0x89, 0x32, 0x1a, 0x68, 0x33, 0x25, - 0x90, 0x9b, 0x2a, 0x6a, 0x1d, 0x5e, 0x43, 0x72, 0x91, 0x32, 0x43, 0x94, 0xce, 0x00, 0x0c, 0x93, - 0x3c, 0x2d, 0xd6, 0x5a, 0xcb, 0xd6, 0x12, 0x6a, 0xda, 0x47, 0x1f, 0x51, 0x90, 0x7a, 0x85, 0x30, - 0xd3, 0x58, 0xef, 0x95, 0x8d, 0xa5, 0xea, 0xb9, 0x59, 0xd4, 0x5d, 0xf9, 0x28, 0xdb, 0xeb, 0x0c, - 0x5a, 0x15, 0x7c, 0x69, 0x8f, 0xdd, 0x2e, 0x9a, 0xc3, 0x2e, 0x8d, 0x21, 0xa2, 0x8d, 0x69, 0xf5, - 0x8a, 0xe1, 0x14, 0x40, 0xab, 0xb2, 0x69, 0xa9, 0xa7, 0x5d, 0xd8, 0x9c, 0x3f, 0xf0, 0xe6, 0xce, - 0x59, 0x84, 0x57, 0xb8, 0xfa, 0xc1, 0x82, 0xf6, 0x61, 0x98, 0xa7, 0x99, 0x48, 0x8a, 0x9c, 0x36, - 0x35, 0x50, 0x94, 0xb6, 0x04, 0x96, 0x57, 0x97, 0xec, 0x40, 0x4d, 0x66, 0x5c, 0x1d, 0xee, 0x6a, - 0x21, 0x14, 0x5c, 0xa9, 0x84, 0xfb, 0xa6, 0x4a, 0xd0, 0x67, 0xd0, 0x38, 0x18, 0xf4, 0x1f, 0x26, - 0x71, 0x3e, 0x59, 0x1a, 0xb1, 0x79, 0xd2, 0xd9, 0x95, 0x27, 0x5d, 0x47, 0x3d, 0x4f, 0x54, 0x54, - 0xf8, 0x22, 0xe9, 0xa8, 0x17, 0x89, 0xab, 0x11, 0x3e, 0xa5, 0x03, 0xd8, 0x52, 0xe1, 0xca, 0x89, - 0x73, 0x99, 0xb1, 0x68, 0x5e, 0x11, 0x4e, 0xf9, 0x8a, 0x90, 0x46, 0xd5, 0xd4, 0xfd, 0x37, 0x8d, - 0xfe, 0x66, 0xc3, 0x16, 0x13, 0x69, 0xf0, 0x52, 0xf4, 0xa3, 0x34, 0x4b, 0x72, 0x5f, 0x4e, 0x1c, - 0xb9, 0xff, 0xb3, 0x78, 0xa8, 0x6b, 0xe1, 0x30, 0x25, 0xbc, 0xfd, 0x94, 0x10, 0x0a, 0xf5, 0xea, - 0x10, 0xa8, 0x2e, 0x30, 0x0a, 0x72, 0x07, 0xea, 0x83, 0x38, 0x4f, 0xfc, 0xa2, 0xf3, 0x71, 0x72, - 0x2b, 0xff, 0x4a, 0xc1, 0xcc, 0x02, 0xf2, 0x08, 0xc8, 0x69, 0xc2, 0xa3, 0x34, 0xe4, 0x92, 0x92, - 0xd9, 0xd6, 0x28, 0x9f, 0x27, 0x15, 0xed, 0x9c, 0x85, 0x25, 0xdb, 0xc8, 0x5e, 0xf5, 0x08, 0x7b, - 0x75, 0xe4, 0xb7, 0x61, 0xf8, 0xe9, 0x73, 0x52, 0x3d, 0xe4, 0xf7, 0x16, 0x3a, 0xd4, 0x5b, 0xc3, - 0x2d, 0x5b, 0x72, 0xcb, 0x9c, 0x82, 0xcd, 0xaf, 0xa3, 0xdf, 0x5a, 0xb0, 0x5e, 0x65, 0xb3, 0x62, - 0x5c, 0x14, 0xe5, 0xb3, 0x57, 0xbf, 0x76, 0x4c, 0xf9, 0xdc, 0x65, 0x2f, 0xcb, 0x5a, 0xf5, 0x05, - 0x14, 0xc3, 0x7f, 0xdf, 0x90, 0x9c, 0x4b, 0xd1, 0xe9, 0x42, 0xeb, 0x84, 0x27, 0x59, 0x20, 0x8d, - 0xe9, 0x7b, 0xba, 0xc6, 0xaa, 0x10, 0x15, 0x70, 0xed, 0xb5, 0x26, 0x3a, 0x8c, 0xc7, 0x13, 0xd9, - 0xad, 0x97, 0x6a, 0x26, 0x39, 0xa6, 0x93, 0x24, 0x4e, 0x4c, 0x06, 0x50, 0xa0, 0x07, 0xd0, 0x38, - 0x8d, 0x27, 0x71, 0x18, 0x3f, 0x9f, 0xad, 0x18, 0x19, 0x1e, 0xd4, 0xd5, 0xd5, 0xa0, 0x46, 0x54, - 0x93, 0x19, 0x91, 0x5e, 0x91, 0xfd, 0xee, 0xf3, 0xd0, 0xcf, 0x43, 0x9e, 0x09, 0x7c, 0x1f, 0x23, - 0xf8, 0x79, 0xcc, 0x47, 0x6a, 0x2a, 0xe8, 0xa3, 0x45, 0xbf, 0xd4, 0x0d, 0xc8, 0x31, 0x9c, 0xca, - 0x15, 0xb4, 0x8f, 0x80, 0xb9, 0x82, 0x94, 0x44, 0x3e, 0x80, 0x56, 0x65, 0xb5, 0x0e, 0x6b, 0xb3, - 0xe8, 0x53, 0x05, 0xb3, 0xea, 0x1a, 0xfa, 0x8b, 0x35, 0xb7, 0xe7, 0xb5, 0x3b, 0x57, 0xbb, 0xba, - 0x50, 0x49, 0x6a, 0x30, 0x2d, 0xc9, 0xd0, 0x8f, 0xa6, 0x7e, 0x98, 0xa7, 0x52, 0xa5, 0x2f, 0xdc, - 0x02, 0x90, 0xa1, 0xcb, 0x1f, 0x9e, 0x38, 0x37, 0x8f, 0x1b, 0x23, 0xca, 0x5f, 0xa3, 0x9e, 0xe0, - 0xa3, 0x30, 0x88, 0x04, 0xf6, 0x8b, 0xc3, 0x0a, 0x99, 0xdc, 0x51, 0x33, 0xd6, 0x34, 0xfa, 0xf6, - 0x02, 0x71, 0xd4, 0xa9, 0xc9, 0x9b, 0x52, 0x02, 0x9d, 0x45, 0x15, 0xdd, 0x06, 0xa2, 0x3a, 0x60, - 0x7f, 0x18, 0x27, 0xe6, 0xb6, 0xa5, 0x87, 0x66, 0xb8, 0xc8, 0xec, 0xaf, 0xba, 0xc4, 0xcb, 0xcc, - 0xda, 0xd5, 0xcc, 0x1e, 0x74, 0x7e, 0x7d, 0xb5, 0x63, 0xfd, 0xfe, 0x6a, 0xc7, 0xfa, 0xe3, 0xd5, - 0x8e, 0xf5, 0xd3, 0x9f, 0x3b, 0xff, 0x19, 0xae, 0xe1, 0xaf, 0xfc, 0x87, 0x7f, 0x05, 0x00, 0x00, - 0xff, 0xff, 0x6d, 0xf8, 0xe6, 0x6b, 0xed, 0x0f, 0x00, 0x00, + // 1639 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45, + 0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91, + 0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d, + 0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f, + 0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01, + 0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99, + 0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49, + 0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9, + 0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5, + 0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22, + 0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7, + 0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3, + 0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8, + 0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41, + 0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0, + 0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7, + 0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17, + 0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28, + 0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a, + 0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84, + 0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69, + 0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71, + 0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7, + 0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83, + 0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7, + 0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a, + 0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2, + 0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45, + 0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3, + 0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65, + 0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2, + 0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e, + 0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04, + 0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe, + 0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c, + 0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5, + 0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17, + 0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50, + 0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa, + 0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d, + 0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a, + 0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09, + 0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, + 0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, + 0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, + 0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a, + 0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7, + 0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98, + 0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8, + 0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f, + 0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4, + 0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9, + 0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba, + 0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34, + 0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6, + 0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, + 0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86, + 0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa, + 0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef, + 0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26, + 0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61, + 0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, + 0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45, + 0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0, + 0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba, + 0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23, + 0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50, + 0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e, + 0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3, + 0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30, + 0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23, + 0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8, + 0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc, + 0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b, + 0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61, + 0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54, + 0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15, + 0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11, + 0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28, + 0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0, + 0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9, + 0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54, + 0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d, + 0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa, + 0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31, + 0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b, + 0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3, + 0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46, + 0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11, + 0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89, + 0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30, + 0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1, + 0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a, + 0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76, + 0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72, + 0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3, + 0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50, + 0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc, + 0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf, + 0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79, + 0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, + 0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38, + 0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -4680,6 +4926,267 @@ func (m *ResizeNodeMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FieldOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Signed) > 0 { + dAtA31 := make([]byte, len(m.Signed)*10) + var j30 int + for _, num1 := range m.Signed { + num := uint64(num1) + for num >= 1<<7 { + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j30++ + } + dAtA31[j30] = uint8(num) + j30++ + } + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPrivate(dAtA, i, uint64(j30)) + i-- + dAtA[i] = 0x1a + } + if len(m.Values) > 0 { + dAtA33 := make([]byte, len(m.Values)*10) + var j32 int + for _, num := range m.Values { + for num >= 1<<7 { + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j32++ + } + dAtA33[j32] = uint8(num) + j32++ + } + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPrivate(dAtA, i, uint64(j32)) + i-- + dAtA[i] = 0x12 + } + if len(m.RecordIDs) > 0 { + dAtA35 := make([]byte, len(m.RecordIDs)*10) + var j34 int + for _, num := range m.RecordIDs { + for num >= 1<<7 { + dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j34++ + } + dAtA35[j34] = uint8(num) + j34++ + } + i -= j34 + copy(dAtA[i:], dAtA35[:j34]) + i = encodeVarintPrivate(dAtA, i, uint64(j34)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.FieldOps) > 0 { + for k := range m.FieldOps { + v := m.FieldOps[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x22 + } + } + if len(m.ClearFields) > 0 { + for iNdEx := len(m.ClearFields) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ClearFields[iNdEx]) + copy(dAtA[i:], m.ClearFields[iNdEx]) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClearFields[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.ClearRecordIDs) > 0 { + dAtA38 := make([]byte, len(m.ClearRecordIDs)*10) + var j37 int + for _, num := range m.ClearRecordIDs { + for num >= 1<<7 { + dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j37++ + } + dAtA38[j37] = uint8(num) + j37++ + } + i -= j37 + copy(dAtA[i:], dAtA38[:j37]) + i = encodeVarintPrivate(dAtA, i, uint64(j37)) + i-- + dAtA[i] = 0x12 + } + if len(m.OpType) > 0 { + i -= len(m.OpType) + copy(dAtA[i:], m.OpType) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.OpType))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperations) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperations) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperations) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *ShardedIngestRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardedIngestRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardedIngestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for k := range m.Ops { + v := m.Ops[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i = encodeVarintPrivate(dAtA, i, uint64(k)) + i-- + dAtA[i] = 0x8 + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { offset -= sovPrivate(v) base := offset @@ -5628,6 +6135,124 @@ func (m *ResizeNodeMessage) Size() (n int) { return n } +func (m *FieldOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.RecordIDs) > 0 { + l = 0 + for _, e := range m.RecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Values) > 0 { + l = 0 + for _, e := range m.Values { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Signed) > 0 { + l = 0 + for _, e := range m.Signed { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.OpType) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.ClearRecordIDs) > 0 { + l = 0 + for _, e := range m.ClearRecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.ClearFields) > 0 { + for _, s := range m.ClearFields { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + if len(m.FieldOps) > 0 { + for k, v := range m.FieldOps { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperations) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for _, e := range m.Ops { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardedIngestRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for k, v := range m.Ops { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + sovPrivate(uint64(k)) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPrivate(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -5709,10 +6334,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6149,10 +6771,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6235,10 +6854,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6423,10 +7039,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6629,10 +7242,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6759,10 +7369,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6909,7 +7516,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > postIndex { @@ -6926,10 +7533,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7063,10 +7667,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7149,10 +7750,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7290,10 +7888,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7463,10 +8058,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7581,10 +8173,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7718,10 +8307,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7891,10 +8477,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7979,10 +8562,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8154,10 +8734,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8291,10 +8868,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8501,10 +9075,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8619,10 +9190,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8728,10 +9296,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8888,10 +9453,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9027,10 +9589,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9208,10 +9767,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9396,10 +9952,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9552,10 +10105,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9702,10 +10252,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9852,10 +10399,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10137,10 +10681,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10342,10 +10883,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10483,10 +11021,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10624,10 +11159,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10742,10 +11274,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10796,10 +11325,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10850,10 +11376,7 @@ func (m *LoadSchemaMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10972,10 +11495,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11172,10 +11692,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11226,10 +11743,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11280,10 +11794,7 @@ func (m *ResizeAbortMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11398,10 +11909,857 @@ func (m *ResizeNodeMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FieldOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FieldOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FieldOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.RecordIDs) == 0 { + m.RecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RecordIDs", wireType) + } + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Values) == 0 { + m.Values = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType) + } + case 3: + if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Signed) == 0 { + m.Signed = make([]int64, 0, elementCount) + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Signed", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OpType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.OpType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.ClearRecordIDs) == 0 { + m.ClearRecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ClearRecordIDs", wireType) + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClearFields", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClearFields = append(m.ClearFields, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldOps", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FieldOps == nil { + m.FieldOps = make(map[string]*FieldOperation) + } + var mapkey string + var mapvalue *FieldOperation + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPrivate + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &FieldOperation{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.FieldOps[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperations) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperations: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperations: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ops = append(m.Ops, &ShardIngestOperation{}) + if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardedIngestRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardedIngestRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardedIngestRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Ops == nil { + m.Ops = make(map[uint64]*ShardIngestOperations) + } + var mapkey uint64 + var mapvalue *ShardIngestOperations + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &ShardIngestOperations{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Ops[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { diff --git a/pb/private.proto b/pb/private.proto index a5a43af24..9f15939c7 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -234,4 +234,25 @@ message ResizeAbortMessage { message ResizeNodeMessage { string NodeID = 1; string Action = 2; -} \ No newline at end of file +} + +message FieldOperation { + repeated uint64 RecordIDs = 1; + repeated uint64 Values = 2; + repeated int64 Signed = 3; +} + +message ShardIngestOperation { + string OpType = 1; + repeated uint64 ClearRecordIDs = 2; + repeated string ClearFields = 3; + map FieldOps = 4; +} + +message ShardIngestOperations { + repeated ShardIngestOperation Ops = 1; +} + +message ShardedIngestRequest { + map Ops = 1; +} diff --git a/pb/public.pb.go b/pb/public.pb.go index 43ebfc641..ca1b84852 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -5980,10 +5980,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6068,10 +6065,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6194,10 +6188,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6356,10 +6347,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6486,10 +6474,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6593,10 +6578,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6713,10 +6695,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6799,10 +6778,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7016,10 +6992,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7156,10 +7129,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7274,10 +7244,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7396,10 +7363,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7520,10 +7484,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7642,10 +7603,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7762,10 +7720,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7835,10 +7790,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8008,10 +7960,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8134,10 +8083,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8273,10 +8219,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8365,10 +8308,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8620,10 +8560,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8740,10 +8677,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9356,10 +9290,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9843,10 +9774,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10308,10 +10236,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10481,10 +10406,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10567,10 +10489,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10737,10 +10656,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10867,10 +10783,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11061,10 +10974,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11147,10 +11057,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11267,10 +11174,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11484,10 +11388,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11604,10 +11505,7 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { From b78ce29a3eeeaa8ee60e61c3768db615be501993 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 27 Sep 2021 11:30:34 -0500 Subject: [PATCH 3/8] unexport ShardedRequest.Merge This function absolutely shouldn't be used outside of testing, so I've made the tests using it internal tests and unexported the method. --- ingest/op.go | 2 +- ingest/op_test.go | 65 +++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/ingest/op.go b/ingest/op.go index 754d4e4e2..99d47eeed 100644 --- a/ingest/op.go +++ b/ingest/op.go @@ -703,7 +703,7 @@ func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) // merge combines the components of a sharded request back into a single // unsharded request, processing shards in numerical order. -func (s *ShardedRequest) Merge() *Request { +func (s *ShardedRequest) merge() *Request { req := &Request{} if s == nil || len(s.Ops) == 0 { return req diff --git a/ingest/op_test.go b/ingest/op_test.go index db3b5981e..1646e8993 100644 --- a/ingest/op_test.go +++ b/ingest/op_test.go @@ -12,30 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ingest_test +package ingest import ( "math/rand" "testing" - "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/shardwidth" ) type opShardingTestCase struct { name string - input *ingest.Request - output *ingest.ShardedRequest + input *Request + output *ShardedRequest } var opShardingTestCases = []opShardingTestCase{ { name: "sample", - input: &ingest.Request{ - Ops: []*ingest.Operation{ + input: &Request{ + Ops: []*Operation{ { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ + OpType: OpSet, + FieldOps: map[string]*FieldOperation{ "shard0": { RecordIDs: []uint64{0, 1}, }, @@ -54,9 +53,9 @@ var opShardingTestCases = []opShardingTestCase{ }, }, { - OpType: ingest.OpRemove, + OpType: OpRemove, Seq: 1, - FieldOps: map[string]*ingest.FieldOperation{ + FieldOps: map[string]*FieldOperation{ "shard0-2": { RecordIDs: []uint64{1, 2< Date: Mon, 27 Sep 2021 11:31:23 -0500 Subject: [PATCH 4/8] improve comments --- api.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 95e2199bf..1caa34873 100644 --- a/api.go +++ b/api.go @@ -1826,11 +1826,18 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } -// helper function: do the apply stuff for a known index with known fields +// ingestNodeOperationsForFields does the actual work of applying operations +// to a given index with a map of known fields and an already-parsed +// ShardedRequest. This is used locally on the node that first receives +// the request, after it does the parsing, and on other nodes because the +// format they get is already that rather than JSON, so it's the common +// path *after* key translation and sorting into shards. func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { eg, ctx := errgroup.WithContext(ctx) for shard, ops := range req.Ops { - // loop variable shadow capture is the go equivalent of man door hook hand + // create new local copies of these values so the goroutine uses these + // copies, and doesn't read the actual loop variables, which are being + // changed by the loop. shard, ops := shard, ops eg.Go(func() error { return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) From b41f3554dad170e03780c05c597f410080ca92c1 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 27 Sep 2021 11:34:46 -0500 Subject: [PATCH 5/8] move stableTranslator into test code It was useful having this in the package to verify code coverage of the translator, but that having been verified, I'd sort of rather have it NOT live in the package at all, it's really a testing-only kind of thing. --- ingest/translate.go | 86 ---------------------------------------- ingest/translate_test.go | 67 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 86 deletions(-) delete mode 100644 ingest/translate.go diff --git a/ingest/translate.go b/ingest/translate.go deleted file mode 100644 index 42f61e291..000000000 --- a/ingest/translate.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2021 Molecula Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ingest - -import ( - "fmt" -) - -// stableTranslator implements a key translator that can be reused and -// will continue to give the same keys for the same values. Possibly -// surprisingly, it will invent new keys for IDs it is asked about but -// hasn't seen. This allows us to give a codec which would use keys on -// translation a request which contains arbitrary numbers, and request -// text that would parse into that request. -type stableTranslator struct { - in map[string]uint64 - out map[uint64]string - next uint64 -} - -func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { - ret := make(map[string]uint64, len(keys)) - for _, key := range keys { - if existing, ok := s.in[key]; ok { - ret[key] = existing - continue - } - id := s.next - // but what if someone already translated that ID, so now it already - // exists? - if _, ok := s.out[id]; ok { - for k := range s.out { - if k > id { - id = k - } - } - // one larger than the largest we already have. this could - // wrap around, in which case, it's your own fault. - id++ - } - s.next = id + 1 - s.in[key] = id - s.out[id] = key - ret[key] = id - } - return ret, nil -} - -func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { - ret := make(map[uint64]string, len(ids)) - for _, id := range ids { - if existing, ok := s.out[id]; ok { - ret[id] = existing - continue - } - key := fmt.Sprintf("k-%d", id) - s.in[key] = id - s.out[id] = key - ret[id] = key - if id >= s.next { - s.next = id + 1 - } - } - return ret, nil -} - -// newStableTranslator produces a translator which can translate forwards -// and backwards and invent new things if it needs to. Don't use this. -func newStableTranslator() *stableTranslator { - return &stableTranslator{ - in: make(map[string]uint64), - out: make(map[uint64]string), - } -} diff --git a/ingest/translate_test.go b/ingest/translate_test.go index 78deefda7..67daf73c3 100644 --- a/ingest/translate_test.go +++ b/ingest/translate_test.go @@ -19,6 +19,73 @@ import ( "testing" ) +// stableTranslator implements a key translator that can be reused and +// will continue to give the same keys for the same values. Possibly +// surprisingly, it will invent new keys for IDs it is asked about but +// hasn't seen. This allows us to give a codec which would use keys on +// translation a request which contains arbitrary numbers, and request +// text that would parse into that request. +type stableTranslator struct { + in map[string]uint64 + out map[uint64]string + next uint64 +} + +func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + ret := make(map[string]uint64, len(keys)) + for _, key := range keys { + if existing, ok := s.in[key]; ok { + ret[key] = existing + continue + } + id := s.next + // but what if someone already translated that ID, so now it already + // exists? + if _, ok := s.out[id]; ok { + for k := range s.out { + if k > id { + id = k + } + } + // one larger than the largest we already have. this could + // wrap around, in which case, it's your own fault. + id++ + } + s.next = id + 1 + s.in[key] = id + s.out[id] = key + ret[key] = id + } + return ret, nil +} + +func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + ret := make(map[uint64]string, len(ids)) + for _, id := range ids { + if existing, ok := s.out[id]; ok { + ret[id] = existing + continue + } + key := fmt.Sprintf("k-%d", id) + s.in[key] = id + s.out[id] = key + ret[id] = key + if id >= s.next { + s.next = id + 1 + } + } + return ret, nil +} + +// newStableTranslator produces a translator which can translate forwards +// and backwards and invent new things if it needs to. Don't use this. +func newStableTranslator() *stableTranslator { + return &stableTranslator{ + in: make(map[string]uint64), + out: make(map[uint64]string), + } +} + func TestTranslateReuse(t *testing.T) { // the original stable-translator design had a flaw in that it // assumed that each new ID would always come from a string translation, From b3f82ac894dfaf535225cc8c2b8afa1b793fb334 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 27 Sep 2021 11:39:19 -0500 Subject: [PATCH 6/8] return early on error instead of writing success status also --- http/handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/handler.go b/http/handler.go index d2eef2e5e..05e97f583 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1360,6 +1360,7 @@ func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { err = qcx.Finish() if err != nil { http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + return } } else { qcx.Abort() From 12882ad14743ff3f377e47ae52d050f45dbaa1fb Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 27 Sep 2021 11:54:02 -0500 Subject: [PATCH 7/8] handle replication I assumed the existing import code handled replicas. It doesn't, actually. It just assumes they're handled. So, in the new import code, when splitting things up by-shard, send each shard's data to *every* node that has that shard, not just the first one. --- api.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/api.go b/api.go index 1caa34873..ca3506a0e 100644 --- a/api.go +++ b/api.go @@ -1942,16 +1942,25 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string if len(snap.Nodes) == 1 { return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) } - // split up by fields in some way + // Created new ShardedRequest objects for every node, giving each of them + // all the shards that apply to them. byNode := make(map[string]*ingest.ShardedRequest) for shard, ops := range sharded.Ops { nodes := snap.ShardNodes(indexName, shard) - forThisShard := byNode[nodes[0].ID] - if forThisShard == nil { - byNode[nodes[0].ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} - continue + for _, node := range nodes { + forThisShard := byNode[node.ID] + if forThisShard == nil { + // Create new ShardedRequest for the target node, with its op map + // mapping this shard to the ops for this shard. + byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} + continue + } + // Add this shard to the existing ShardedRequest's Ops map. Note that + // we don't have to worry about overwrites; we can't have seen this + // shard before, because we're in a range loop on a map where the shard + // is the key. + forThisShard.Ops[shard] = ops } - forThisShard.Ops[shard] = ops } eg, ctx := errgroup.WithContext(ctx) for _, node := range snap.Nodes { From 02d3d24bc52ca3e1de591977613a37684c6fd7d4 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 27 Sep 2021 11:58:30 -0500 Subject: [PATCH 8/8] code review cleanup --- cluster.go | 3 +-- encoding/proto/proto.go | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index 29c4c2164..2843b732f 100644 --- a/cluster.go +++ b/cluster.go @@ -1567,8 +1567,7 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } -// This implements ingest's key translator interface on a cluster/index -// pair. +// This implements ingest's key translator interface on a cluster/index pair. type clusterKeyTranslator struct { ctx context.Context // we're created within a request context and need to pass that to cluster ops c *cluster diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 9a8a262a0..c5672e563 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -958,9 +958,6 @@ func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.S func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { out := &pb.ShardIngestOperations{} - if len(ops) == 0 { - return out - } for _, op := range ops { if op == nil { continue