diff --git a/api.go b/api.go index ce8ffda33..ca3506a0e 100644 --- a/api.go +++ b/api.go @@ -1826,6 +1826,44 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } +// ingestNodeOperationsForFields does the actual work of applying operations +// to a given index with a map of known fields and an already-parsed +// ShardedRequest. This is used locally on the node that first receives +// the request, after it does the parsing, and on other nodes because the +// format they get is already that rather than JSON, so it's the common +// path *after* key translation and sorting into shards. +func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { + eg, ctx := errgroup.WithContext(ctx) + for shard, ops := range req.Ops { + // create new local copies of these values so the goroutine uses these + // copies, and doesn't read the actual loop variables, which are being + // changed by the loop. + shard, ops := shard, ops + eg.Go(func() error { + return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) + }) + } + return eg.Wait() +} + +// IngestNodeOperations handles protobuf-formatted data which does not need +// key translation and is applicable to this specific node. +func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { + index := api.holder.Index(indexName) + if index == nil { + api.server.logger.Errorf("ingest: no such index %q", indexName) + return newNotFoundError(ErrIndexNotFound, indexName) + } + fields := index.Fields() + knownFields := map[string]*Field{} + for _, field := range fields { + knownFields[field.name] = field + } + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) +} + +// IngestOperations handles JSON-formatted data which may need key translation +// and may be for any or all nodes. func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") defer span.Finish() @@ -1841,34 +1879,32 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return newNotFoundError(ErrIndexNotFound, indexName) } fields := index.Fields() - var lookup ingest.KeyLookupFunc + var indexKeys ingest.KeyTranslator if index.Keys() { - lookup = func(keys ...string) (map[string]uint64, error) { - return api.cluster.createIndexKeys(ctx, indexName, keys...) - } + indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) } - codec, err := ingest.NewJSONCodec(lookup) + codec, err := ingest.NewJSONCodec(indexKeys) if err != nil { return errors.Wrap(err, "creating JSON codec") } knownFields := map[string]*Field{} for _, field := range fields { - var lookup ingest.KeyLookupFunc + var keys ingest.KeyTranslator if field.usesKeys { - lookup = field.translateStore.CreateKeys + keys = newIngestKeyTranslatorFromStore(field.translateStore) } knownFields[field.name] = field switch field.Type() { case "set": - if err = codec.AddSetField(field.name, lookup); err != nil { + if err = codec.AddSetField(field.name, keys); err != nil { return fmt.Errorf("adding set field to codec: %w", err) } case "time": - if err = codec.AddTimeQuantumField(field.name, lookup); err != nil { + if err = codec.AddTimeQuantumField(field.name, keys); err != nil { return fmt.Errorf("adding time quantum field to codec: %w", err) } case "mutex": - if err = codec.AddMutexField(field.name, lookup); err != nil { + if err = codec.AddMutexField(field.name, keys); err != nil { return fmt.Errorf("adding mutex field to codec: %w", err) } case "bool": @@ -1876,7 +1912,7 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return fmt.Errorf("adding bool field to codec: %w", err) } case "int": - if err = codec.AddIntField(field.name, lookup); err != nil { + if err = codec.AddIntField(field.name, keys); err != nil { return fmt.Errorf("adding int field to codec: %w", err) } case "decimal": @@ -1896,17 +1932,53 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string if err != nil { return errors.Wrap(err, "parsing input data") } - sharded, err := req.ByShard() + sharded, err := codec.RequestByShard(req) if err != nil { return errors.Wrap(err, "sharding input data") } - eg, ctx := errgroup.WithContext(ctx) + // now that we have this, let's assign the shards to nodes + snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) + // oh hey an easy case: we're presumably the only node + if len(snap.Nodes) == 1 { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + } + // Created new ShardedRequest objects for every node, giving each of them + // all the shards that apply to them. + byNode := make(map[string]*ingest.ShardedRequest) for shard, ops := range sharded.Ops { - // loop variable shadow capture is the go equivalent of man door hook hand - shard, ops := shard, ops - eg.Go(func() error { - return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) - }) + nodes := snap.ShardNodes(indexName, shard) + for _, node := range nodes { + forThisShard := byNode[node.ID] + if forThisShard == nil { + // Create new ShardedRequest for the target node, with its op map + // mapping this shard to the ops for this shard. + byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} + continue + } + // Add this shard to the existing ShardedRequest's Ops map. Note that + // we don't have to worry about overwrites; we can't have seen this + // shard before, because we're in a range loop on a map where the shard + // is the key. + forThisShard.Ops[shard] = ops + } + } + eg, ctx := errgroup.WithContext(ctx) + for _, node := range snap.Nodes { + node := node + sharded := byNode[node.ID] + // Sometimes, there's nothing for a specific node. + if sharded == nil { + continue + } + if node.ID == api.NodeID() { + eg.Go(func() error { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + }) + } else { + eg.Go(func() error { + return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) + }) + } } return eg.Wait() } @@ -3090,6 +3162,7 @@ const ( apiIDReset apiPartitionNodes apiIngestOperations + apiIngestNodeOperations apiMutexCheck ) @@ -3159,5 +3232,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIDReset: {}, apiPartitionNodes: {}, apiIngestOperations: {}, + apiIngestNodeOperations: {}, apiMutexCheck: {}, } diff --git a/client.go b/client.go index 62f16e240..8726c440a 100644 --- a/client.go +++ b/client.go @@ -19,6 +19,7 @@ import ( "io" "time" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" ) @@ -81,6 +82,7 @@ type InternalClient interface { ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) + IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) @@ -218,6 +220,10 @@ func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, return nil, nil } +func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + return nil +} + func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index 6419f3681..2843b732f 100644 --- a/cluster.go +++ b/cluster.go @@ -24,6 +24,7 @@ import ( "time" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/topology" @@ -1566,6 +1567,38 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } +// This implements ingest's key translator interface on a cluster/index pair. +type clusterKeyTranslator struct { + ctx context.Context // we're created within a request context and need to pass that to cluster ops + c *cluster + indexName string +} + +var _ ingest.KeyTranslator = &clusterKeyTranslator{} + +func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.c.createIndexKeys(i.ctx, i.indexName, keys...) +} + +func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator { + return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName} +} + // TODO: remove this when it is no longer used func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keys := make([]string, 0, len(keySet)) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 69a456904..c5672e563 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -21,6 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" @@ -324,7 +325,18 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeResizeAbortMessage(msg, mt) return nil - + case *ingest.ShardedRequest: + msg := &pb.ShardedIngestRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshalling ShardedRequest") + } + req, err := s.decodeShardedIngestRequest(msg) + if err != nil { + return err + } + *mt = *req + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -398,6 +410,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeResizeNodeMessage(mt) case *pilosa.ResizeAbortMessage: return s.encodeResizeAbortMessage(mt) + case *ingest.ShardedRequest: + return s.encodeShardedIngestRequest(mt) } return nil } @@ -931,6 +945,48 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr return &pb.TransactionStats{} } +func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest { + if req == nil || len(req.Ops) == 0 { + return &pb.ShardedIngestRequest{} + } + out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))} + for shard, ops := range req.Ops { + out.Ops[shard] = s.encodeShardIngestOperations(ops) + } + return out +} + +func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { + out := &pb.ShardIngestOperations{} + for _, op := range ops { + if op == nil { + continue + } + out.Ops = append(out.Ops, s.encodeShardIngestOperation(op)) + } + return out +} + +func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation { + out := &pb.ShardIngestOperation{ + OpType: op.OpType.String(), + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + if v == nil { + continue + } + out.FieldOps[k] = &pb.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out +} + func (s Serializer) decodeResizeInstruction(ri *pb.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID m.Node = &topology.Node{} @@ -1829,3 +1885,57 @@ func decodeResizeNodeMessage(pb *pb.ResizeNodeMessage, m *pilosa.ResizeNodeMessa func decodeResizeAbortMessage(pb *pb.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) { } + +func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) { + if req == nil || len(req.Ops) == 0 { + return &ingest.ShardedRequest{}, nil + } + out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))} + for shard, ops := range req.Ops { + var err error + out.Ops[shard], err = s.decodeShardIngestOperations(ops) + if err != nil { + return nil, err + } + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) { + out := []*ingest.Operation{} + if len(ops.Ops) == 0 { + return out, nil + } + for _, op := range ops.Ops { + if op == nil { + continue + } + decoded, err := s.decodeShardIngestOperation(op) + if err != nil { + return nil, err + } + out = append(out, decoded) + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) { + opType, err := ingest.ParseOpType(op.OpType) + if err != nil { + return nil, err + } + out := &ingest.Operation{ + OpType: opType, + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + out.FieldOps[k] = &ingest.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out, nil +} diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go new file mode 100644 index 000000000..7ea635881 --- /dev/null +++ b/encoding/proto/proto_test.go @@ -0,0 +1,149 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proto + +import ( + "errors" + "reflect" + "testing" + + "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/ingest" +) + +func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) { + repr, err := s.Marshal(obj) + if err != nil { + if expectedMarshalErr == nil { + t.Fatalf("unexpected marshalling error %q", err.Error()) + } + if err.Error() != expectedMarshalErr.Error() { + t.Fatalf("expecting marshalling error %q, got %q", expectedMarshalErr.Error(), err.Error()) + } + } else { + if expectedMarshalErr != nil { + t.Fatalf("expected marshalling error %q, got no error", expectedMarshalErr.Error()) + } + } + + obj2 := reflect.New(reflect.TypeOf(obj).Elem()).Interface() + err = s.Unmarshal(repr, obj2) + if err != nil { + if expectedUnmarshalErr == nil { + t.Fatalf("unexpected unmarshalling error %q", err.Error()) + } + if err.Error() != expectedUnmarshalErr.Error() { + t.Fatalf("expecting unmarshalling error %q, got %q", expectedUnmarshalErr.Error(), err.Error()) + } + } else { + if expectedUnmarshalErr != nil { + t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error()) + } + } + switch real := obj.(type) { + case *ingest.ShardedRequest: + real2 := obj2.(*ingest.ShardedRequest) + err := real.Compare(real2) + if err != nil { + if expectedMismatchErr == nil { + t.Fatalf("unexpected compare error %q", err.Error()) + } + if err.Error() != expectedMismatchErr.Error() { + t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error()) + } + } else { + if expectedMismatchErr != nil { + t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error()) + } + } + default: + if !reflect.DeepEqual(obj, obj2) { + t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) + } + } +} + +type shardedIngestRequestTest struct { + req *ingest.ShardedRequest + err error +} + +var shardedIngestRequestTestcases = []shardedIngestRequestTest{ + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + ClearFields: []string{"clearField", "clearField2"}, + ClearRecordIDs: []uint64{1, 7, 9}, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + Signed: []int64{-9, 23, 17}, + }, + "writeValues": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + }, + "writeSigned": { + RecordIDs: []uint64{3, 6, 8}, + Signed: []int64{-9, 23, 17}, + }, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + }, + 2: {}, + 3: nil, + }, + }, + }, + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": {}, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + nil, + }, + }, + }, + err: errors.New("shard 1: expected 3 ops, got 2"), + }, +} + +func TestIngestRoundTrip(t *testing.T) { + for _, tc := range shardedIngestRequestTestcases { + t.Logf("next case") + testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err) + } +} diff --git a/http/client.go b/http/client.go index fb079218f..1d41a114f 100644 --- a/http/client.go +++ b/http/client.go @@ -32,6 +32,7 @@ import ( pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -284,6 +285,38 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in return nil } +// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data +func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + if uri == nil { + uri = c.defaultURI + } + u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName)) + + buf, err := c.serializer.Marshal(ireq) + if err != nil { + return errors.Wrap(err, "marshalling") + } + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("unexpected status code: %s", resp.Status) + } + return nil +} + // MutexCheck uses the mutex-check endpoint to request mutex collision data // from a single node. It produces per-shard results, and does not translate // them. diff --git a/http/handler.go b/http/handler.go index ac5a4dd68..05e97f583 100644 --- a/http/handler.go +++ b/http/handler.go @@ -42,6 +42,7 @@ import ( "github.com/gorilla/mux" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/rbf" @@ -432,7 +433,9 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/ingest/{index}", handler.handleIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode") + router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema") router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys") router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys") @@ -1337,7 +1340,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { +// handlePostIngestData handles JSON ingest data that may need key +// translation, for the entire cluster. +func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return @@ -1351,6 +1356,15 @@ func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { qcx := h.api.Txf().NewQcx() err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + return + } + } else { + qcx.Abort() + } resp := successResponse{h: h, Name: indexName} resp.write(w, err) @@ -2908,6 +2922,52 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } } +// handlePostIngestNode is the internal endpoint taking already-translated +// ingest operations, sorted by shard, for a single node. +func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + ctx := r.Context() + + // Read entire body. + span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + body, err := readBody(r) + span.LogKV("bodySize", len(body)) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + req := &ingest.ShardedRequest{} + span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") + err = proto.DefaultSerializer.Unmarshal(body, req) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + urlVars := mux.Vars(r) + indexName := urlVars["index"] + + qcx := h.api.Txf().NewQcx() + err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + } + } else { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + qcx.Abort() + } +} + func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { diff --git a/ingest/codec.go b/ingest/codec.go index c966e3f1c..91b63000f 100644 --- a/ingest/codec.go +++ b/ingest/codec.go @@ -15,10 +15,13 @@ package ingest import ( + "bytes" + "encoding/json" "fmt" "io" "io/ioutil" "math" + "sort" "strconv" "time" @@ -37,15 +40,22 @@ import ( // decimal yes yes no // timestamp yes yes no -type KeyLookupFunc func(...string) (map[string]uint64, error) +// KeyTranslator is a thing that can translate strings to IDs, and also +// IDs back to strings. The ID->string conversion is used only to render +// a request back to JSON, which in turn is only used in testing. The +// functions optionally take an existing map which they then augment. +type KeyTranslator interface { + TranslateKeys(keys ...string) (map[string]uint64, error) + TranslateIDs(ids ...uint64) (map[uint64]string, error) +} // Codec is a single-use parser which decodes data into columnar vectors. type Codec interface { - AddSetField(name string, lookup KeyLookupFunc) error - AddTimeQuantumField(name string, lookup KeyLookupFunc) error - AddMutexField(name string, lookup KeyLookupFunc) error + AddSetField(name string, keys KeyTranslator) error + AddTimeQuantumField(name string, keys KeyTranslator) error + AddMutexField(name string, keys KeyTranslator) error AddBoolField(name string) error - AddIntField(name string, lookup KeyLookupFunc) error + AddIntField(name string, keys KeyTranslator) error AddDecimalField(name string, scale int64) error AddTimestampField(name string, scale time.Duration, epoch int64) error @@ -56,138 +66,299 @@ type Codec interface { type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error -// applyTranslationFn is a function which applies key lookups to the values -// of an operation, meaning it needs to know whether it's applying them -// to the signed or unsigned values. -type applyTranslationFn func(*FieldOperation, []uint64) error +// jsonEncFn encodes a value, or range of values, according to a given +// fieldCodec's translation rules. key values, if present, will be used to +// replace whichever values could have been provided as keys. so for instance, +// with an int field which is keyed, values are actually of type int64, but +// if strings are present, those are used. for a time quantum field, the time +// is set from the signed field, and the value is replaced by the key if +// keys are provided. +type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error -type jsonFieldCodec struct { +// fieldCodec represents something that can encode and decode a +// particular field. Its methods are not reentrant, it uses internal +// buffers. +type fieldCodec struct { valueKeys *StringTable currentOp *FieldOperation decode jsonDecFn - translate applyTranslationFn + encode jsonEncFn // For timestamp: scale-in-nanoseconds; for instance, if scaleUnit is // 1,000,000,000, we are storing numbers-of-seconds since the Unix epoch. // The actual value recorded in BSI will be offset by the field's // epoch, but we don't need to know that. // For decimal: Decimal digits of precision. So for instance, with - // scaleUnit 2, "1" is stored as 100 and "1.2" is stored as 120. + // scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as + // 120. scaleUnit int64 + scale int64 epoch int64 // used only by Timestamp fields scratch []uint64 // reusable scratch space for sets of values - lookup KeyLookupFunc + keys KeyTranslator + buf []byte // scratch space for format operations. fieldType FieldType } // JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps. type JSONCodec struct { - recKeys *StringTable - fields map[string]*jsonFieldCodec - keyLookup KeyLookupFunc - currentOp *Operation + fieldTypes map[string]FieldType + recKeys *StringTable + fields map[string]*fieldCodec + keys KeyTranslator + currentOp *Operation +} + +// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which +// can be used to write to its buffer. Both types are just embedded because +// they have non-overlapping APIs. Please don't look at my horrible face. +// +// This has some internal objects it can use to stash things +// so that it can pass &foo to enc.Encode() without needing to heap-allocate +// something for the interface conversion, and a buffer it can use for +// converting numbers or times. +// +// It also has an internal error buffer. In fact, in many cases, errors +// are so far as I can tell absolutely impossible -- bytes.Buffer specifically +// promises never to yield an error, and nothing seems to hint that +// enc.Encode can error on integers, strings, booleans, or arrays. So we +// have dozens of error checks that we can't cause to happen even with +// malformed inputs... so we eat those errors, don't require them to be +// checked externally, and return them when done if anyone checks. +type jsonBuffer struct { + *bytes.Buffer + enc *json.Encoder + buf [48]byte // scratch space for using strconv.AppendInt, etc + uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64 + strbuf []string // dummy buffer so that we don't have to alloc to print []string + str string // and again, "so we don't have to alloc a copy" + err error +} + +// Encode removes the stray newlines added by json.Encoder. +func (j *jsonBuffer) Encode(v interface{}) { + err := j.enc.Encode(v) + if err == nil { + j.Truncate(j.Len() - 1) + } else { + j.err = err + } +} + +// EncodeInt uses AppendInt into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeInt(i int64) { + rep := strconv.AppendInt(j.buf[:0], i, 10) + _, _ = j.Write(rep) +} + +// EncodeUint uses AppendUint into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeUint(u uint64) { + rep := strconv.AppendUint(j.buf[:0], u, 10) + _, _ = j.Write(rep) +} + +// EncodeUints uses a static copy of a []uint64 -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeUints(u []uint64) { + j.uintbuf = u + j.Encode(&j.uintbuf) + j.strbuf = nil +} + +// EncodeStrings uses a static copy of a []string -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeStrings(s []string) { + j.strbuf = s + j.Encode(&j.strbuf) + j.strbuf = nil +} + +// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs, +// while also surrounding the value with quotes. +func (j *jsonBuffer) EncodeQuotedUint(u uint64) { + j.buf[0] = '"' + rep := strconv.AppendUint(j.buf[1:1], u, 10) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeString appends the JSON encoding of a string. This exists +// because otherwise runtime allocates a heap-allocated copy of the +// string to live inside an interface{} for the duration of a function +// call... +func (j *jsonBuffer) EncodeString(s string) { + j.str = s + j.Encode(&j.str) + j.str = "" +} + +// EncodeTime exists because time's MarshalJSON allocates a new +// buffer every time it gets called, resulting in a full 13% of +// all the allocations produced in a test run, plus another 13% +// or so of them which were for the copies of the time objects +// made to stuff them into an interface{}. Eww. +func (j *jsonBuffer) EncodeTime(t time.Time) { + j.buf[0] = '"' + rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeBool writes a literal representation directly to reduce allocs. +func (j *jsonBuffer) EncodeBool(b bool) { + if b { + _, _ = j.WriteString("true") + } else { + _, _ = j.WriteString("false") + } +} + +func (j *jsonBuffer) Err() error { + return j.err +} + +func newJSONBuffer(data []byte) *jsonBuffer { + e := &jsonBuffer{Buffer: bytes.NewBuffer(data)} + e.enc = json.NewEncoder(e.Buffer) + e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON + return e } var _ Codec = &JSONCodec{} -func NewJSONCodec(lookup KeyLookupFunc) (*JSONCodec, error) { - j := &JSONCodec{fields: map[string]*jsonFieldCodec{}} - if lookup != nil { +func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) { + j := &JSONCodec{ + fields: map[string]*fieldCodec{}, + fieldTypes: map[string]FieldType{}, + } + if keys != nil { j.recKeys = NewStringTable() - j.keyLookup = lookup + j.keys = keys } return j, nil } -func (codec *JSONCodec) AddTimeQuantumField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeQuantum, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeTimeQuantum, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddSetField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeSet, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeSetValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeSet, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeSetValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddIntField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeInt, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeIntValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateSigned - } - return codec.addField(name, FieldTypeInt, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeIntValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddMutexField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} - fieldCodec.decode = fieldCodec.DecodeMutexValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned +func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeMutex, + keys: keys, } - return codec.addField(name, FieldTypeMutex, fieldCodec, lookup) + fieldCodec.decode = fieldCodec.DecodeMutexValue + fieldCodec.encode = fieldCodec.EncodeMutexValue + return codec.addField(name, fieldCodec) } func (codec *JSONCodec) AddBoolField(name string) error { - fieldCodec := &jsonFieldCodec{} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeBool, + } fieldCodec.decode = fieldCodec.DecodeBoolValue - return codec.addField(name, FieldTypeBool, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeBoolValue + return codec.addField(name, fieldCodec) } // TimestampField is used to store seconds since unix epoch. The numeric values // stored are adjusted based on the given time.Duration; for instance, if the // time scale is time.Second, then the second after the epoch is stored as 1, -// if it's time.Millisecond, then it's stored as 1000, etcetera. +// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch +// passed to this function should be the offset from the Unix epoch to the +// desired epoch, in the same scale. (So if the scale is milliseconds, +// it should be the Unix timestamp in seconds, times 1000.) func (codec *JSONCodec) AddTimestampField(name string, timeScale time.Duration, epoch int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(timeScale), epoch: epoch} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeStamp, + scaleUnit: int64(timeScale), + epoch: epoch, + } fieldCodec.decode = fieldCodec.DecodeTimeValue - return codec.addField(name, FieldTypeTimeStamp, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeTimeValue + return codec.addField(name, fieldCodec) } // AddDecimalField adds a decimal field, which is stored as integer values // with a scale offset, but parsed as floating point values. For instance, // with decimalScale=2, `0.01` would store the value 1. func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(math.Pow10(int(decimalScale)))} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeDecimal, + scale: decimalScale, + scaleUnit: int64(math.Pow(10, float64(decimalScale))), + } fieldCodec.decode = fieldCodec.DecodeDecimalValue - return codec.addField(name, FieldTypeDecimal, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeDecimalValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) addField(name string, fieldType FieldType, fieldCodec *jsonFieldCodec, lookup KeyLookupFunc) error { +func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error { if _, ok := codec.fields[name]; ok { return fmt.Errorf("duplicate field %q", name) } - if lookup != nil { + if fieldCodec.keys != nil { fieldCodec.valueKeys = NewStringTable() - fieldCodec.lookup = lookup } - fieldCodec.fieldType = fieldType codec.fields[name] = fieldCodec + codec.fieldTypes[name] = fieldCodec.fieldType return nil } // decodeSetOrValue decodes a value which might be either an array of values or a // single value, where values might be string keys or bare numbers, calling cb for // each value it finds. -func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { +func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { switch dataType { case jsonparser.Array: _, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - id, idErr := j.valueKeys.ID(value) - if idErr != nil { - err = idErr - return - } - // stash an error if we got one - valueErr := cb(id) - if valueErr != nil { - err = valueErr + var id uint64 + switch dataType { + case jsonparser.String: + id, err = j.valueKeys.ID(value) + if err != nil { + return + } + err = cb(id) + case jsonparser.Number: + if j.valueKeys != nil { + err = errors.New("expecting key, got numeric value") + return + } + id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64) + if err != nil { + return + } + err = cb(id) + default: + err = fmt.Errorf("expecting value or array, got %v", dataType) } }) if err != nil { @@ -212,23 +383,32 @@ func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data [] } return cb(id) default: - return fmt.Errorf("expecting array, got %v", dataType) + return fmt.Errorf("expecting value or array, got %v", dataType) } return err } // DecodeSetValue decodes a set of unsigned values from the provided data into the // associated currentOp. -func (j *jsonFieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { return j.decodeSetOrValue(dataType, data, func(id uint64) error { j.currentOp.AddPair(recID, id) return nil }) } +func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 1 || len(values) == 1 { + // simplify: just hand this off to a single-value case + return j.EncodeMutexValue(dst, values, signed, keys) + } + appendKeysJSON(dst, values, keys) + return nil +} + // DecodeIntValue decodes a single signed value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String: value, err := j.valueKeys.IntID(data) @@ -255,25 +435,62 @@ func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueT return nil } +func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(signed) == 0 { + return errors.New("encodeIntValue: need a value") + } + dst.EncodeInt(signed[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeMutexValue decodes a single unsigned value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { - case jsonparser.Number, jsonparser.String: - id, err := j.valueKeys.ID(data) + case jsonparser.String: + value, err := j.valueKeys.ID(data) if err != nil { return err } - j.currentOp.AddPair(recID, id) + j.currentOp.AddPair(recID, value) + case jsonparser.Number: + if j.valueKeys != nil { + return errors.New("expecting string key, got numeric value") + } + value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) + if err != nil { + return err + } + j.currentOp.AddPair(recID, value) default: - return fmt.Errorf("expecting integer value, got %v", dataType) + if j.valueKeys != nil { + return fmt.Errorf("expecting string key, got %v", dataType) + } else { + return fmt.Errorf("expecting integer value, got %v", dataType) + } } return nil } +func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(values) == 0 { + return errors.New("encodeMutexValue: need a value") + } + dst.EncodeUint(values[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeBoolValue decodes a single true/false value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { value := uint64(0) switch typ { case jsonparser.String: @@ -303,9 +520,21 @@ func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, return nil } +func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding boolean value, but none provided") + } + var x bool + if values[0] != 0 { + x = true + } + dst.EncodeBool(x) + return nil +} + // DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the // provided data into the associated currentOp. -func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { j.scratch = j.scratch[:0] stamp := time.Unix(0, 0).UTC() err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { @@ -346,8 +575,27 @@ func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.Val return nil } +// Our format does not allow a record to have multiple values set with +// different timestamps at the same time. We use the first timestamp for +// the whole set. +func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding time quantum value: no value provided") + } + dst.WriteString(`{"values":`) + appendKeysJSON(dst, values, keys) + // a zero timestamp is idiomatic for no-time-provided, and i sort of + // hate that, but here we are. + if len(signed) > 0 && signed[0] != 0 { + _, _ = dst.WriteString(`,"time":`) + dst.EncodeTime(time.Unix(0, signed[0]).UTC()) + } + _, _ = dst.WriteString(`}`) + return nil +} + // DecodeTimeValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { var stamp time.Time switch dataType { case jsonparser.String: @@ -364,12 +612,22 @@ func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.Value return fmt.Errorf("parsing numeric timestamp: %w", err) } j.currentOp.AddSignedPair(recID, i64) + default: + return fmt.Errorf("expecting time, got %s", dataType) } return nil } +func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(signed) == 0 { + return errors.New("encoding time value: no value provided") + } + dst.EncodeTime(time.Unix(0, (signed[0]+j.epoch)*j.scaleUnit).UTC()) + return nil +} + // DecodeDecimalValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String, jsonparser.Number: value, err := jsonparser.GetFloat(data) @@ -384,6 +642,28 @@ func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.Va return nil } +func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10) + scale := int(j.scale) + if len(j.buf) > scale { + j.buf = append(j.buf, '.') + // shove the last scaleUnit values over + copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:]) + j.buf[len(j.buf)-scale-1] = '.' + } + _, _ = dst.Write(j.buf) + return nil +} + +// FieldTypes gives a mapping of fields to their basic types used by this +// codec. +func (codec *JSONCodec) FieldTypes() map[string]FieldType { + return codec.fieldTypes +} + +// ParseKeyedRecords parses the records it finds. Note that, when you're doing +// a Write op, this will update ClearRecordIDs automatically as it goes, +// even though record_ids isn't specified in the JSON for that case. func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { seen := make(map[uint64]struct{}) return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { @@ -412,8 +692,8 @@ func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { }) } -func (codec *JSONCodec) ParseOperation(data []byte) (op *Operation, err error) { - op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields))} +func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) { + op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq} codec.currentOp = op err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { switch string(key) { @@ -491,10 +771,12 @@ func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { var ops []*Operation var lastErr error + var seq int _, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { switch dataType { case jsonparser.Object: - op, err := codec.ParseOperation(value) + op, err := codec.ParseOperation(value, seq) + seq++ if err != nil { lastErr = fmt.Errorf("parsing operation: %v", err) return @@ -512,34 +794,39 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { } // and now, key translation! var keyMap []uint64 - if codec.keyLookup != nil { - keyMap, err = MapForStringTable(codec.recKeys, codec.keyLookup) + if codec.keys != nil { + keyMap, err = codec.recKeys.MakeIDMap(codec.keys) if err != nil { return nil, fmt.Errorf("trying to find record key mapping: %w", err) } } - req = &Request{FieldTypes: make(map[string]FieldType, len(codec.fields))} + req = &Request{} valueMaps := map[string]func(*FieldOperation) error{} for name, fieldCodec := range codec.fields { // make closure survive iteration fieldCodec := fieldCodec - if fieldCodec.lookup != nil { - fieldMap, err := MapForStringTable(fieldCodec.valueKeys, fieldCodec.lookup) + if fieldCodec.keys != nil { + fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys) if err != nil { return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err) } - valueMaps[name] = func(fo *FieldOperation) error { - return fieldCodec.translate(fo, fieldMap) + if fieldCodec.fieldType == FieldTypeInt { + valueMaps[name] = func(fo *FieldOperation) error { + return translateSigned(fieldMap, fo.Signed) + } + } else { + valueMaps[name] = func(fo *FieldOperation) error { + return translateUnsigned(fieldMap, fo.Values) + } } } - req.FieldTypes[name] = fieldCodec.fieldType } for _, op := range ops { // For Clear and Write, we need to translate/sort our record ID // list. if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { if keyMap != nil { - if err = translateUnsignedSlice(op.ClearRecordIDs, keyMap); err != nil { + if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for clear op: %w", err) } } @@ -554,7 +841,7 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { continue } if keyMap != nil { - if err = fieldOp.TranslateKeys(keyMap); err != nil { + if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err) } } @@ -574,6 +861,283 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { return req, nil } +// AppendBytes appends bytes which we would expect to produce the same +// request, using this codec. It does not try to recreate FieldTypes, which +// would be handled by the codec anyway. It's written as an Append so you +// can reuse a buffer. +func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) { + if req == nil || len(req.Ops) == 0 { + return append(data, "[]"...), nil + } + dst := newJSONBuffer(data) + // we ignore the FieldTypes part of the Request, which is just there to + // let the request's ByShard use the correct sorting routines, which is + // itself sort of awful. The codec will insert it again when parsing the + // bytes. + _, _ = dst.WriteString(`,`) + for _, op := range req.Ops { + // EncodeJSON needs to have access to this codec's field data + // and key translators. + err = op.EncodeJSON(dst, codec) + if err != nil { + return nil, err + } + _, _ = dst.WriteString(`,`) + } + // delete the last comma + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + // there's very few ways an error could occur, possibly none, but + // just in case lots of internal operations stashed an error if one + // happened, so we'll return anything they came up with. + return dst.Bytes(), dst.Err() +} + +// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed +// list of quoted strings if there's a key translator, or integer values +// if the KeyTranslator is nil, comma-separated. It can error because a +// translator can error. +func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + return appendIDsJSON(dst, values, keys) +} + +func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) { + if len(values) == 0 && len(keys) == 0 { + _, _ = dst.WriteString("[]") + return + } + if len(keys) != 0 { + dst.EncodeStrings(keys) + } else { + dst.EncodeUints(values) + } +} + +func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + if len(values) == 0 { + _, _ = dst.WriteString("[]") + return nil + } + + if keys == nil { + dst.EncodeUints(values) + return nil + } + _, _ = dst.WriteString(`[`) + translated, err := keys.TranslateIDs(values...) + if err != nil { + return err + } + for _, v := range values { + dst.EncodeString(translated[v]) + _, _ = dst.WriteString(`,`) + } + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + return nil +} + +// RequestByShard makes up for the fact that we don't want to stash the +// field type data in the request, but we need it to actually do the by-shard. +func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) { + return req.ByShard(codec.fieldTypes) +} + +// EncodeJSON encodes an operation as JSON using a provided buffer. +// It uses the provided codec where necessary to help with key +// translation. +func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) { + // We're not trying to guarantee that what we produce makes sense or is + // identical to what produced us, just to write our current state out. + if o == nil { + _, _ = dst.WriteString(`{}`) + return nil + } + _, _ = dst.WriteString(`{"action":`) + dst.EncodeString(o.OpType.String()) + // it is intentional that o.Seq isn't encoded here; it makes no sense + // to allow an op to specify its seq in JSON. + if o.OpType != OpWrite { + // for Write, ClearRecordIDs and ClearFields were computed + // from the records being written, and aren't actually part of the data. + if len(o.ClearRecordIDs) != 0 { + _, _ = dst.WriteString(`,"record_ids":`) + err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys) + if err != nil { + return err + } + } + if len(o.ClearFields) != 0 { + _, _ = dst.WriteString(`,"fields":`) + dst.EncodeStrings(o.ClearFields) + } + } + if len(o.FieldOps) == 0 { + _, _ = dst.WriteString(`}`) + return + } + // collect all the fields that have a non-empty set of records. we can + // just ignore the others. + fieldNames := make([]string, 0, len(o.FieldOps)) + for field, op := range o.FieldOps { + if op != nil && len(op.RecordIDs) != 0 { + fieldNames = append(fieldNames, field) + } + } + // nevermind then + if len(fieldNames) == 0 { + _, _ = dst.WriteString(`}`) + return nil + } + _, _ = dst.WriteString(`,"records":{`) + + // now we have to invert the logic, creating records from + // fields with corresponding ops. uh-oh. + fieldOps := make([]*FieldOperation, len(o.FieldOps)) + fieldCodecs := make([]*fieldCodec, len(o.FieldOps)) + fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys + indexes := make([]int, len(o.FieldOps)) + sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] }) + next := ^uint64(0) + var idKeys map[uint64]string + if codec.keys != nil { + idKeys = make(map[uint64]string) + } + for i, field := range fieldNames { + // populate a parallel slice of field ops so we don't have + // to do map lookups for every single piece of data + op := o.FieldOps[field] + fieldOps[i] = op + fc := codec.fields[field] + if fc == nil { + return fmt.Errorf("unknown field: %q", field) + } + fieldCodecs[i] = fc + if codec.keys != nil { + // we'll build a list of keys we need for any records + for _, v := range op.RecordIDs { + idKeys[v] = "" + } + } + id := op.RecordIDs[0] + if id < next { + next = id + } + if fc.keys != nil { + thisFieldKeys := make([]string, len(op.RecordIDs)) + if fc.fieldType == FieldTypeInt { + u := make([]uint64, len(op.Signed)) + for i := range op.Signed { + u[i] = uint64(op.Signed[i]) + } + valueKeys, err := fc.keys.TranslateIDs(u...) + if err != nil { + return err + } + for j, v := range op.Signed { + thisFieldKeys[j] = valueKeys[uint64(v)] + } + } else { + valueKeys, err := fc.keys.TranslateIDs(op.Values...) + if err != nil { + return err + } + for j, v := range op.Values { + thisFieldKeys[j] = valueKeys[v] + } + } + fieldKeys[i] = thisFieldKeys + } + } + // ... no fieldops actually have any entries. therefore no record will + // have any fields set, therefore no records exist... + if next == ^uint64(0) { + _, _ = dst.WriteString(`}}`) + return nil + } + if codec.keys != nil { + idList := make([]uint64, 0, len(idKeys)) + for k := range idKeys { + idList = append(idList, k) + } + idKeys, err = codec.keys.TranslateIDs(idList...) + if err != nil { + return err + } + } + + // next is the ID of a record which exists for at least one field. + // we'll recompute it every loop. + for next != ^uint64(0) { + if codec.keys != nil { + dst.EncodeString(idKeys[next]) + } else { + dst.EncodeQuotedUint(next) + } + _, _ = dst.WriteString(`:{`) + current := next + next = ^uint64(0) + for i, field := range fieldNames { + op := fieldOps[i] + idx := indexes[i] + if idx >= len(op.RecordIDs) { + continue + } + id := op.RecordIDs[idx] + if id == current { + var j int + // count ahead to first index which is either outside the list + // or a different id + for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ { + } + // fmt.Printf("field %s encoding %d-%d (v %d, s %d, k %d)\n", + // field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i])) + // print this one, and advance this index to next position + dst.EncodeString(field) + _, _ = dst.WriteString(`:`) + var values []uint64 + var signed []int64 + var keys []string + if len(op.Values) >= j { + values = op.Values[idx:j] + } + if len(op.Signed) >= j { + signed = op.Signed[idx:j] + } + if len(fieldKeys[i]) >= j { + keys = fieldKeys[i][idx:j] + } + err = fieldCodecs[i].encode(dst, values, signed, keys) + if err != nil { + return err + } + _, _ = dst.WriteString(`,`) + indexes[i] = j + // and we'll fall through to the id < next check, so we + // just set id here. + if indexes[i] < len(op.RecordIDs) { + id = op.RecordIDs[indexes[i]] + } else { + id = ^uint64(0) + } + } + if id < next { + next = id + } + } + // we should always have a trailing comma after any entry, and + // if there were no entries we shouldn't have had a list... + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`},`) + } + dst.Truncate(dst.Len() - 1) + // close both the records list, and the whole object. + _, _ = dst.WriteString(`}}`) + // In theory, there's no way for most of these to produce errors, + // but just in case, we'll check for an error every operation or so. + return dst.Err() +} + type errFieldNotFound struct { field string } diff --git a/ingest/codec_test.go b/ingest/codec_test.go index 4490d3729..8605eef5a 100644 --- a/ingest/codec_test.go +++ b/ingest/codec_test.go @@ -16,36 +16,481 @@ package ingest import ( "fmt" + "sort" + "strings" "testing" "time" "github.com/molecula/featurebase/v2/shardwidth" ) -func unusableSampleTranslator(keys ...string) (map[string]uint64, error) { - out := make(map[string]uint64, len(keys)) - for _, key := range keys { - out[key] = uint64(len(out)) * 13 +func TestStableTranslator(t *testing.T) { + tr := newStableTranslator() + m1, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("translation error on initial keys: %v", err) + } + m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6) + if err != nil { + t.Fatalf("translation error on reverse lookup: %v", err) + } + m3, err := tr.TranslateKeys("a", "k-6") + if err != nil { + t.Fatalf("translation error on new keys: %v", err) + } + for k, v := range m3 { + if m2[v] != k { + t.Fatalf("expected round trip to equate %q and %d", k, v) + } } - return out, nil } -func TestSimpleCodec(t *testing.T) { - c, _ := NewJSONCodec(nil) - _ = c.AddSetField("set", nil) - _ = c.AddSetField("setkeys", unusableSampleTranslator) - _ = c.AddMutexField("mutex", nil) - _ = c.AddMutexField("mutexkeys", unusableSampleTranslator) - _ = c.AddTimeQuantumField("tq", nil) - _ = c.AddIntField("int", nil) - _ = c.AddIntField("intkeys", unusableSampleTranslator) +func TestMakeCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + err := codec.AddSetField("set", nil) + if err != nil { + t.Fatalf("unexpected error creating field: %v", err) + } + err = codec.AddSetField("set", nil) + if err == nil { + t.Fatalf("expected error creating duplicate field, didn't get it") + } +} + +func TestEncode(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) epoch, err := time.Parse("2006-01-02", "2020-01-01") if err != nil { t.Fatalf("can't parse sample epoch time: %v", err) } - _ = c.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) - _ = c.AddDecimalField("dec", 2) - _ = c.AddBoolField("bool") + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs := []*JSONCodec{codec} + + // redo all of that, only on a keyed translator + codec, _ = NewJSONCodec(newStableTranslator()) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs = append(codecs, codec) + + encodeTests := []*Request{ + { + Ops: []*Operation{ + { + OpType: OpWrite, + ClearRecordIDs: []uint64{0, 1, 2, 3, 5}, + ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"}, + FieldOps: map[string]*FieldOperation{ + "int": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, -3}, + }, + "intkeys": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, 1}, + }, + "set": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "setkeys": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "mutex": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "mutexkeys": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "tq": { + RecordIDs: []uint64{5, 5}, + Values: []uint64{8, 9}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "tqkeys": { + RecordIDs: []uint64{3, 3}, + Values: []uint64{2, 4}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "ts": { + RecordIDs: []uint64{0}, + Signed: []int64{1}, + }, + "bool": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{0, 1}, + }, + "dec": { + RecordIDs: []uint64{0, 1, 2}, + Signed: []int64{123, -123, 0}, + }, + }, + }, + { + OpType: OpClear, + Seq: 1, + ClearRecordIDs: []uint64{6}, + ClearFields: []string{"tq"}, + }, + }, + }, + { + // this one needs to get filled in programmatically; see below + Ops: []*Operation{ + { + OpType: OpSet, + FieldOps: map[string]*FieldOperation{}, + }, + }, + }, + { + Ops: []*Operation{ + { + OpType: OpRemove, + FieldOps: map[string]*FieldOperation{ + "set": {}, + }, + }, + }, + }, + } + // and now we populate encodeTests[1] with a larger pool of data + const dataSize = 5000 + shardCount := uint64(600) // shards we want to target + passes := uint64(0) + recordIDs := make([]uint64, dataSize) + values := make([]uint64, dataSize) + timeStamps := make([]int64, dataSize) + signedValues := make([]int64, dataSize) + for i := uint64(0); i < dataSize; i++ { + if (i % shardCount) == 0 { + passes++ + } + recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes + values[i] = (i % 4) + timeStamps[i] = int64(1234567890e9 + (i * 100e9)) + signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys + } + // ensure record IDs are sorted, because other stuff might rely on this + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op := encodeTests[1].Ops[0] + op.FieldOps["tq"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + Signed: append([]int64{}, timeStamps...), + } + op.FieldOps["tqkeys"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: values, + Signed: timeStamps, + } + op.FieldOps["int"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Signed: append([]int64{}, signedValues...), + } + op.FieldOps["intkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Signed: signedValues, + } + // for sets, we want to shuffle things into fewer shards, and ensure + // non-duplication of values within each record, but also have lots + // of duplication of record IDs in the low shards + recordIDs = make([]uint64, dataSize) + values = make([]uint64, dataSize) + valuesPerRecord := uint64(5) + recordsPerShard := dataSize / valuesPerRecord / 30 + if recordsPerShard < 1 { + recordsPerShard = 1 + } + shard := uint64(0) + nextID := uint64(0) + nextValue := uint64(0) + for i := uint64(0); i < dataSize; i++ { + recordIDs[i] = nextID + values[i] = nextValue + (i % valuesPerRecord) + nextValue++ + if nextValue == valuesPerRecord { + nextValue = 0 + nextID++ + if nextID%(1< 1 { + valuesPerRecord-- + } + } + } + } + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op.FieldOps["set"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + } + op.FieldOps["setkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + var buf []byte + for i, tc := range encodeTests { + for _, c := range codecs { + data, err := c.AppendBytes(tc, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + data, err = c.AppendBytes(req, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req2, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req2.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + } + } +} + +func TestCodecErrors(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + testCases := []struct { + name string + json []byte + error string + }{ + { + name: "no action", + json: []byte(`[{"records":{"0":{"set":[0]}}}]`), + error: "action not specified", + }, + { + name: "unknown action", + json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`), + error: "unknown action", + }, + { + name: "unknown field", + json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`), + error: "field not found", + }, + { + name: "unknown operation field", + json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`), + error: "unknown operation field", + }, + { + name: "expected operation", + json: []byte(`[true]`), + error: "expected operation", + }, + { + name: "expecting key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`), + error: "expecting key", + }, + { + name: "invalid int for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`), + error: "boolean should be", + }, + { + name: "invalid number for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`), + error: "looks like Number", + }, + { + name: "invalid string for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`), + error: "expecting boolean", + }, + { + name: "nonsense bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`), + error: "boolean should be", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`), + error: "expecting value", + }, + { + name: "expecting array-key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`), + error: "expecting key", + }, + { + name: "expecting array-value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`), + error: "expecting value", + }, + { + name: "expecting numeric array-value", + json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`), + error: "invalid syntax", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting int key", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting int value", + json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting string array-value", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "expecting numeric mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting mutex key", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting mutex string value", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "time quantum invalid time", + json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`), + error: "expecting time", + }, + { + name: "time stamp invalid integer", + json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`), + error: "parsing numeric time", + }, + { + name: "time stamp invalid string", + json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`), + error: "parsing time", + }, + { + name: "time stamp invalid type", + json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`), + error: "expecting time", + }, + { + name: "invalid decimal", + json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`), + error: "expecting floating", + }, + { + name: "duplicate record", + json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`), + error: "duplicated in input", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req, err := codec.ParseBytes(tc.json) + if err == nil { + req.Dump(t.Logf) + t.Fatalf("expected error like %q, got request instead", tc.error) + } else { + msg := err.Error() + if !strings.Contains(msg, tc.error) { + t.Fatalf("expected error like %q, got %q", tc.error, msg) + } + } + }) + } +} + +func TestSimpleCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") var nextShard = uint64(1< 0 { - subOp.Values = f.Values[prev:endIndex] - } - if len(f.Signed) > 0 { - subOp.Signed = f.Signed[prev:endIndex] - } - target[shard] = subOp - prev = endIndex +// clone makes a duplicate of the operation without shared storage +func (f *FieldOperation) clone() *FieldOperation { + f2 := &FieldOperation{ + RecordIDs: append([]uint64{}, f.RecordIDs...), + Values: append([]uint64{}, f.Values...), + Signed: append([]int64{}, f.Signed...), } + return f2 } func ShardIDs(ids []uint64) (out map[uint64][]uint64) { @@ -349,10 +396,22 @@ func (f *FieldOperation) SortByKeys(keys []uint64) { // doesn't help as much as you'd hope. This beats using stdlib sort by about // a factor of two for those small N, for larger N we're using the radix sort // that calls this. +// +// External keys exist only when we are sorting by value, which is to say, +// when we're using row-oriented formats (set, mutex, time quantum). +// For int/decimal/timestamp fields, we're sorting by record only. +// So, if keys is the same as f.RecordIDs, we're looking at an int field +// or equivalent, so Signed exists and Values doesn't exist. +// Otherwise, we might be looking at a time quantum field (both exist) +// or set/mutex (only Values exist). func simpleSort(f *FieldOperation, keys []uint64) { - if keys != nil { + // keys might actually just point to record IDs, in which case, we don't + // want to shuffle the corresponding RecordIDs too, because that would just + // reverse our swaps. If they're different, we actually need to swap them + // both. + if &keys[0] != &f.RecordIDs[0] { // sorting by record IDs - if f.Values != nil && f.Signed != nil { + if f.Values != nil && f.Signed != nil { // time quantum field for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -362,7 +421,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } - } else if f.Values != nil { + } else if f.Values != nil { // set/mutex/bool for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -371,7 +430,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { + } else if f.Signed != nil { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -379,7 +438,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else { + } else { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -388,7 +447,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } } else { - if f.Values != nil && f.Signed != nil { + if f.Values == nil && f.Signed != nil { + for i := 1; i < len(f.RecordIDs); i++ { + for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { + f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] + f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] + } + } + } else if f.Values != nil && f.Signed != nil { // can't happen, we think for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -396,22 +462,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else if f.Values != nil { + } else if f.Values != nil { // only happens during testing for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else { - // why do we only have record IDs? I don't know + } else { // should definitely not happen for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -493,12 +551,7 @@ func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) { if end-start > 32 { sortPartialByKeys(&bucketOp, keys[start:end], nextShift) } else { - // naive stdlib sort - if externalKeys { - simpleSort(&bucketOp, keys[start:end]) - } else { - simpleSort(&bucketOp, nil) - } + simpleSort(&bucketOp, keys[start:end]) } } } @@ -534,16 +587,15 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { if expected == nil { return nil } - if len(expected.RecordIDs) == 0 && len(expected.Values) == 0 && len(expected.Signed) == 0 { + // We don't worry about non-empty Values or Signed here, because in theory + // RecordIDs are the Source of Truth as to what's in the op. + if len(expected.RecordIDs) == 0 { return nil } return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs)) } if expected == nil { - if got == nil { - return nil - } - if len(got.RecordIDs) == 0 && len(got.Values) == 0 && len(got.Signed) == 0 { + if len(got.RecordIDs) == 0 { return nil } return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs)) @@ -578,52 +630,6 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { return nil } -func translateUnsignedSlice(target []uint64, mapping []uint64) (err error) { - oops := 0 - for i, v := range target { - if v >= uint64(len(mapping)) { - oops++ - } else { - target[i] = mapping[v] - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range keys when applying translation mapping", oops) - } - return nil -} - -// TranslateUnsigned translates keys according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateKeys(mapping []uint64) error { - return translateUnsignedSlice(op.RecordIDs, mapping) -} - -// TranslateUnsigned translates values according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateUnsigned(mapping []uint64) error { - return translateUnsignedSlice(op.Values, mapping) -} - -// TranslateSigned translates signed values according to the provided mapping. -// If we're using this, it's because we're in an integer-type field, which -// admits using keys for fields, so all key values are actually non-negative, -// but the field's type still requires values be expressed as signed ints. -func (op *FieldOperation) TranslateSigned(mapping []uint64) error { - oops := 0 - for i, v := range op.Signed { - if v >= int64(len(mapping)) { - oops++ - } else { - op.Signed[i] = int64(mapping[v]) - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} - // ShardOperations is a set of Operations associated with a specific shard. type ShardOperations struct { Shard uint64 @@ -633,19 +639,17 @@ type ShardOperations struct { // Request is a complete ingest request, which may be any combination // of operations, which may apply to multiple shards. type Request struct { - FieldTypes map[string]FieldType - Ops []*Operation + Ops []*Operation } // ShardedRequest is an ingest request, split up into individual per-shard // operations. type ShardedRequest struct { - FieldTypes map[string]FieldType - Ops map[uint64][]*Operation + Ops map[uint64][]*Operation } // ByShard converts a request into the same request, only sharded. -func (r *Request) ByShard() (*ShardedRequest, error) { +func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) { if len(r.Ops) == 0 { return &ShardedRequest{Ops: nil}, nil } @@ -662,12 +666,12 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { sharded := ShardIDs(op.ClearRecordIDs) for shard, data := range sharded { - shards[shard] = &Operation{OpType: op.OpType, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} + shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} } } for field, fieldOp := range op.FieldOps { sharded := fieldOp.ByShard() - sorter := fieldTypeSorts[r.FieldTypes[field]] + sorter := fieldTypeSorts[fields[field]] if sorter == nil { sorter = (*FieldOperation).SortByRecords } @@ -678,7 +682,7 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpWrite { return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard) } - shardOp = &Operation{OpType: op.OpType} + shardOp = &Operation{OpType: op.OpType, Seq: op.Seq} shards[shard] = shardOp shardOp.FieldOps = map[string]*FieldOperation{field: data} } else { @@ -697,18 +701,139 @@ func (r *Request) ByShard() (*ShardedRequest, error) { return &ShardedRequest{Ops: req}, nil } +// merge combines the components of a sharded request back into a single +// unsharded request, processing shards in numerical order. +func (s *ShardedRequest) merge() *Request { + req := &Request{} + if s == nil || len(s.Ops) == 0 { + return req + } + shards := make([]uint64, 0, len(s.Ops)) + for shard := range s.Ops { + shards = append(shards, shard) + } + sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] }) + for _, shard := range shards { + ops := s.Ops[shard] + for _, op := range ops { + var _ *Operation + if op.Seq >= len(req.Ops) { + // Pad out with nil *Operations to the required length + req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...) + } + if req.Ops[op.Seq] == nil { + req.Ops[op.Seq] = op.clone() + continue + } + req.Ops[op.Seq].merge(op) + } + } + return req +} + func (r *Request) Dump(logf func(string, ...interface{})) { logf("req: %#v", r) for _, op := range r.Ops { logf("op: %#v", op) if len(op.ClearRecordIDs) > 0 { - logf(" clearRecordIDs: %d", op.ClearRecordIDs) + if len(op.ClearRecordIDs) > 8 { + logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8) + } else { + logf(" clearRecordIDs: %d", op.ClearRecordIDs) + } } if len(op.ClearFields) > 0 { - logf(" clearFields: %s", op.ClearFields) + if len(op.ClearFields) > 8 { + logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8) + } else { + logf(" clearFields: %s", op.ClearFields) + } } for field, fieldOp := range op.FieldOps { - logf(" field %q: %#v", field, fieldOp) + if fieldOp != nil { + logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed)) + if len(fieldOp.RecordIDs) > 0 { + if len(fieldOp.RecordIDs) > 8 { + logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8) + } else { + logf(" records %d", fieldOp.RecordIDs) + } + } + } else { + logf(" field %q: nil op", field) + } } } } + +func (r *Request) Compare(other *Request) error { + if other == nil { + if r != nil && len(r.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if r == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + ops := r.Ops + ops2 := other.Ops + if len(ops2) != len(ops) { + return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("op %d: %v", i, err) + } + } + return nil +} + +// Compare checks whether two ShardedRequest objects represent the same +// data. Empty shards shouldn't have entries in the map in the first place, +// so we don't accept a nil or 0-length slice of ops as equal to the +// shard key not existing, but we do accept nil or empty requests as +// equal to each other. +func (s *ShardedRequest) Compare(other *ShardedRequest) error { + if other == nil { + if s != nil && len(s.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if s == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + for shard, ops := range s.Ops { + ops2, ok := other.Ops[shard] + if !ok { + return fmt.Errorf("shard %d missing in other", shard) + } + if len(ops2) != len(ops) { + return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("shard %d, op %d: %v", shard, i, err) + } + } + } + if len(other.Ops) != len(s.Ops) { + for shard := range other.Ops { + if _, ok := s.Ops[shard]; !ok { + return fmt.Errorf("shard %d missing in self", shard) + } + } + } + return nil +} diff --git a/ingest/op_test.go b/ingest/op_test.go index b0b196e07..1646e8993 100644 --- a/ingest/op_test.go +++ b/ingest/op_test.go @@ -12,31 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ingest_test +package ingest import ( "math/rand" - "reflect" "testing" - "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/shardwidth" ) type opShardingTestCase struct { name string - input ingest.Request - output *ingest.ShardedRequest + input *Request + output *ShardedRequest } var opShardingTestCases = []opShardingTestCase{ { name: "sample", - input: ingest.Request{ - Ops: []*ingest.Operation{ + input: &Request{ + Ops: []*Operation{ { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ + OpType: OpSet, + FieldOps: map[string]*FieldOperation{ "shard0": { RecordIDs: []uint64{0, 1}, }, @@ -55,8 +53,9 @@ var opShardingTestCases = []opShardingTestCase{ }, }, { - OpType: ingest.OpRemove, - FieldOps: map[string]*ingest.FieldOperation{ + OpType: OpRemove, + Seq: 1, + FieldOps: map[string]*FieldOperation{ "shard0-2": { RecordIDs: []uint64{1, 2< 1 { + valuesPerRecord-- + } + } + } + } + op.FieldOps["set"] = &FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + fieldTypes := codec.FieldTypes() + sharded, err := req.ByShard(fieldTypes) + if err != nil { + t.Errorf("sharding: unexpected error %v", err) + } + merged := sharded.merge() + if err := req.Compare(merged); err != nil { + t.Fatalf("merge comparison: %v", err) } } @@ -139,7 +240,7 @@ func TestFancySharding(t *testing.T) { const recordCount = 5000 grr := rand.New(rand.NewSource(0)) for i := 0; i < 100; i++ { - f := &ingest.FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} + f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} shards := make([]int, shardLimit) for j := range f.RecordIDs { v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent)) diff --git a/ingest/translate.go b/ingest/translate.go deleted file mode 100644 index 7ce7ea6dd..000000000 --- a/ingest/translate.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 Molecula Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ingest diff --git a/ingest/translate_test.go b/ingest/translate_test.go new file mode 100644 index 000000000..67daf73c3 --- /dev/null +++ b/ingest/translate_test.go @@ -0,0 +1,137 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "fmt" + "testing" +) + +// stableTranslator implements a key translator that can be reused and +// will continue to give the same keys for the same values. Possibly +// surprisingly, it will invent new keys for IDs it is asked about but +// hasn't seen. This allows us to give a codec which would use keys on +// translation a request which contains arbitrary numbers, and request +// text that would parse into that request. +type stableTranslator struct { + in map[string]uint64 + out map[uint64]string + next uint64 +} + +func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + ret := make(map[string]uint64, len(keys)) + for _, key := range keys { + if existing, ok := s.in[key]; ok { + ret[key] = existing + continue + } + id := s.next + // but what if someone already translated that ID, so now it already + // exists? + if _, ok := s.out[id]; ok { + for k := range s.out { + if k > id { + id = k + } + } + // one larger than the largest we already have. this could + // wrap around, in which case, it's your own fault. + id++ + } + s.next = id + 1 + s.in[key] = id + s.out[id] = key + ret[key] = id + } + return ret, nil +} + +func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + ret := make(map[uint64]string, len(ids)) + for _, id := range ids { + if existing, ok := s.out[id]; ok { + ret[id] = existing + continue + } + key := fmt.Sprintf("k-%d", id) + s.in[key] = id + s.out[id] = key + ret[id] = key + if id >= s.next { + s.next = id + 1 + } + } + return ret, nil +} + +// newStableTranslator produces a translator which can translate forwards +// and backwards and invent new things if it needs to. Don't use this. +func newStableTranslator() *stableTranslator { + return &stableTranslator{ + in: make(map[string]uint64), + out: make(map[uint64]string), + } +} + +func TestTranslateReuse(t *testing.T) { + // the original stable-translator design had a flaw in that it + // assumed that each new ID would always come from a string translation, + // never from a key translation, and that they'd show up sequentially. + tr := newStableTranslator() + orig, err := tr.TranslateIDs(1, 2) + tr.next = 1 // intentionally break the translation logic for testing purposes + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var s [6]string + stash := s[:0] + for _, v := range orig { + stash = append(stash, v) + } + for i := range s[len(orig):] { + stash = append(stash, fmt.Sprintf("key-%d", i)) + } + keys, err := tr.TranslateKeys(stash...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + ids := make([]uint64, 0, len(keys)) + for _, v := range keys { + ids = append(ids, v) + } + idMap, err := tr.TranslateIDs(ids...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for k, v := range keys { + if idMap[v] != k { + t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q", + k, v, v, idMap[v]) + } + } + for id, key := range idMap { + if keys[key] != id { + t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } + for id, key := range orig { + if keys[key] != id { + t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } +} diff --git a/ingest/vec.go b/ingest/vec.go index 082ff8575..14d62cbbc 100644 --- a/ingest/vec.go +++ b/ingest/vec.go @@ -19,8 +19,6 @@ import ( "reflect" "strconv" "unsafe" - - "github.com/pkg/errors" ) // StringTable is a mapping of strings to temporary IDs. @@ -85,11 +83,8 @@ func (tbl *StringTable) IntID(in []byte) (int64, error) { // integers and a translation function from strings to "real" keys, yields // a translation/lookup slice. If it cannot translate all the keys, it // returns an error. -// -// The lookup function corresponds to the FindKeys/CreateKeys methods of -// featurebase translators, by an AMAZING coincidence. -func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint64, error)) ([]uint64, error) { - lookedUp, err := lookup(tbl.names...) +func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) { + lookedUp, err := keys.TranslateKeys(tbl.names...) if err != nil { return nil, err } @@ -103,22 +98,38 @@ func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint return out, nil } -// TimeFormatForUnit returns the time transfer format (between the update encoder and the update applier) with appropriate resolution for a quantum unit. -func TimeFormatForUnit(unit rune) string { - switch unit { - case 'Y': - return "2006" - case 'M': - return "200601" - case 'D': - return "20060102" - case 'H': - return "2006010203" - default: - panic(errors.Errorf("invalid quantum unit: %q", unit)) +// translateSigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateSigned(mapping []uint64, values []int64) error { + oops := 0 + for i, v := range values { + if v >= int64(len(mapping)) { + oops++ + } else { + values[i] = int64(mapping[v]) + } } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil } -// TODO: bool - -// TODO: timestamp (just sugar on top of IntVector) +// translateUnsigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateUnsigned(mapping []uint64, values []uint64) error { + oops := 0 + for i, v := range values { + if v >= uint64(len(mapping)) { + oops++ + } else { + values[i] = mapping[v] + } + } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil +} diff --git a/ingest/vec_test.go b/ingest/vec_test.go new file mode 100644 index 000000000..45a103645 --- /dev/null +++ b/ingest/vec_test.go @@ -0,0 +1,114 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "errors" + "testing" +) + +type badTranslator struct{} + +func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + if len(keys) == 0 { + return nil, errors.New("no keys") + } + m := make(map[string]uint64) + skip := true + for i, k := range keys { + if skip { + skip = false + continue + } + m[k] = uint64(i) + } + out := make([]uint64, len(keys)-1) + for i := range out { + out[i] = uint64(i) + } + return nil, nil +} + +func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) { + return nil, nil +} + +func TestStringTableErrors(t *testing.T) { + tbl := NewStringTable() + btr := badTranslator{} + _, keyErr := tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error passed up from failed translate, didn't get it") + } + a1, err := tbl.ID([]byte("a")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + b1, err := tbl.ID([]byte("b")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + _, keyErr = tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error for short translate, didn't get it") + } + tr := newStableTranslator() + _, err = tr.TranslateKeys("c", "d") + if err != nil { + t.Fatalf("translating stray keys: %v", err) + } + m, err := tbl.MakeIDMap(tr) + if err != nil { + t.Fatalf("creating lookup: %v", err) + } + var y = []uint64{a1, b1} + err = translateUnsigned(m, y) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + trResults, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("unexpected translation error: %v", err) + } + if y[0] != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], y[0]) + } + if y[1] != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], y[1]) + } + y[0] = a1 + y[1] = (a1 + b1 + 1) // assumed not to be any of them + err = translateUnsigned(m, y) + if err == nil { + t.Fatalf("no error from translating invalid table") + } + z := []int64{int64(a1), int64(b1)} + err = translateSigned(m, z) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + if uint64(z[0]) != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], z[0]) + } + if uint64(z[1]) != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], z[1]) + } + z[0] = int64(a1) + z[1] = int64(a1 + b1 + 1) // assumed not to be any of them + err = translateSigned(m, z) + if err == nil { + t.Fatalf("no error from translating invalid table") + } +} diff --git a/ingest_test.go b/ingest_test.go index 649f5dd16..5cd4beac4 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -26,9 +26,7 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v2" - "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/ingest" - "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/test" "github.com/pkg/errors" ) @@ -448,14 +446,7 @@ func TestIngestTestcases(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c := test.MustRunCluster(t, 1, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("node0"), - pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - ) + c := test.MustRunCluster(t, 3) defer c.Close() coord := c.GetPrimary() diff --git a/pb/private.pb.go b/pb/private.pb.go index 54366c2cb..3a2807420 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -2479,6 +2479,234 @@ func (m *ResizeNodeMessage) GetAction() string { return "" } +type FieldOperation struct { + RecordIDs []uint64 `protobuf:"varint,1,rep,packed,name=RecordIDs,proto3" json:"RecordIDs,omitempty"` + Values []uint64 `protobuf:"varint,2,rep,packed,name=Values,proto3" json:"Values,omitempty"` + Signed []int64 `protobuf:"varint,3,rep,packed,name=Signed,proto3" json:"Signed,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldOperation) Reset() { *m = FieldOperation{} } +func (m *FieldOperation) String() string { return proto.CompactTextString(m) } +func (*FieldOperation) ProtoMessage() {} +func (*FieldOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{39} +} +func (m *FieldOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FieldOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FieldOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FieldOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOperation.Merge(m, src) +} +func (m *FieldOperation) XXX_Size() int { + return m.Size() +} +func (m *FieldOperation) XXX_DiscardUnknown() { + xxx_messageInfo_FieldOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldOperation proto.InternalMessageInfo + +func (m *FieldOperation) GetRecordIDs() []uint64 { + if m != nil { + return m.RecordIDs + } + return nil +} + +func (m *FieldOperation) GetValues() []uint64 { + if m != nil { + return m.Values + } + return nil +} + +func (m *FieldOperation) GetSigned() []int64 { + if m != nil { + return m.Signed + } + return nil +} + +type ShardIngestOperation struct { + OpType string `protobuf:"bytes,1,opt,name=OpType,proto3" json:"OpType,omitempty"` + ClearRecordIDs []uint64 `protobuf:"varint,2,rep,packed,name=ClearRecordIDs,proto3" json:"ClearRecordIDs,omitempty"` + ClearFields []string `protobuf:"bytes,3,rep,name=ClearFields,proto3" json:"ClearFields,omitempty"` + FieldOps map[string]*FieldOperation `protobuf:"bytes,4,rep,name=FieldOps,proto3" json:"FieldOps,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperation) Reset() { *m = ShardIngestOperation{} } +func (m *ShardIngestOperation) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperation) ProtoMessage() {} +func (*ShardIngestOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{40} +} +func (m *ShardIngestOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperation.Merge(m, src) +} +func (m *ShardIngestOperation) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperation) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperation proto.InternalMessageInfo + +func (m *ShardIngestOperation) GetOpType() string { + if m != nil { + return m.OpType + } + return "" +} + +func (m *ShardIngestOperation) GetClearRecordIDs() []uint64 { + if m != nil { + return m.ClearRecordIDs + } + return nil +} + +func (m *ShardIngestOperation) GetClearFields() []string { + if m != nil { + return m.ClearFields + } + return nil +} + +func (m *ShardIngestOperation) GetFieldOps() map[string]*FieldOperation { + if m != nil { + return m.FieldOps + } + return nil +} + +type ShardIngestOperations struct { + Ops []*ShardIngestOperation `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperations) Reset() { *m = ShardIngestOperations{} } +func (m *ShardIngestOperations) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperations) ProtoMessage() {} +func (*ShardIngestOperations) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{41} +} +func (m *ShardIngestOperations) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperations) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperations.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperations) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperations.Merge(m, src) +} +func (m *ShardIngestOperations) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperations) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperations.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperations proto.InternalMessageInfo + +func (m *ShardIngestOperations) GetOps() []*ShardIngestOperation { + if m != nil { + return m.Ops + } + return nil +} + +type ShardedIngestRequest struct { + Ops map[uint64]*ShardIngestOperations `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardedIngestRequest) Reset() { *m = ShardedIngestRequest{} } +func (m *ShardedIngestRequest) String() string { return proto.CompactTextString(m) } +func (*ShardedIngestRequest) ProtoMessage() {} +func (*ShardedIngestRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{42} +} +func (m *ShardedIngestRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardedIngestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardedIngestRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardedIngestRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardedIngestRequest.Merge(m, src) +} +func (m *ShardedIngestRequest) XXX_Size() int { + return m.Size() +} +func (m *ShardedIngestRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ShardedIngestRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardedIngestRequest proto.InternalMessageInfo + +func (m *ShardedIngestRequest) GetOps() map[uint64]*ShardIngestOperations { + if m != nil { + return m.Ops + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "pb.IndexMeta") proto.RegisterType((*FieldOptions)(nil), "pb.FieldOptions") @@ -2520,103 +2748,121 @@ func init() { proto.RegisterType((*TransactionStats)(nil), "pb.TransactionStats") proto.RegisterType((*ResizeAbortMessage)(nil), "pb.ResizeAbortMessage") proto.RegisterType((*ResizeNodeMessage)(nil), "pb.ResizeNodeMessage") + proto.RegisterType((*FieldOperation)(nil), "pb.FieldOperation") + proto.RegisterType((*ShardIngestOperation)(nil), "pb.ShardIngestOperation") + proto.RegisterMapType((map[string]*FieldOperation)(nil), "pb.ShardIngestOperation.FieldOpsEntry") + proto.RegisterType((*ShardIngestOperations)(nil), "pb.ShardIngestOperations") + proto.RegisterType((*ShardedIngestRequest)(nil), "pb.ShardedIngestRequest") + proto.RegisterMapType((map[uint64]*ShardIngestOperations)(nil), "pb.ShardedIngestRequest.OpsEntry") } func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1450 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xf7, 0xc3, 0xb1, 0x7d, 0x1c, 0x27, 0xce, 0x34, 0xea, 0xbb, 0xfd, 0x78, 0x23, 0x77, - 0x5e, 0x44, 0x43, 0x25, 0x22, 0x51, 0x2e, 0x8a, 0xe0, 0xa6, 0x49, 0x9c, 0x16, 0x53, 0xd2, 0x86, - 0x71, 0xda, 0x5b, 0x34, 0x5e, 0x8f, 0x9a, 0x55, 0xd6, 0xbb, 0x66, 0x3f, 0x52, 0xbb, 0x17, 0x48, - 0x20, 0x10, 0xfc, 0x04, 0x7e, 0x06, 0x37, 0xfc, 0x07, 0x6e, 0x90, 0xf8, 0x09, 0xa8, 0xfc, 0x11, - 0x34, 0x67, 0x66, 0x76, 0xd7, 0xae, 0x5b, 0x43, 0xc4, 0xdd, 0x9e, 0xe7, 0xcc, 0x9c, 0xf3, 0x9c, - 0x8f, 0x39, 0x33, 0x0b, 0xed, 0x49, 0x12, 0x5c, 0xf0, 0x4c, 0xec, 0x4d, 0x92, 0x38, 0x8b, 0x89, - 0x3d, 0x19, 0x5e, 0x5f, 0x9f, 0xe4, 0xc3, 0x30, 0xf0, 0x15, 0x42, 0x1f, 0x42, 0xb3, 0x1f, 0x8d, - 0xc4, 0xf4, 0x58, 0x64, 0x9c, 0x10, 0x70, 0x1f, 0x89, 0x59, 0xea, 0x39, 0x5d, 0x6b, 0xb7, 0xc1, - 0xf0, 0x9b, 0xbc, 0x0b, 0x1b, 0xa7, 0x09, 0xf7, 0xcf, 0x8f, 0xa6, 0x41, 0x9a, 0x89, 0xc8, 0x17, - 0x9e, 0x8b, 0xda, 0x05, 0x94, 0xfe, 0xec, 0xc0, 0xfa, 0x83, 0x40, 0x84, 0xa3, 0x27, 0x93, 0x2c, - 0x88, 0xa3, 0x54, 0x1a, 0x3b, 0x9d, 0x4d, 0x84, 0xd7, 0xe8, 0x5a, 0xbb, 0x4d, 0x86, 0xdf, 0xe4, - 0x26, 0x34, 0x0f, 0xb9, 0x7f, 0x26, 0x50, 0xe1, 0xa0, 0xa2, 0x04, 0x0a, 0xed, 0x20, 0x78, 0xa9, - 0xbc, 0xb4, 0x59, 0x09, 0x90, 0x2e, 0xb4, 0x4e, 0x83, 0xb1, 0xf8, 0x22, 0xe7, 0x51, 0x96, 0x8f, - 0xbd, 0x1a, 0xee, 0xae, 0x42, 0xe4, 0x2a, 0xac, 0x3d, 0x09, 0x47, 0xc7, 0x41, 0xe4, 0x35, 0xbb, - 0xd6, 0xae, 0xc3, 0xb4, 0x64, 0x70, 0x3e, 0xf5, 0xa0, 0xc4, 0xf9, 0xb4, 0x08, 0xb7, 0x35, 0x1f, - 0xee, 0xe3, 0x78, 0x90, 0xf1, 0x68, 0xc4, 0x93, 0xd1, 0xb3, 0x40, 0xbc, 0xf0, 0xd6, 0x55, 0xb8, - 0xf3, 0xa8, 0xdc, 0x7b, 0xc0, 0x53, 0xe1, 0xb5, 0xd1, 0x22, 0x7e, 0x93, 0xeb, 0xd0, 0x38, 0x08, - 0xb2, 0x9e, 0x98, 0x64, 0x67, 0xde, 0x46, 0xd7, 0xda, 0x75, 0x59, 0x21, 0x93, 0x6d, 0xa8, 0x0d, - 0x7c, 0x1e, 0x0a, 0x6f, 0x13, 0x37, 0x28, 0x81, 0x50, 0x58, 0x7f, 0x10, 0x27, 0x22, 0x78, 0x1e, - 0x61, 0x11, 0xbc, 0x0e, 0x06, 0x35, 0x87, 0x91, 0xff, 0x81, 0x23, 0x43, 0xda, 0xea, 0x5a, 0xbb, - 0xad, 0xbb, 0xad, 0xbd, 0xc9, 0x70, 0xaf, 0x27, 0xfc, 0x60, 0xcc, 0x43, 0x26, 0x71, 0x54, 0xf3, - 0xa9, 0x47, 0x96, 0xa9, 0xf9, 0x54, 0x72, 0x92, 0x29, 0x7a, 0x1a, 0x05, 0x99, 0x77, 0x05, 0xad, - 0x17, 0x32, 0xa5, 0xb0, 0xd1, 0x1f, 0x4f, 0xe2, 0x24, 0x63, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48, - 0x07, 0x9c, 0xa3, 0x24, 0xf1, 0x2c, 0x5c, 0x28, 0x3f, 0xe9, 0xd7, 0xd0, 0x39, 0x08, 0x63, 0xff, - 0xbc, 0xc7, 0x33, 0xce, 0xc4, 0x57, 0xb9, 0x48, 0x33, 0x19, 0x8b, 0xa2, 0xab, 0xd6, 0x29, 0x41, - 0xa2, 0x58, 0x7f, 0xcf, 0x56, 0x28, 0x0a, 0x32, 0x4f, 0x98, 0x45, 0x55, 0x2e, 0xfc, 0xc6, 0x5c, - 0x9c, 0xf1, 0x64, 0x84, 0x35, 0x76, 0x99, 0x12, 0x24, 0x8a, 0x9e, 0xb0, 0x2f, 0x5c, 0xa6, 0x04, - 0xda, 0x87, 0xad, 0x8a, 0x7f, 0x4d, 0xf3, 0x2a, 0xac, 0xb1, 0xf8, 0x45, 0xbf, 0x97, 0x7a, 0x56, - 0xd7, 0xd9, 0x75, 0x99, 0x96, 0xb0, 0x81, 0xe2, 0x30, 0x1f, 0x47, 0x52, 0x65, 0xa3, 0xaa, 0x04, - 0xe8, 0x35, 0xa8, 0x61, 0x37, 0xc9, 0x28, 0xcb, 0xbd, 0xf2, 0x93, 0x7e, 0x63, 0x41, 0xf3, 0x98, - 0x4f, 0x91, 0x48, 0x4a, 0xee, 0x41, 0xc3, 0xd4, 0x1a, 0x17, 0xb5, 0xee, 0xde, 0x90, 0x79, 0x2d, - 0x16, 0xec, 0x19, 0xed, 0x51, 0x94, 0x25, 0x33, 0x56, 0x2c, 0xbe, 0xfe, 0x09, 0xb4, 0xe7, 0x54, - 0xd2, 0xd3, 0xb9, 0x98, 0x99, 0x7c, 0x9e, 0x8b, 0x99, 0x8c, 0xf2, 0x82, 0x87, 0xb9, 0xc0, 0x2c, - 0xb9, 0x4c, 0x09, 0x1f, 0xdb, 0x1f, 0x59, 0xf4, 0x19, 0x90, 0xc3, 0x44, 0xf0, 0x4c, 0xa0, 0x93, - 0x63, 0x91, 0xa6, 0xfc, 0xb9, 0x58, 0x95, 0x6b, 0xa7, 0x9a, 0xeb, 0x22, 0xaf, 0x76, 0x25, 0xaf, - 0xf4, 0x0e, 0x90, 0x9e, 0x08, 0x45, 0x26, 0xf4, 0x39, 0x7f, 0x8b, 0x5d, 0x7a, 0x6e, 0x38, 0xac, - 0x5e, 0x4b, 0x6e, 0x81, 0x2b, 0x87, 0x06, 0x3a, 0x6b, 0xdd, 0x6d, 0xcb, 0x0c, 0x15, 0x93, 0x84, - 0xa1, 0x0a, 0xeb, 0x81, 0xe6, 0x46, 0xfb, 0x19, 0x52, 0x75, 0x58, 0x09, 0xd0, 0xef, 0x2c, 0xe3, - 0x0d, 0xe9, 0xff, 0xcd, 0x88, 0xe7, 0xba, 0xeb, 0x1d, 0xcd, 0xc1, 0x41, 0x0e, 0x1d, 0xc9, 0xa1, - 0x3a, 0x83, 0x96, 0xd1, 0x70, 0x17, 0x69, 0xdc, 0x37, 0xf9, 0xb9, 0x2c, 0x0b, 0xea, 0xc3, 0x0d, - 0x65, 0x61, 0xff, 0x82, 0x07, 0x21, 0x1f, 0x86, 0xff, 0xa8, 0x84, 0x73, 0x01, 0x79, 0x50, 0xc7, - 0xbd, 0xfd, 0x9e, 0x3e, 0x06, 0x46, 0xa4, 0x39, 0x94, 0x27, 0xea, 0x31, 0x1f, 0x0b, 0x6d, 0x0d, - 0xbf, 0x8b, 0x3c, 0xd8, 0x6f, 0xcd, 0xc3, 0x36, 0xd4, 0xe4, 0xf9, 0x93, 0xf3, 0xdd, 0x91, 0x2e, - 0x51, 0x58, 0x91, 0x9d, 0xf7, 0x61, 0x6d, 0xe0, 0x9f, 0x89, 0x31, 0x27, 0xff, 0x87, 0x3a, 0x32, - 0x17, 0xa9, 0x3e, 0x14, 0xcd, 0xa2, 0xe4, 0xcc, 0x68, 0xe8, 0xf7, 0x96, 0x0e, 0x76, 0x29, 0xcd, - 0x39, 0x57, 0xf6, 0x82, 0x2b, 0x72, 0x1b, 0xea, 0x9a, 0x2f, 0x4e, 0x8b, 0xd7, 0x7a, 0xca, 0x68, - 0xc9, 0x2d, 0x58, 0xc3, 0xe8, 0x52, 0xcf, 0x2d, 0x89, 0x20, 0xc2, 0xb4, 0x82, 0x1e, 0x81, 0xf3, - 0x94, 0xf5, 0xe5, 0xa0, 0x40, 0xf6, 0x86, 0x86, 0x96, 0x24, 0xb9, 0x4f, 0xe3, 0x34, 0xd3, 0xb9, - 0xc7, 0x6f, 0x89, 0x9d, 0xc4, 0x89, 0xea, 0xd3, 0x36, 0xc3, 0x6f, 0xfa, 0xa3, 0x05, 0xee, 0xe3, - 0x78, 0x24, 0xc8, 0x06, 0xd8, 0xfd, 0x9e, 0x36, 0x62, 0xf7, 0x7b, 0xe4, 0x1a, 0xda, 0xd7, 0xf9, - 0xae, 0x4b, 0xff, 0x4f, 0x59, 0x9f, 0xa1, 0xcf, 0x9b, 0xd0, 0xec, 0xa7, 0x27, 0x49, 0x30, 0xe6, - 0xc9, 0x4c, 0xdf, 0xa4, 0x25, 0x80, 0x67, 0x34, 0xe3, 0x99, 0xba, 0xdf, 0x9a, 0x4c, 0x09, 0xe4, - 0x16, 0xd4, 0x1f, 0xb2, 0x93, 0x43, 0x69, 0xb2, 0x36, 0x6f, 0xd2, 0xe0, 0xf4, 0x3e, 0x74, 0x24, - 0x13, 0x5c, 0x6f, 0x3a, 0xeb, 0x2a, 0xac, 0x49, 0xac, 0x60, 0xa6, 0xa5, 0xd2, 0x89, 0x5d, 0x71, - 0x42, 0x1f, 0x28, 0x0b, 0x47, 0x17, 0x22, 0xca, 0x2a, 0xbd, 0x89, 0x32, 0x1a, 0x68, 0x33, 0x25, - 0x90, 0x9b, 0x2a, 0x6a, 0x1d, 0x5e, 0x43, 0x72, 0x91, 0x32, 0x43, 0x94, 0xce, 0x00, 0x0c, 0x93, - 0x3c, 0x2d, 0xd6, 0x5a, 0xcb, 0xd6, 0x12, 0x6a, 0xda, 0x47, 0x1f, 0x51, 0x90, 0x7a, 0x85, 0x30, - 0xd3, 0x58, 0xef, 0x95, 0x8d, 0xa5, 0xea, 0xb9, 0x59, 0xd4, 0x5d, 0xf9, 0x28, 0xdb, 0xeb, 0x0c, - 0x5a, 0x15, 0x7c, 0x69, 0x8f, 0xdd, 0x2e, 0x9a, 0xc3, 0x2e, 0x8d, 0x21, 0xa2, 0x8d, 0x69, 0xf5, - 0x8a, 0xe1, 0x14, 0x40, 0xab, 0xb2, 0x69, 0xa9, 0xa7, 0x5d, 0xd8, 0x9c, 0x3f, 0xf0, 0xe6, 0xce, - 0x59, 0x84, 0x57, 0xb8, 0xfa, 0xc1, 0x82, 0xf6, 0x61, 0x98, 0xa7, 0x99, 0x48, 0x8a, 0x9c, 0x36, - 0x35, 0x50, 0x94, 0xb6, 0x04, 0x96, 0x57, 0x97, 0xec, 0x40, 0x4d, 0x66, 0x5c, 0x1d, 0xee, 0x6a, - 0x21, 0x14, 0x5c, 0xa9, 0x84, 0xfb, 0xa6, 0x4a, 0xd0, 0x67, 0xd0, 0x38, 0x18, 0xf4, 0x1f, 0x26, - 0x71, 0x3e, 0x59, 0x1a, 0xb1, 0x79, 0xd2, 0xd9, 0x95, 0x27, 0x5d, 0x47, 0x3d, 0x4f, 0x54, 0x54, - 0xf8, 0x22, 0xe9, 0xa8, 0x17, 0x89, 0xab, 0x11, 0x3e, 0xa5, 0x03, 0xd8, 0x52, 0xe1, 0xca, 0x89, - 0x73, 0x99, 0xb1, 0x68, 0x5e, 0x11, 0x4e, 0xf9, 0x8a, 0x90, 0x46, 0xd5, 0xd4, 0xfd, 0x37, 0x8d, - 0xfe, 0x66, 0xc3, 0x16, 0x13, 0x69, 0xf0, 0x52, 0xf4, 0xa3, 0x34, 0x4b, 0x72, 0x5f, 0x4e, 0x1c, - 0xb9, 0xff, 0xb3, 0x78, 0xa8, 0x6b, 0xe1, 0x30, 0x25, 0xbc, 0xfd, 0x94, 0x10, 0x0a, 0xf5, 0xea, - 0x10, 0xa8, 0x2e, 0x30, 0x0a, 0x72, 0x07, 0xea, 0x83, 0x38, 0x4f, 0xfc, 0xa2, 0xf3, 0x71, 0x72, - 0x2b, 0xff, 0x4a, 0xc1, 0xcc, 0x02, 0xf2, 0x08, 0xc8, 0x69, 0xc2, 0xa3, 0x34, 0xe4, 0x92, 0x92, - 0xd9, 0xd6, 0x28, 0x9f, 0x27, 0x15, 0xed, 0x9c, 0x85, 0x25, 0xdb, 0xc8, 0x5e, 0xf5, 0x08, 0x7b, - 0x75, 0xe4, 0xb7, 0x61, 0xf8, 0xe9, 0x73, 0x52, 0x3d, 0xe4, 0xf7, 0x16, 0x3a, 0xd4, 0x5b, 0xc3, - 0x2d, 0x5b, 0x72, 0xcb, 0x9c, 0x82, 0xcd, 0xaf, 0xa3, 0xdf, 0x5a, 0xb0, 0x5e, 0x65, 0xb3, 0x62, - 0x5c, 0x14, 0xe5, 0xb3, 0x57, 0xbf, 0x76, 0x4c, 0xf9, 0xdc, 0x65, 0x2f, 0xcb, 0x5a, 0xf5, 0x05, - 0x14, 0xc3, 0x7f, 0xdf, 0x90, 0x9c, 0x4b, 0xd1, 0xe9, 0x42, 0xeb, 0x84, 0x27, 0x59, 0x20, 0x8d, - 0xe9, 0x7b, 0xba, 0xc6, 0xaa, 0x10, 0x15, 0x70, 0xed, 0xb5, 0x26, 0x3a, 0x8c, 0xc7, 0x13, 0xd9, - 0xad, 0x97, 0x6a, 0x26, 0x39, 0xa6, 0x93, 0x24, 0x4e, 0x4c, 0x06, 0x50, 0xa0, 0x07, 0xd0, 0x38, - 0x8d, 0x27, 0x71, 0x18, 0x3f, 0x9f, 0xad, 0x18, 0x19, 0x1e, 0xd4, 0xd5, 0xd5, 0xa0, 0x46, 0x54, - 0x93, 0x19, 0x91, 0x5e, 0x91, 0xfd, 0xee, 0xf3, 0xd0, 0xcf, 0x43, 0x9e, 0x09, 0x7c, 0x1f, 0x23, - 0xf8, 0x79, 0xcc, 0x47, 0x6a, 0x2a, 0xe8, 0xa3, 0x45, 0xbf, 0xd4, 0x0d, 0xc8, 0x31, 0x9c, 0xca, - 0x15, 0xb4, 0x8f, 0x80, 0xb9, 0x82, 0x94, 0x44, 0x3e, 0x80, 0x56, 0x65, 0xb5, 0x0e, 0x6b, 0xb3, - 0xe8, 0x53, 0x05, 0xb3, 0xea, 0x1a, 0xfa, 0x8b, 0x35, 0xb7, 0xe7, 0xb5, 0x3b, 0x57, 0xbb, 0xba, - 0x50, 0x49, 0x6a, 0x30, 0x2d, 0xc9, 0xd0, 0x8f, 0xa6, 0x7e, 0x98, 0xa7, 0x52, 0xa5, 0x2f, 0xdc, - 0x02, 0x90, 0xa1, 0xcb, 0x1f, 0x9e, 0x38, 0x37, 0x8f, 0x1b, 0x23, 0xca, 0x5f, 0xa3, 0x9e, 0xe0, - 0xa3, 0x30, 0x88, 0x04, 0xf6, 0x8b, 0xc3, 0x0a, 0x99, 0xdc, 0x51, 0x33, 0xd6, 0x34, 0xfa, 0xf6, - 0x02, 0x71, 0xd4, 0xa9, 0xc9, 0x9b, 0x52, 0x02, 0x9d, 0x45, 0x15, 0xdd, 0x06, 0xa2, 0x3a, 0x60, - 0x7f, 0x18, 0x27, 0xe6, 0xb6, 0xa5, 0x87, 0x66, 0xb8, 0xc8, 0xec, 0xaf, 0xba, 0xc4, 0xcb, 0xcc, - 0xda, 0xd5, 0xcc, 0x1e, 0x74, 0x7e, 0x7d, 0xb5, 0x63, 0xfd, 0xfe, 0x6a, 0xc7, 0xfa, 0xe3, 0xd5, - 0x8e, 0xf5, 0xd3, 0x9f, 0x3b, 0xff, 0x19, 0xae, 0xe1, 0xaf, 0xfc, 0x87, 0x7f, 0x05, 0x00, 0x00, - 0xff, 0xff, 0x6d, 0xf8, 0xe6, 0x6b, 0xed, 0x0f, 0x00, 0x00, + // 1639 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45, + 0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91, + 0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d, + 0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f, + 0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01, + 0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99, + 0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49, + 0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9, + 0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5, + 0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22, + 0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7, + 0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3, + 0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8, + 0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41, + 0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0, + 0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7, + 0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17, + 0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28, + 0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a, + 0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84, + 0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69, + 0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71, + 0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7, + 0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83, + 0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7, + 0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a, + 0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2, + 0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45, + 0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3, + 0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65, + 0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2, + 0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e, + 0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04, + 0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe, + 0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c, + 0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5, + 0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17, + 0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50, + 0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa, + 0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d, + 0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a, + 0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09, + 0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, + 0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, + 0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, + 0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a, + 0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7, + 0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98, + 0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8, + 0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f, + 0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4, + 0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9, + 0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba, + 0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34, + 0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6, + 0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, + 0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86, + 0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa, + 0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef, + 0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26, + 0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61, + 0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, + 0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45, + 0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0, + 0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba, + 0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23, + 0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50, + 0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e, + 0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3, + 0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30, + 0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23, + 0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8, + 0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc, + 0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b, + 0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61, + 0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54, + 0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15, + 0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11, + 0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28, + 0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0, + 0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9, + 0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54, + 0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d, + 0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa, + 0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31, + 0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b, + 0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3, + 0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46, + 0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11, + 0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89, + 0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30, + 0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1, + 0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a, + 0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76, + 0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72, + 0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3, + 0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50, + 0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc, + 0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf, + 0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79, + 0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, + 0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38, + 0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -4680,6 +4926,267 @@ func (m *ResizeNodeMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FieldOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Signed) > 0 { + dAtA31 := make([]byte, len(m.Signed)*10) + var j30 int + for _, num1 := range m.Signed { + num := uint64(num1) + for num >= 1<<7 { + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j30++ + } + dAtA31[j30] = uint8(num) + j30++ + } + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPrivate(dAtA, i, uint64(j30)) + i-- + dAtA[i] = 0x1a + } + if len(m.Values) > 0 { + dAtA33 := make([]byte, len(m.Values)*10) + var j32 int + for _, num := range m.Values { + for num >= 1<<7 { + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j32++ + } + dAtA33[j32] = uint8(num) + j32++ + } + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPrivate(dAtA, i, uint64(j32)) + i-- + dAtA[i] = 0x12 + } + if len(m.RecordIDs) > 0 { + dAtA35 := make([]byte, len(m.RecordIDs)*10) + var j34 int + for _, num := range m.RecordIDs { + for num >= 1<<7 { + dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j34++ + } + dAtA35[j34] = uint8(num) + j34++ + } + i -= j34 + copy(dAtA[i:], dAtA35[:j34]) + i = encodeVarintPrivate(dAtA, i, uint64(j34)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.FieldOps) > 0 { + for k := range m.FieldOps { + v := m.FieldOps[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x22 + } + } + if len(m.ClearFields) > 0 { + for iNdEx := len(m.ClearFields) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ClearFields[iNdEx]) + copy(dAtA[i:], m.ClearFields[iNdEx]) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClearFields[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.ClearRecordIDs) > 0 { + dAtA38 := make([]byte, len(m.ClearRecordIDs)*10) + var j37 int + for _, num := range m.ClearRecordIDs { + for num >= 1<<7 { + dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j37++ + } + dAtA38[j37] = uint8(num) + j37++ + } + i -= j37 + copy(dAtA[i:], dAtA38[:j37]) + i = encodeVarintPrivate(dAtA, i, uint64(j37)) + i-- + dAtA[i] = 0x12 + } + if len(m.OpType) > 0 { + i -= len(m.OpType) + copy(dAtA[i:], m.OpType) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.OpType))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperations) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperations) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperations) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *ShardedIngestRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardedIngestRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardedIngestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for k := range m.Ops { + v := m.Ops[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i = encodeVarintPrivate(dAtA, i, uint64(k)) + i-- + dAtA[i] = 0x8 + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { offset -= sovPrivate(v) base := offset @@ -5628,6 +6135,124 @@ func (m *ResizeNodeMessage) Size() (n int) { return n } +func (m *FieldOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.RecordIDs) > 0 { + l = 0 + for _, e := range m.RecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Values) > 0 { + l = 0 + for _, e := range m.Values { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Signed) > 0 { + l = 0 + for _, e := range m.Signed { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.OpType) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.ClearRecordIDs) > 0 { + l = 0 + for _, e := range m.ClearRecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.ClearFields) > 0 { + for _, s := range m.ClearFields { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + if len(m.FieldOps) > 0 { + for k, v := range m.FieldOps { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperations) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for _, e := range m.Ops { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardedIngestRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for k, v := range m.Ops { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + sovPrivate(uint64(k)) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPrivate(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -5709,10 +6334,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6149,10 +6771,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6235,10 +6854,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6423,10 +7039,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6629,10 +7242,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6759,10 +7369,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6909,7 +7516,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > postIndex { @@ -6926,10 +7533,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7063,10 +7667,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7149,10 +7750,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7290,10 +7888,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7463,10 +8058,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7581,10 +8173,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7718,10 +8307,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7891,10 +8477,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7979,10 +8562,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8154,10 +8734,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8291,10 +8868,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8501,10 +9075,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8619,10 +9190,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8728,10 +9296,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8888,10 +9453,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9027,10 +9589,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9208,10 +9767,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9396,10 +9952,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9552,10 +10105,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9702,10 +10252,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9852,10 +10399,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10137,10 +10681,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10342,10 +10883,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10483,10 +11021,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10624,10 +11159,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10742,10 +11274,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10796,10 +11325,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10850,10 +11376,7 @@ func (m *LoadSchemaMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10972,10 +11495,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11172,10 +11692,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11226,10 +11743,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11280,10 +11794,7 @@ func (m *ResizeAbortMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11398,10 +11909,857 @@ func (m *ResizeNodeMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FieldOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FieldOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FieldOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.RecordIDs) == 0 { + m.RecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RecordIDs", wireType) + } + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Values) == 0 { + m.Values = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType) + } + case 3: + if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Signed) == 0 { + m.Signed = make([]int64, 0, elementCount) + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Signed", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OpType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.OpType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.ClearRecordIDs) == 0 { + m.ClearRecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ClearRecordIDs", wireType) + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClearFields", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClearFields = append(m.ClearFields, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldOps", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FieldOps == nil { + m.FieldOps = make(map[string]*FieldOperation) + } + var mapkey string + var mapvalue *FieldOperation + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPrivate + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &FieldOperation{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.FieldOps[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperations) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperations: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperations: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ops = append(m.Ops, &ShardIngestOperation{}) + if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardedIngestRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardedIngestRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardedIngestRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Ops == nil { + m.Ops = make(map[uint64]*ShardIngestOperations) + } + var mapkey uint64 + var mapvalue *ShardIngestOperations + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &ShardIngestOperations{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Ops[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { diff --git a/pb/private.proto b/pb/private.proto index a5a43af24..9f15939c7 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -234,4 +234,25 @@ message ResizeAbortMessage { message ResizeNodeMessage { string NodeID = 1; string Action = 2; -} \ No newline at end of file +} + +message FieldOperation { + repeated uint64 RecordIDs = 1; + repeated uint64 Values = 2; + repeated int64 Signed = 3; +} + +message ShardIngestOperation { + string OpType = 1; + repeated uint64 ClearRecordIDs = 2; + repeated string ClearFields = 3; + map FieldOps = 4; +} + +message ShardIngestOperations { + repeated ShardIngestOperation Ops = 1; +} + +message ShardedIngestRequest { + map Ops = 1; +} diff --git a/pb/public.pb.go b/pb/public.pb.go index 43ebfc641..ca1b84852 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -5980,10 +5980,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6068,10 +6065,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6194,10 +6188,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6356,10 +6347,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6486,10 +6474,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6593,10 +6578,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6713,10 +6695,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6799,10 +6778,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7016,10 +6992,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7156,10 +7129,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7274,10 +7244,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7396,10 +7363,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7520,10 +7484,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7642,10 +7603,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7762,10 +7720,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7835,10 +7790,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8008,10 +7960,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8134,10 +8083,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8273,10 +8219,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8365,10 +8308,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8620,10 +8560,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8740,10 +8677,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9356,10 +9290,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9843,10 +9774,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10308,10 +10236,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10481,10 +10406,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10567,10 +10489,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10737,10 +10656,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10867,10 +10783,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11061,10 +10974,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11147,10 +11057,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11267,10 +11174,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11484,10 +11388,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11604,10 +11505,7 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { diff --git a/translate.go b/translate.go index 738fc1686..f449d75be 100644 --- a/translate.go +++ b/translate.go @@ -23,6 +23,7 @@ import ( "sort" "sync" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/topology" "github.com/pkg/errors" ) @@ -98,6 +99,39 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul ReadFrom(io.Reader) (int64, error) } +// This implements ingest's key translator interface, which differs +// slightly because we want to be able to do fast lookups on arbitrary +// IDs which are not necessarily contiguous small values, so the []string +// from TranslateIDs isn't a good fit. +type ingestKeyTranslator struct { + store TranslateStore +} + +var _ ingest.KeyTranslator = &ingestKeyTranslator{} + +func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.store.CreateKeys(keys...) +} + +func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.store.TranslateIDs(ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator { + return &ingestKeyTranslator{store: s} +} + // TranslatorSummary is returned, for example from the boltdb string key translators, // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. type TranslatorSummary struct {