scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Drop the ingest subpackage and related endpoints.

Browse source 
The internal/ingest and internal/schema endpoints were developed with
intent that they'd be the primary interface new users would work with,
because they were Easy To Use, and did not require any kind of setup,
the counterpoint being that ingest done this way had performance issues
because it ended up with huge amounts of JSON parsing to reformat
things into our native format. But this was understood to be the price
of providing a new-user-friendly JSON ingest experience.

A year later, we have no evidence that it's ever been used. We never
even moved it out of the `/internal` path. It's a lot of very complex
fiddly code and we don't seem to be using it, and at this point, our
anticipation is that if we really need something, we'll use CSV, which
we already have working, or something in the new SQL code. Either way,
we don't seem to be using this.
This commit is contained in:
Seebs 2022-10-31 10:24:23 -05:00 • committed by seebs
parent 4104577f6d
commit 16ccbc461a
32 changed files with 163 additions and 6300 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

408
api.go
View file

@ -22,7 +22,6 @@ import (
"time"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/rbf"
//"github.com/molecula/featurebase/v3/pg"
@ -1122,164 +1121,6 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error {
return nil
}
// applyOneIngestSchema applies a single ingestSpec, which specifies operations on
// a single index and possibly fields. If it is successful, it returns the name
// of the index and an empty slice (if it created the index), or the name of the
// index and a slice of the fields within that index that it created. If it
// is unsuccessful, it tries to delete whatever it created.
//
// The intended idiom is that if the returned list of fields isn't empty, the index
// already existed and only those fields need to be cleaned up in the event of
// a later error, but if the list of fields is empty, the entire index was new,
// and should be cleaned up, in which case there's no need to track or delete
// the specific fields separately.
func (api *API) ApplyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) {
if api.PrimaryNode().ID != api.NodeID() {
return nil, nil, RedirectError{
HostPort: api.PrimaryNode().URI.Normalize(),
error: "request made to non-primary node",
}
}
// create index
indexName := schema.IndexName
var createdFields []string
var useKeys bool
switch schema.PrimaryKeyType {
case "string":
useKeys = true
case "uint":
useKeys = false
default:
return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType)
}
opts := IndexOptions{
Keys: useKeys,
TrackExistence: true,
}
createdIndex := false
// We check this up here because, if there's at least one field but we don't know what to do with
// it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in
// trying to create it. We don't care about this if there's no fields specified.
if len(schema.Fields) > 0 {
switch schema.FieldAction {
case "create", "ensure", "require":
// do nothing
case "":
schema.FieldAction = schema.IndexAction
default:
return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction)
}
}
switch schema.IndexAction {
case "ensure", "require":
index, err = api.Index(ctx, indexName)
if err != nil {
if _, ok := err.(NotFoundError); !ok {
return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err)
} else {
err = nil
}
}
if index != nil {
existingOpts := index.Options()
if existingOpts != opts {
return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts)
}
break
}
if schema.IndexAction == "require" {
return nil, nil, fmt.Errorf("index %q does not exist", indexName)
}
fallthrough
case "create":
index, err = api.CreateIndex(ctx, indexName, opts)
if err != nil {
return nil, nil, err
}
createdIndex = true
default:
return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction)
}
// Now we might have an index, so we need our cleanup code.
defer func() {
if err == nil {
return
}
if createdIndex {
err := api.DeleteIndex(ctx, indexName)
if err != nil {
api.server.logger.Printf("trying to undo failed index %q creation: %v", indexName, err)
}
return
}
for _, field := range createdFields {
err := api.DeleteField(ctx, indexName, field)
if err != nil {
api.server.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err)
}
}
}()
// create all the fields specified in the index
for _, fSpec := range schema.Fields {
fieldName := fSpec.FieldName
opt := fieldSpecToFieldOption(fSpec)
err = opt.validate()
if err != nil {
return nil, nil, err
}
switch schema.FieldAction {
case "ensure", "require":
field, schemaErr := api.Field(ctx, indexName, fieldName)
if schemaErr != nil {
// NotFoundError is fine
if _, ok := schemaErr.(NotFoundError); !ok {
return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err)
}
}
if field != nil {
existing := field.Options()
if opt.Type != existing.Type {
return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type)
}
if ((opt.Keys != nil) && *opt.Keys) != existing.Keys {
if existing.Keys {
return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName)
} else {
return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName)
}
}
// TODO: verify compatibility of other field opts, this is sorta hard
break
}
if schema.FieldAction == "require" {
return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName)
}
fallthrough
case "create":
fos := fieldOptionsToFunctionalOpts(opt)
_, err = api.CreateField(ctx, indexName, fieldName, fos...)
if err != nil {
return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err)
}
createdFields = append(createdFields, fieldName)
}
}
// we don't report the fields back, so we can distinguish "created index"
// from "created fields within index"
if createdIndex {
createdFields = nil
}
return index, createdFields, nil
}
// Views returns the views in the given field.
func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Views")
@ -1936,251 +1777,6 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
return nil
}
// ingestNodeOperationsForFields does the actual work of applying operations
// to a given index with a map of known fields and an already-parsed
// ShardedRequest. This is used locally on the node that first receives
// the request, after it does the parsing, and on other nodes because the
// format they get is already that rather than JSON, so it's the common
// path *after* key translation and sorting into shards.
func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error {
eg, ctx := errgroup.WithContext(ctx)
for shard, ops := range req.Ops {
// create new local copies of these values so the goroutine uses these
// copies, and doesn't read the actual loop variables, which are being
// changed by the loop.
shard, ops := shard, ops
eg.Go(func() error {
return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)
})
}
return eg.Wait()
}
// IngestNodeOperations handles protobuf-formatted data which does not need
// key translation and is applicable to this specific node.
func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error {
index := api.holder.Index(indexName)
if index == nil {
api.server.logger.Errorf("ingest: no such index %q", indexName)
return newNotFoundError(ErrIndexNotFound, indexName)
}
fields := index.Fields()
knownFields := map[string]*Field{}
for _, field := range fields {
knownFields[field.name] = field
}
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req)
}
// IngestOperations handles JSON-formatted data which may need key translation
// and may be for any or all nodes.
func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations")
defer span.Finish()
if api.PrimaryNode().ID != api.NodeID() {
return RedirectError{
HostPort: api.PrimaryNode().URI.Normalize(),
error: "request made to non-primary node",
}
}
if err := api.validate(apiIngestOperations); err != nil {
return errors.Wrap(err, "validating api method")
}
// Find the Index.
index := api.holder.Index(indexName)
if index == nil {
api.server.logger.Errorf("ingest: no such index %q", indexName)
return newNotFoundError(ErrIndexNotFound, indexName)
}
fields := index.Fields()
var indexKeys ingest.KeyTranslator
if index.Keys() {
indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName)
}
codec, err := ingest.NewJSONCodec(indexKeys)
if err != nil {
return errors.Wrap(err, "creating JSON codec")
}
knownFields := map[string]*Field{}
for _, field := range fields {
var keys ingest.KeyTranslator
if field.usesKeys {
keys = newIngestKeyTranslatorFromStore(field.translateStore)
}
knownFields[field.name] = field
switch field.Type() {
case "set":
if err = codec.AddSetField(field.name, keys); err != nil {
return fmt.Errorf("adding set field to codec: %w", err)
}
case "time":
if err = codec.AddTimeQuantumField(field.name, keys); err != nil {
return fmt.Errorf("adding time quantum field to codec: %w", err)
}
case "mutex":
if err = codec.AddMutexField(field.name, keys); err != nil {
return fmt.Errorf("adding mutex field to codec: %w", err)
}
case "bool":
if err = codec.AddBoolField(field.name); err != nil {
return fmt.Errorf("adding bool field to codec: %w", err)
}
case "int":
if err = codec.AddIntField(field.name, keys); err != nil {
return fmt.Errorf("adding int field to codec: %w", err)
}
case "decimal":
if err = codec.AddDecimalField(field.name, field.options.Scale); err != nil {
return fmt.Errorf("adding decimal field to codec: %w", err)
}
case "timestamp":
if err = codec.AddTimestampField(field.name, field.options.TimeUnit, field.options.Base); err != nil {
return fmt.Errorf("adding timestamp field to codec: %w", err)
}
default:
return fmt.Errorf("unhandled field type %q", field.Type())
}
}
req, err := codec.Parse(stream)
if err != nil {
return errors.Wrap(err, "parsing input data")
}
sharded, err := codec.RequestByShard(req)
if err != nil {
return errors.Wrap(err, "sharding input data")
}
// now that we have this, let's assign the shards to nodes
snap := api.cluster.NewSnapshot()
// oh hey an easy case: we're presumably the only node
if len(snap.Nodes) == 1 {
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
}
// Created new ShardedRequest objects for every node, giving each of them
// all the shards that apply to them.
byNode := make(map[string]*ingest.ShardedRequest)
for shard, ops := range sharded.Ops {
nodes := snap.ShardNodes(indexName, shard)
for _, node := range nodes {
forThisShard := byNode[node.ID]
if forThisShard == nil {
// Create new ShardedRequest for the target node, with its op map
// mapping this shard to the ops for this shard.
byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}}
continue
}
// Add this shard to the existing ShardedRequest's Ops map. Note that
// we don't have to worry about overwrites; we can't have seen this
// shard before, because we're in a range loop on a map where the shard
// is the key.
forThisShard.Ops[shard] = ops
}
}
eg, ctx := errgroup.WithContext(ctx)
for _, node := range snap.Nodes {
node := node
sharded := byNode[node.ID]
// Sometimes, there's nothing for a specific node.
if sharded == nil {
continue
}
if node.ID == api.NodeID() {
eg.Go(func() error {
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
})
} else {
eg.Go(func() error {
return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded)
})
}
}
return eg.Wait()
}
// applyOperations applies a set of operations to one specific shard.
func (api *API) applyOperations(ctx context.Context, qcx *Qcx, index *Index, shard uint64, fields map[string]*Field, ops []*ingest.Operation) error {
// For each operation, we may have a set of records/fields to clear, and then
// also a set of fields to set/remove specific bits in.
opts := &ImportOptions{Presorted: true, IgnoreKeyCheck: true, fullySorted: true}
for _, op := range ops {
// ClearRecordIDs should exist only for delete, clear, and write. For clear and write,
// we'll have a list of fields, for delete, it should be all the fields.
if len(op.ClearRecordIDs) > 0 {
// anonymous func lets us defer a finisher from any of the inner error returns
err := func() (e0 error) {
// WARNING: Depends on GetTx being per-shard/index, not per-field.
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
if err != nil {
return fmt.Errorf("getting Tx: %w", err)
}
defer finisher(&e0)
// For a delete, we don't look at the fields the codec was defined with,
// We delete from the existence field unconditionally and other fields
// if we know they exist.
if op.OpType == ingest.OpDelete {
err = clearExistenceColumns(tx, index, op.ClearRecordIDs, shard)
if err != nil {
return fmt.Errorf("clearing existence columns: %w", err)
}
for name, field := range fields {
if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil {
return fmt.Errorf("clearing field %q: %w", name, err)
}
}
return nil
}
// clear things that we need to wipe out, whether it's because
// this is a Clear op, or because it's a write op that
// specifies clears for the fields it's going to write to.
if len(op.ClearFields) > 0 {
for _, fieldName := range op.ClearFields {
field, ok := fields[fieldName]
if !ok {
return fmt.Errorf("can't find a field named %q", fieldName)
}
if err = field.ClearBits(tx, shard, op.ClearRecordIDs...); err != nil {
return fmt.Errorf("clearing record IDs: %w", err)
}
}
}
return nil
}()
if err != nil {
return err
}
}
opts.Clear = (op.OpType == ingest.OpRemove)
// for "set" and "write" ops, we'll be setting bits, for
// "remove" ops we'll be clearing them, and for "clear" ops
// there shouldn't be anything here.
for fieldName, fieldOp := range op.FieldOps {
field, ok := fields[fieldName]
if !ok {
return fmt.Errorf("can't find a field named %q", fieldName)
}
var err error
err = importExistenceColumns(qcx, index, fieldOp.RecordIDs, shard)
if err != nil {
return errors.Wrap(err, "importing existence columns")
}
switch field.Type() {
case "set", "time", "mutex", "bool":
err = field.Import(qcx, fieldOp.Values, fieldOp.RecordIDs, fieldOp.Signed, shard, opts)
case "int", "timestamp", "decimal":
err = field.importValue(qcx, fieldOp.RecordIDs, fieldOp.Signed, shard, opts)
default:
err = fmt.Errorf("unhandled field type %q", field.Type())
}
if err != nil {
return err
}
}
}
return nil
}
func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
ef := index.existenceField()
if ef == nil {
@ -3287,8 +2883,6 @@ const (
apiIDCommit
apiIDReset
apiPartitionNodes
apiIngestOperations
apiIngestNodeOperations
apiMutexCheck
)
@ -3349,8 +2943,6 @@ var methodsNormal = map[apiMethod]struct{}{
apiIDCommit: {},
apiIDReset: {},
apiPartitionNodes: {},
apiIngestOperations: {},
apiIngestNodeOperations: {},
apiMutexCheck: {},
}

115
api_test.go
View file

@ -521,72 +521,6 @@ func TestAPI_Ingest(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
t.Run("IngestAPI", func(t *testing.T) {
sampleJson := []byte(`
[
{
"action": "set",
"records": {
"2": {
"set": [2],
"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] }
},
"5": { "set": [3] },
"8": { "set": [3] },
"1": {
"set": [2],
"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] }
},
"4": { "set": [3, 7] }
}
},
{
"action": "clear",
"record_ids": [ 5, 6, 7 ],
"fields": [ "tq", "set" ]
},
{
"action": "write",
"records": {
"8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } },
"9": { "set": [7, 3] }
}
},
{
"action": "delete",
"record_ids": [ 9 ]
}
]
`)
// just for set row 3:
// first operation should set it for 4, 5, and 8.
// clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8.
// the write operation should clear set for record 8, even though record 8 doesn't
// contain that field in that op, because set is present in record 9, which also
// gets row 3 set. but then we delete 9.
// so after all that we expect Row(set=3) to be 4...
sampleBuf := bytes.NewBuffer(sampleJson)
qcx := coord.API.Txf().NewQcx()
defer func() {
if err := qcx.Finish(); err != nil {
t.Fatalf("finishing qcx: %v", err)
}
}()
err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf)
if err != nil {
t.Fatalf("importing data: %v", err)
}
query := "Row(set=3)"
res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
if err != nil {
t.Errorf("query: %v", err)
}
r := res.Results[0].(*pilosa.Row).Columns()
if len(r) != 1 || r[0] != 4 {
t.Fatalf("expected row with 4 set, got %d", r)
}
})
t.Run("ImportRoaringShard", func(t *testing.T) {
setBuf := &bytes.Buffer{}
setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7)
@ -697,55 +631,6 @@ func ingestBenchmarkHelper() []byte {
return data
}
func BenchmarkIngest(b *testing.B) {
b.StopTimer()
data := ingestBenchmarkHelper()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(b, 1)
defer c.Close()
coord := c.GetPrimary()
m0 := c.GetNode(0)
// m1 := c.GetNode(1)
// m2 := c.GetNode(2)
index := c.Idx()
setField := "set"
intField := "int"
tqField := "tq"
_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
if err != nil {
b.Fatalf("creating index: %v", err)
}
_, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0))
if err != nil {
b.Fatalf("creating field: %v", err)
}
_, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840))
if err != nil {
b.Fatalf("creating field: %v", err)
}
_, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0"))
if err != nil {
b.Fatalf("creating field: %v", err)
}
b.ReportAllocs()
b.StartTimer()
for i := 0; i < b.N; i++ {
qcx := m0.API.Txf().NewQcx()
defer qcx.Abort()
err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data))
if err != nil {
b.Fatalf("ingest: %v", err)
}
err = qcx.Finish()
if err != nil {
b.Fatalf("finish: %v", err)
}
}
}
func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()

8
apimethod_string.go
View file

@ -42,14 +42,12 @@ func _() {
_ = x[apiIDCommit-31]
_ = x[apiIDReset-32]
_ = x[apiPartitionNodes-33]
_ = x[apiIngestOperations-34]
_ = x[apiIngestNodeOperations-35]
_ = x[apiMutexCheck-36]
_ = x[apiMutexCheck-34]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 499, 522, 535}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

28
client/client.go
View file

@ -790,34 +790,6 @@ func (c *Client) readSchema() ([]SchemaIndex, error) {
return schemaInfo.Indexes, nil
}
func (c *Client) IngestSchema(reqBody map[string]interface{}) (body []byte, err error) {
data, err := json.Marshal(reqBody)
if err != nil {
return data, errors.Wrap(err, "error building Schema body to Ingest")
}
return c.IngestRequest("/internal/schema", data)
}
func (c *Client) IngestData(index string, reqBody []map[string]interface{}) (body []byte, err error) {
data, err := json.Marshal(reqBody)
if err != nil {
return data, errors.Wrap(err, "error building request body to Ingest")
}
return c.IngestRequest("/internal/ingest/"+index, data)
}
func (c *Client) IngestRequest(uri string, data []byte) (body []byte, err error) {
var header = make(map[string]string)
header["Content-Type"] = "application/json"
header["Accept"] = "application/json"
header["User-Agent"] = "pilosa/" + pilosa.Version
status, body, err := c.HTTPRequest("POST", uri, data, header)
if err != nil {
return nil, errors.Wrapf(err, "requesting %s status: %d", uri, status)
}
return body, err
}
func (c *Client) shardsMax() (map[string]uint64, error) {
_, data, err := c.HTTPRequest("GET", "/internal/shards/max", nil, nil)
if err != nil {

33
cluster.go
View file

@ -8,7 +8,6 @@ import (
"time"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
@ -504,38 +503,6 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys
return ids, nil
}
// This implements ingest's key translator interface on a cluster/index pair.
type clusterKeyTranslator struct {
ctx context.Context // we're created within a request context and need to pass that to cluster ops
c *cluster
indexName string
}
var _ ingest.KeyTranslator = &clusterKeyTranslator{}
func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
return i.c.createIndexKeys(i.ctx, i.indexName, keys...)
}
func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids)
if err != nil {
return nil, err
}
if len(keys) != len(ids) {
return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
}
out := make(map[uint64]string, len(keys))
for i, id := range ids {
out[id] = keys[i]
}
return out, nil
}
func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator {
return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName}
}
// TODO: remove this when it is no longer used
func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) {
keys := make([]string, 0, len(keySet))

112
encoding/proto/proto.go
View file

@ -9,7 +9,6 @@ import (
"github.com/gogo/protobuf/proto"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
@ -294,19 +293,6 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
*mt = s.decodeRowMatrix(msg)
return nil
case *ingest.ShardedRequest:
msg := &pb.ShardedIngestRequest{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshalling ShardedRequest")
}
req, err := s.decodeShardedIngestRequest(msg)
if err != nil {
return err
}
*mt = *req
return nil
default:
panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
}
@ -376,8 +362,6 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeTransactionMessage(mt)
case *pilosa.AtomicRecord:
return s.encodeAtomicRecord(mt)
case *ingest.ShardedRequest:
return s.encodeShardedIngestRequest(mt)
}
return nil
}
@ -900,48 +884,6 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr
return &pb.TransactionStats{}
}
func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest {
if req == nil || len(req.Ops) == 0 {
return &pb.ShardedIngestRequest{}
}
out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))}
for shard, ops := range req.Ops {
out.Ops[shard] = s.encodeShardIngestOperations(ops)
}
return out
}
func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations {
out := &pb.ShardIngestOperations{}
for _, op := range ops {
if op == nil {
continue
}
out.Ops = append(out.Ops, s.encodeShardIngestOperation(op))
}
return out
}
func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation {
out := &pb.ShardIngestOperation{
OpType: op.OpType.String(),
ClearRecordIDs: op.ClearRecordIDs,
ClearFields: op.ClearFields,
FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)),
}
for k, v := range op.FieldOps {
if v == nil {
continue
}
out.FieldOps[k] = &pb.FieldOperation{
RecordIDs: v.RecordIDs,
Values: v.Values,
Signed: v.Signed,
}
}
return out
}
func (s Serializer) decodeSchema(sc *pb.Schema, m *pilosa.Schema) {
m.Indexes = make([]*pilosa.IndexInfo, len(sc.Indexes))
s.decodeIndexes(sc.Indexes, m.Indexes)
@ -1844,57 +1786,3 @@ func (s Serializer) encodeDecimal(p *pql.Decimal) *pb.Decimal {
}
return retval
}
func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) {
if req == nil || len(req.Ops) == 0 {
return &ingest.ShardedRequest{}, nil
}
out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))}
for shard, ops := range req.Ops {
var err error
out.Ops[shard], err = s.decodeShardIngestOperations(ops)
if err != nil {
return nil, err
}
}
return out, nil
}
func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) {
out := []*ingest.Operation{}
if len(ops.Ops) == 0 {
return out, nil
}
for _, op := range ops.Ops {
if op == nil {
continue
}
decoded, err := s.decodeShardIngestOperation(op)
if err != nil {
return nil, err
}
out = append(out, decoded)
}
return out, nil
}
func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) {
opType, err := ingest.ParseOpType(op.OpType)
if err != nil {
return nil, err
}
out := &ingest.Operation{
OpType: opType,
ClearRecordIDs: op.ClearRecordIDs,
ClearFields: op.ClearFields,
FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)),
}
for k, v := range op.FieldOps {
out.FieldOps[k] = &ingest.FieldOperation{
RecordIDs: v.RecordIDs,
Values: v.Values,
Signed: v.Signed,
}
}
return out, nil
}

97
encoding/proto/proto_test.go
View file

@ -2,12 +2,10 @@
package proto
import (
"errors"
"reflect"
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/pb"
)
@ -40,99 +38,8 @@ func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, exp
t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error())
}
}
switch real := obj.(type) {
case *ingest.ShardedRequest:
real2 := obj2.(*ingest.ShardedRequest)
err := real.Compare(real2)
if err != nil {
if expectedMismatchErr == nil {
t.Fatalf("unexpected compare error %q", err.Error())
}
if err.Error() != expectedMismatchErr.Error() {
t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error())
}
} else {
if expectedMismatchErr != nil {
t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error())
}
}
default:
if !reflect.DeepEqual(obj, obj2) {
t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2)
}
}
}
type shardedIngestRequestTest struct {
req *ingest.ShardedRequest
err error
}
var shardedIngestRequestTestcases = []shardedIngestRequestTest{
{
req: &ingest.ShardedRequest{
Ops: map[uint64][]*ingest.Operation{
1: {
{
OpType: ingest.OpWrite,
ClearFields: []string{"clearField", "clearField2"},
ClearRecordIDs: []uint64{1, 7, 9},
FieldOps: map[string]*ingest.FieldOperation{
"writeAll": {
RecordIDs: []uint64{3, 6, 8},
Values: []uint64{0, 17, 34},
Signed: []int64{-9, 23, 17},
},
"writeValues": {
RecordIDs: []uint64{3, 6, 8},
Values: []uint64{0, 17, 34},
},
"writeSigned": {
RecordIDs: []uint64{3, 6, 8},
Signed: []int64{-9, 23, 17},
},
},
},
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
"foo": nil,
},
},
},
2: {},
3: nil,
},
},
},
{
req: &ingest.ShardedRequest{
Ops: map[uint64][]*ingest.Operation{
1: {
{
OpType: ingest.OpWrite,
FieldOps: map[string]*ingest.FieldOperation{
"writeAll": {},
},
},
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
"foo": nil,
},
},
nil,
},
},
},
err: errors.New("shard 1: expected 3 ops, got 2"),
},
}
func TestIngestRoundTrip(t *testing.T) {
for _, tc := range shardedIngestRequestTestcases {
t.Logf("next case")
testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err)
if !reflect.DeepEqual(obj, obj2) {
t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2)
}
}

1
go.mod
View file

@ -12,7 +12,6 @@ require (
github.com/aws/aws-sdk-go v1.42.39
github.com/beevik/ntp v0.3.0
github.com/benbjohnson/immutable v0.3.0
github.com/buger/jsonparser v1.1.1
github.com/cespare/xxhash v1.1.0
github.com/chzyer/readline v1.5.0
github.com/confluentinc/confluent-kafka-go v1.9.1

2
go.sum
View file

@ -144,8 +144,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=

118
handler.go
View file

@ -3,9 +3,10 @@ package pilosa
import (
"encoding/json"
"math/bits"
"time"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
)
@ -156,7 +157,6 @@ func (ivr *ImportValueRequest) Clone() *ImportValueRequest {
// The top level Shard has to agree with Ivr[i].Shard and the Iv[i].Shard
// for all i included (in Ivr and Ir). The same goes for the top level Index: all records
// have to be writes to the same Index. These requirements are checked.
//
type AtomicRecord struct {
Index string
Shard uint64
@ -299,28 +299,108 @@ func (ir *ImportRequest) Clone() *ImportRequest {
// requests. We don't sort the entries within each shard because the correct
// sorting depends on the field type and we don't want to deal with that
// here.
func (ir *ImportRequest) SortToShards() map[uint64]*ImportRequest {
// cheat: use ingest
fo := ingest.FieldOperation{
RecordIDs: ir.ColumnIDs,
Values: ir.RowIDs,
Signed: ir.Timestamps,
func (ir *ImportRequest) SortToShards() (result map[uint64]*ImportRequest) {
if len(ir.ColumnIDs) == 0 {
return nil
}
sharded := fo.SortToShards()
output := make(map[uint64]*ImportRequest, len(sharded))
for shard, shardOp := range sharded {
shardReq := *ir
shardReq.ColumnKeys = nil
shardReq.RowKeys = nil
shardReq.Shard = shard
shardReq.ColumnIDs = shardOp.RecordIDs
shardReq.RowIDs = shardOp.Values
shardReq.Timestamps = shardOp.Signed
output[shard] = &shardReq
diffMask := uint64(0)
prev := ir.ColumnIDs[0]
for _, r := range ir.ColumnIDs[1:] {
diffMask |= r ^ prev
prev = r
}
bitsRemaining := bits.Len64(diffMask)
if bitsRemaining <= shardwidth.Exponent {
shard := ir.ColumnIDs[0] >> shardwidth.Exponent
ir.Shard = shard
ir.ColumnKeys = nil
ir.RowKeys = nil
return map[uint64]*ImportRequest{shard: ir}
}
output := make(map[uint64]*ImportRequest)
sortToShardsInto(ir, bitsRemaining-8, output)
return output
}
// sortToShardsInto puts the shards it finds into the given map, so that
// as we split off buckets, they can be inserted into the same map.
func sortToShardsInto(ir *ImportRequest, shift int, into map[uint64]*ImportRequest) {
if shift < shardwidth.Exponent {
shift = shardwidth.Exponent
}
nextShift := shift - 8
if nextShift < shardwidth.Exponent {
nextShift = shardwidth.Exponent
}
// count things that belong in each of the 256 buckets
var buckets [256]int
var starts [256]int
// compute the buckets ourselves
for _, r := range ir.ColumnIDs {
b := (r >> shift) & 0xFF
buckets[b]++
}
total := 0
// compute starting points of each bucket, converting the
// bucket counts into ends
for i := range buckets {
starts[i] = total
total += buckets[i]
buckets[i] = total
}
// starts[n] is the index of the first thing that should
// go in that bucket, buckets[n] is the index of the first
// thing that shouldn't
var bucketOp ImportRequest = *ir
bucketOp.ColumnKeys = nil
bucketOp.RowKeys = nil
origStarts := make([]int, len(starts))
copy(origStarts, starts[:])
for bucket, start := range origStarts {
end := buckets[bucket]
if end <= start {
continue
}
for j := start; j < end; j++ {
want := int((ir.ColumnIDs[j] >> shift) & 0xFF)
for want != bucket {
// move this to the beginning of the
// bucket it wants to be in, swapping
// the thing there here
dst := starts[want]
ir.ColumnIDs[j], ir.ColumnIDs[dst] = ir.ColumnIDs[dst], ir.ColumnIDs[j]
if ir.RowIDs != nil {
ir.RowIDs[j], ir.RowIDs[dst] = ir.RowIDs[dst], ir.RowIDs[j]
}
if ir.Timestamps != nil {
ir.Timestamps[j], ir.Timestamps[dst] = ir.Timestamps[dst], ir.Timestamps[j]
}
starts[want]++
want = int((ir.ColumnIDs[j] >> shift) & 0xFF)
}
}
// If shift == shardwidth.Exponent, then this is a completed
// shard and can go into the sharded output. otherwise, we
// can subdivide it.
bucketOp.ColumnIDs = ir.ColumnIDs[start:end]
if ir.RowIDs != nil {
bucketOp.RowIDs = ir.RowIDs[start:end]
}
if ir.Timestamps != nil {
bucketOp.Timestamps = ir.Timestamps[start:end]
}
if shift == shardwidth.Exponent {
x := bucketOp
shard := ir.ColumnIDs[start] >> shardwidth.Exponent
x.Shard = shard
into[shard] = &x
} else {
sortToShardsInto(&bucketOp, nextShift, into)
}
}
}
// ValidateWithTimestamp ensures that the payload of the request is valid.
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error {
if (ir.IndexCreatedAt != 0 && ir.IndexCreatedAt != indexCreatedAt) ||

28
handler_test.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package pilosa_test
import (
"math/rand"
"testing"
pilosa "github.com/molecula/featurebase/v3"
)
func TestSortToShards(t *testing.T) {
var ir pilosa.ImportRequest
expected := make(map[uint64][]uint64)
const n = 50
rng := rand.New(rand.NewSource(3))
for i := 0; i < n; i++ {
x := uint64(rng.Intn(8 * pilosa.ShardWidth))
shard := x / pilosa.ShardWidth
expected[shard] = append(expected[shard], x)
ir.ColumnIDs = append(ir.ColumnIDs, x)
}
out := ir.SortToShards()
for shard, values := range out {
if len(values.ColumnIDs) != len(expected[shard]) {
t.Fatalf("shard %d: expected values %d, got values %d", shard, expected[shard], values.ColumnIDs)
}
}
}

157
http_handler.go
View file

@ -33,7 +33,6 @@ import (
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/authz"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/monitor"
"github.com/molecula/featurebase/v3/pql"
@ -584,10 +583,7 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/index/{index}/shards", handler.chkAuthZ(handler.handleGetIndexAvailableShards, authz.Read)).Methods("GET").Name("GetIndexAvailableShards")
router.HandleFunc("/internal/nodes", handler.chkAuthN(handler.handleGetNodes)).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.chkAuthN(handler.handleGetShardsMax)).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/ingest/{index}", handler.chkAuthZ(handler.handlePostIngestData, authz.Write)).Methods("POST").Name("PostIngestData")
router.HandleFunc("/internal/ingest/{index}/node", handler.chkAuthZ(handler.handlePostIngestNode, authz.Write)).Methods("POST").Name("PostIngestNode")
router.HandleFunc("/internal/schema", handler.chkAuthZ(handler.handleIngestSchema, authz.Admin)).Methods("POST").Name("PostIngestSchema")
router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.chkAuthZ(handler.handleFindIndexKeys, authz.Admin)).Methods("POST").Name("FindIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.chkAuthZ(handler.handleCreateIndexKeys, authz.Admin)).Methods("POST").Name("CreateIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.chkAuthZ(handler.handlePostTranslateIndexDB, authz.Admin)).Methods("POST").Name("PostTranslateIndexDB")
@ -2087,40 +2083,6 @@ func (h *Handler) handlePatchField(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
}
// handlePostIngestData handles JSON ingest data that may need key
// translation, for the entire cluster.
func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
qcx := h.api.Txf().NewQcx()
err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body)
if err != nil {
qcx.Abort()
switch e := err.(type) {
case RedirectError:
http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect)
return
}
}
err = qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
return
}
resp := successResponse{h: h, Name: indexName}
resp.write(w, err)
}
type ingestSpec struct {
IndexName string `json:"index-name"`
IndexAction string `json:"index-action"`
@ -2182,79 +2144,6 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions {
return opt
}
func (h *Handler) handleIngestSchema(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
resp := successResponse{h: h}
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
schema := ingestSpec{}
// if a key in cleanupIndexes points to a 0-length slice, the
// entire index should be cleaned; otherwise, only the named
// fields within that index should be cleaned.
cleanupIndexes := map[string][]string{}
var schemaErr error
defer func() {
// we set schemaErr in any case where we need to do cleanup
if schemaErr != nil {
for index, fields := range cleanupIndexes {
if len(fields) == 0 {
err := h.api.DeleteIndex(r.Context(), index)
if err != nil {
h.logger.Printf("deleting index %q after schema err: %v", index, err)
}
} else {
for _, field := range fields {
err := h.api.DeleteField(r.Context(), index, field)
if err != nil {
h.logger.Printf("deleting field %q from index %q after schema err: %v", field, index, err)
}
}
}
}
}
}()
for dec.More() {
err := dec.Decode(&schema)
if err != nil {
resp.write(w, err)
return
}
index, fields, err := h.api.ApplyOneIngestSchema(r.Context(), &schema)
if err != nil {
switch e := err.(type) {
case RedirectError:
http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect)
return
default:
// if a previous schema created things, clean them up...
schemaErr = err
resp.write(w, err)
return
}
}
// we only have one slot to report these, sorry.
resp.Name = index.Name()
resp.CreatedAt = index.CreatedAt()
cleanupIndexes[index.Name()] = fields
}
// if we got here, we have a cleanupIndexes which we want to return,
// so we want to do that *instead* of the successResponse we'd be
// using otherwise (ironically, to indicate an error)
var mapBody []byte
var err error
if mapBody, err = json.Marshal(cleanupIndexes); err != nil {
resp.write(w, err)
}
if _, err = w.Write(mapBody); err != nil {
h.logger.Printf("error trying to write response: %v", err)
}
}
type postFieldRequest struct {
Options fieldOptions `json:"options"`
}
@ -3600,52 +3489,6 @@ func (h *Handler) handlePostShardImportRoaring(w http.ResponseWriter, r *http.Re
}
}
// handlePostIngestNode is the internal endpoint taking already-translated
// ingest operations, sorted by shard, for a single node.
func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
ctx := r.Context()
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body")
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
req := &ingest.ShardedRequest{}
span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal")
err = h.serializer.Unmarshal(body, req)
span.Finish()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
urlVars := mux.Vars(r)
indexName := urlVars["index"]
qcx := h.api.Txf().NewQcx()
err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req)
if err == nil {
err = qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
}
} else {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
qcx.Abort()
}
}
func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {

199
http_handler_test.go
View file

@ -345,110 +345,15 @@ func TestUpdateFieldNoStandardView(t *testing.T) {
}
}
func TestIngestSchemaHandler(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
schema := fmt.Sprintf(`
{
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
{
"field-name": "idset",
"field-type": "id",
"field-options": {
"cache-type": "none"
}
},
{
"field-name": "id",
"field-type": "id",
"field-options": {
"enforce-mutual-exclusion": true
}
},
{
"field-name": "bool",
"field-type": "bool"
},
{
"field-name": "stringset",
"field-type": "string",
"field-options": {
"cache-type": "ranked",
"cache-size": 100000
}
},
{
"field-name": "string",
"field-type": "string",
"field-options": {
"enforce-mutual-exclusion": true
}
},
{
"field-name": "int",
"field-type": "int"
},
{
"field-name": "decimal",
"field-type": "decimal",
"field-options": {
"scale": 2
}
},
{
"field-name": "timestamp",
"field-type": "timestamp",
"field-options": {
"epoch": "1996-12-19T16:39:57-08:00",
"unit": "µs"
}
},
{
"field-name": "quantum",
"field-type": "string",
"field-options": {
"time-quantum": "YMDH"
}
}
]
}
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != http.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
// now, try again, expecting a failure:
resp = test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != http.StatusConflict {
t.Errorf("invalid status: expected 409, got %d, body=%s", resp.StatusCode, resp.Body)
}
}
func TestPostFieldWithTTL(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
indexName := c.Idx("%s")
schema := fmt.Sprintf(`
{
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields":[]
}
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != http.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
indexName := c.Idx("s")
_, err := c.GetNode(0).API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
m := c.GetPrimary()
tests := []struct {
name string
@ -543,27 +448,15 @@ func TestGetViewAndDelete(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := fmt.Sprintf(`
{
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
{
"field-name": "test_view",
"field-type": "time",
"field-options": {
"time-quantum": "YMDH"
}
}
]
}
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != http.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
_, err := m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m.API.CreateField(context.Background(), c.Idx("s"), "test_view", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"))
if err != nil {
t.Fatalf("creating field: %v", err)
}
// Send sample data
@ -658,30 +551,16 @@ func TestTranslationHandlers(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
schema := fmt.Sprintf(`
{
"index-name": "%s",
"primary-key-type": "string",
"index-action": "create",
"fields": [
{
"field-name": "stringset",
"field-type": "string",
"field-options": {
"cache-type": "ranked",
"cache-size": 100000
}
}
]
}
`, c)
m := c.GetPrimary()
schemaURL := fmt.Sprintf("%s/internal/schema", m.URL())
resp := test.Do(t, "POST", schemaURL, string(schema))
if resp.StatusCode != http.StatusOK {
t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
_, err = m.API.CreateIndex(context.Background(), c.Idx("s"), pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m.API.CreateField(context.Background(), c.Idx("s"), "stringset", pilosa.OptFieldTypeSet("ranked", 100000), pilosa.OptFieldKeys())
if err != nil {
t.Fatalf("creating field: %v", err)
}
baseURLs := []string{
fmt.Sprintf("%s/internal/translate/index/%s/", m.URL(), c),
fmt.Sprintf("%s/internal/translate/field/%s/stringset/", m.URL(), c),
@ -802,10 +681,18 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
defer c.Close()
m := c.GetPrimary()
index := "allowed-networks-index"
keyedIndex := "allowed-networks-index-keyed"
index := c.Idx("s")
keyedIndex := c.Idx("k")
field := "field1"
// This keyed index used to be created by the Post-Schema subtest, but that doesn't
// exist anymore, so we create it up here. We don't create the other one because it's
// supposed to get created by Post-Index.
_, err = m.API.CreateIndex(context.Background(), c.Idx("k"), pilosa.IndexOptions{Keys: true})
if err != nil {
t.Fatalf("creating index: %v", err)
}
// needed for key translation
nameBytes, err := json.Marshal([]string{"a", "b", "c"})
if err != nil {
@ -813,24 +700,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
}
names := string(nameBytes)
schema := `
{
"index-name": "allowed-networks-index-keyed",
"primary-key-type": "string",
"index-action": "create",
"fields": [
{
"field-name": "stringset",
"field-type": "string",
"field-options": {
"cache-type": "ranked",
"cache-size": 100000
}
}
]
}
`
IPTests := []struct {
TestName string
ClientIP string
@ -864,12 +733,6 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
url: fmt.Sprintf("%s/schema", m.URL()),
body: "",
},
{
testName: "Post-Schema",
method: "POST",
url: fmt.Sprintf("%s/internal/schema", m.URL()),
body: schema,
},
{
testName: "Get-Shards",
method: "GET",

1169
ingest/codec.go
View file

File diff suppressed because it is too large Load diff

978
ingest/codec_test.go
View file

@ -1,978 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"fmt"
"sort"
"strings"
"testing"
"time"
"github.com/molecula/featurebase/v3/shardwidth"
)
func TestStableTranslator(t *testing.T) {
tr := newStableTranslator()
m1, err := tr.TranslateKeys("a", "b")
if err != nil {
t.Fatalf("translation error on initial keys: %v", err)
}
m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6)
if err != nil {
t.Fatalf("translation error on reverse lookup: %v", err)
}
m3, err := tr.TranslateKeys("a", "k-6")
if err != nil {
t.Fatalf("translation error on new keys: %v", err)
}
for k, v := range m3 {
if m2[v] != k {
t.Fatalf("expected round trip to equate %q and %d", k, v)
}
}
}
func TestMakeCodec(t *testing.T) {
codec, _ := NewJSONCodec(nil)
err := codec.AddSetField("set", nil)
if err != nil {
t.Fatalf("unexpected error creating field: %v", err)
}
err = codec.AddSetField("set", nil)
if err == nil {
t.Fatalf("expected error creating duplicate field, didn't get it")
}
}
func TestEncode(t *testing.T) {
codec, _ := NewJSONCodec(nil)
_ = codec.AddSetField("set", nil)
_ = codec.AddSetField("setkeys", newStableTranslator())
_ = codec.AddMutexField("mutex", nil)
_ = codec.AddMutexField("mutexkeys", newStableTranslator())
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
_ = codec.AddIntField("int", nil)
_ = codec.AddIntField("intkeys", newStableTranslator())
epoch, err := time.Parse("2006-01-02", "2020-01-01")
if err != nil {
t.Fatalf("can't parse sample epoch time: %v", err)
}
_ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000)
_ = codec.AddDecimalField("dec", 2)
_ = codec.AddBoolField("bool")
codecs := []*JSONCodec{codec}
// redo all of that, only on a keyed translator
codec, _ = NewJSONCodec(newStableTranslator())
_ = codec.AddSetField("set", nil)
_ = codec.AddSetField("setkeys", newStableTranslator())
_ = codec.AddMutexField("mutex", nil)
_ = codec.AddMutexField("mutexkeys", newStableTranslator())
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
_ = codec.AddIntField("int", nil)
_ = codec.AddIntField("intkeys", newStableTranslator())
_ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000)
_ = codec.AddDecimalField("dec", 2)
_ = codec.AddBoolField("bool")
codecs = append(codecs, codec)
encodeTests := []*Request{
{
Ops: []*Operation{
{
OpType: OpWrite,
ClearRecordIDs: []uint64{0, 1, 2, 3, 5},
ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"},
FieldOps: map[string]*FieldOperation{
"int": {
RecordIDs: []uint64{0, 1},
Signed: []int64{1, -3},
},
"intkeys": {
RecordIDs: []uint64{0, 1},
Signed: []int64{1, 1},
},
"set": {
RecordIDs: []uint64{0, 1, 2, 2},
Values: []uint64{1, 1, 0, 1},
},
"setkeys": {
RecordIDs: []uint64{0, 1, 2, 2},
Values: []uint64{1, 1, 0, 1},
},
"mutex": {
RecordIDs: []uint64{0, 1},
Values: []uint64{1, 2},
},
"mutexkeys": {
RecordIDs: []uint64{0, 1},
Values: []uint64{1, 2},
},
"tq": {
RecordIDs: []uint64{5, 5},
Values: []uint64{8, 9},
Signed: []int64{1234567890e9, 1234567890e9},
},
"tqkeys": {
RecordIDs: []uint64{3, 3},
Values: []uint64{2, 4},
Signed: []int64{1234567890e9, 1234567890e9},
},
"ts": {
RecordIDs: []uint64{0},
Signed: []int64{1},
},
"bool": {
RecordIDs: []uint64{0, 1},
Values: []uint64{0, 1},
},
"dec": {
RecordIDs: []uint64{0, 1, 2},
Signed: []int64{123, -123, 0},
},
},
},
{
OpType: OpClear,
Seq: 1,
ClearRecordIDs: []uint64{6},
ClearFields: []string{"tq"},
},
},
},
{
// this one needs to get filled in programmatically; see below
Ops: []*Operation{
{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{},
},
},
},
{
Ops: []*Operation{
{
OpType: OpRemove,
FieldOps: map[string]*FieldOperation{
"set": {},
},
},
},
},
}
// and now we populate encodeTests[1] with a larger pool of data
const dataSize = 5000
shardCount := uint64(600) // shards we want to target
passes := uint64(0)
recordIDs := make([]uint64, dataSize)
values := make([]uint64, dataSize)
timeStamps := make([]int64, dataSize)
signedValues := make([]int64, dataSize)
for i := uint64(0); i < dataSize; i++ {
if (i % shardCount) == 0 {
passes++
}
recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes
values[i] = (i % 4)
timeStamps[i] = int64(1234567890e9 + (i * 100e9))
signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
}
// ensure record IDs are sorted, because other stuff might rely on this
sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] })
op := encodeTests[1].Ops[0]
op.FieldOps["tq"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Values: append([]uint64{}, values...),
Signed: append([]int64{}, timeStamps...),
}
op.FieldOps["tqkeys"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Values: values,
Signed: timeStamps,
}
op.FieldOps["int"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Signed: append([]int64{}, signedValues...),
}
op.FieldOps["intkeys"] = &FieldOperation{
RecordIDs: recordIDs,
Signed: signedValues,
}
// for sets, we want to shuffle things into fewer shards, and ensure
// non-duplication of values within each record, but also have lots
// of duplication of record IDs in the low shards
recordIDs = make([]uint64, dataSize)
values = make([]uint64, dataSize)
valuesPerRecord := uint64(5)
recordsPerShard := dataSize / valuesPerRecord / 30
if recordsPerShard < 1 {
recordsPerShard = 1
}
shard := uint64(0)
nextID := uint64(0)
nextValue := uint64(0)
for i := uint64(0); i < dataSize; i++ {
recordIDs[i] = nextID
values[i] = nextValue + (i % valuesPerRecord)
nextValue++
if nextValue == valuesPerRecord {
nextValue = 0
nextID++
if nextID%(1<<shardwidth.Exponent) == recordsPerShard {
shard++
nextID = (shard << shardwidth.Exponent)
if valuesPerRecord > 1 {
valuesPerRecord--
}
}
}
}
sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] })
op.FieldOps["set"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Values: append([]uint64{}, values...),
}
op.FieldOps["setkeys"] = &FieldOperation{
RecordIDs: recordIDs,
Values: values,
}
var buf []byte
for i, tc := range encodeTests {
for _, c := range codecs {
data, err := c.AppendBytes(tc, buf[:0])
if err != nil {
t.Fatalf("encode test %d: error encoding: %v", i, err)
}
// t.Logf("data:\n%s", data)
req, err := c.ParseBytes(data)
if err != nil {
t.Logf("encode test %d: data:\n%s", i, data)
t.Fatalf("encode test %d: error parsing: %v", i, err)
}
err = req.Compare(tc)
if err != nil {
t.Logf("encode test %d: data:\n%s", i, data)
t.Fatalf("encode test %d: round-trip mismatch: %v", i, err)
}
data, err = c.AppendBytes(req, buf[:0])
if err != nil {
t.Fatalf("encode test %d: error encoding: %v", i, err)
}
// t.Logf("data:\n%s", data)
req2, err := c.ParseBytes(data)
if err != nil {
t.Logf("encode test %d: data:\n%s", i, data)
t.Fatalf("encode test %d: error parsing: %v", i, err)
}
err = req2.Compare(tc)
if err != nil {
t.Logf("encode test %d: data:\n%s", i, data)
t.Fatalf("encode test %d: round-trip mismatch: %v", i, err)
}
}
}
}
func TestCodecErrors(t *testing.T) {
codec, _ := NewJSONCodec(nil)
_ = codec.AddSetField("set", nil)
_ = codec.AddSetField("setkeys", newStableTranslator())
_ = codec.AddMutexField("mutex", nil)
_ = codec.AddMutexField("mutexkeys", newStableTranslator())
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddTimeQuantumField("tqkeys", newStableTranslator())
_ = codec.AddIntField("int", nil)
_ = codec.AddIntField("intkeys", newStableTranslator())
epoch, err := time.Parse("2006-01-02", "2020-01-01")
if err != nil {
t.Fatalf("can't parse sample epoch time: %v", err)
}
_ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000)
_ = codec.AddDecimalField("dec", 2)
_ = codec.AddBoolField("bool")
testCases := []struct {
name string
json []byte
error string
}{
{
name: "no action",
json: []byte(`[{"records":{"0":{"set":[0]}}}]`),
error: "action not specified",
},
{
name: "unknown action",
json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`),
error: "unknown action",
},
{
name: "unknown field",
json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`),
error: "field not found",
},
{
name: "unknown operation field",
json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`),
error: "unknown operation field",
},
{
name: "expected operation",
json: []byte(`[true]`),
error: "expected operation",
},
{
name: "expecting key",
json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`),
error: "expecting key",
},
{
name: "invalid int for bool",
json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`),
error: "boolean should be",
},
{
name: "invalid number for bool",
json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`),
error: "looks like Number",
},
{
name: "invalid string for bool",
json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`),
error: "expecting boolean",
},
{
name: "nonsense bool",
json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`),
error: "boolean should be",
},
{
name: "expecting numeric value",
json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`),
error: "invalid syntax",
},
{
name: "expecting value",
json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`),
error: "expecting value",
},
{
name: "expecting array-key",
json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`),
error: "expecting key",
},
{
name: "expecting array-value",
json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`),
error: "expecting value",
},
{
name: "expecting numeric array-value",
json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`),
error: "invalid syntax",
},
{
name: "expecting numeric value",
json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`),
error: "invalid syntax",
},
{
name: "expecting int key",
json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`),
error: "expecting string key",
},
{
name: "expecting int value",
json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`),
error: "expecting integer value",
},
{
name: "expecting string array-value",
json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`),
error: "expecting string key",
},
{
name: "expecting numeric mutex value",
json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`),
error: "invalid syntax",
},
{
name: "expecting mutex key",
json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`),
error: "expecting string key",
},
{
name: "expecting mutex value",
json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`),
error: "expecting integer value",
},
{
name: "expecting mutex string value",
json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`),
error: "expecting string key",
},
{
name: "time quantum invalid time",
json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`),
error: "expecting time",
},
{
name: "time stamp invalid integer",
json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`),
error: "parsing numeric time",
},
{
name: "time stamp invalid string",
json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`),
error: "parsing time",
},
{
name: "time stamp invalid type",
json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`),
error: "expecting time",
},
{
name: "invalid decimal",
json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`),
error: "expecting floating",
},
{
name: "duplicate record",
json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`),
error: "duplicated in input",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req, err := codec.ParseBytes(tc.json)
if err == nil {
req.Dump(t.Logf)
t.Fatalf("expected error like %q, got request instead", tc.error)
} else {
msg := err.Error()
if !strings.Contains(msg, tc.error) {
t.Fatalf("expected error like %q, got %q", tc.error, msg)
}
}
})
}
}
func TestSimpleCodec(t *testing.T) {
codec, _ := NewJSONCodec(nil)
_ = codec.AddSetField("set", nil)
_ = codec.AddSetField("setkeys", newStableTranslator())
_ = codec.AddMutexField("mutex", nil)
_ = codec.AddMutexField("mutexkeys", newStableTranslator())
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddIntField("int", nil)
_ = codec.AddIntField("intkeys", newStableTranslator())
epoch, err := time.Parse("2006-01-02", "2020-01-01")
if err != nil {
t.Fatalf("can't parse sample epoch time: %v", err)
}
_ = codec.AddTimestampField("ts", "ms", epoch.Unix()*1000)
_ = codec.AddDecimalField("dec", 2)
_ = codec.AddBoolField("bool")
var nextShard = uint64(1<<shardwidth.Exponent) + 5
sampleJson := []byte(fmt.Sprintf(`
[
{
"action": "set",
"records": {
"2": {
"set": [ 2 ],
"tq": {
"time": "2006-01-02T15:04:05.999999999Z",
"values": [ 6 ]
},
"dec": 1.02,
"int": "3",
"mutex": 4,
"bool": 1,
"mutexkeys": "key-a",
"intkeys": "key-a"
},
"%d": {
"set": [ 3 ],
"bool": true
},
"1": {
"set": [ 2 ],
"bool": 0,
"setkeys": [
"key-a",
"key-b"
],
"tq": { "values": [ 3, 4 ] }
},
"3":{
"bool": "true",
"ts": 27
}
}
},
{
"action": "clear",
"record_ids": [ 5, 6, 7 ],
"fields": [ "tq" ]
},
{
"action": "write",
"records": {
"3": {
"set": 2,
"mutex": 4,
"mutexkeys": "key-a",
"tq": {
"time": 1234567890,
"values": 6
},
"ts": "2020-01-01T00:01:00.000000000Z",
"int": 3,
"intkeys": "key-a"
},
"4": {
"set": [ 3 ],
"ts": 1577836860000
},
"5": {
"set": [ 2 ],
"setkeys": [ "key-a", "key-b" ],
"tq": { "values": [ 3, 4 ] }
}
}
},
{
"action": "delete",
"record_ids": [ 9 ]
},
{
"action": "set",
"records": {
"2": {
"mutex": 5,
"mutexkeys": "key-b"
}
}
}
]
`, nextShard))
var expected = &ShardedRequest{
Ops: map[uint64][]*Operation{
0: {
{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"mutex": {
RecordIDs: []uint64{2},
Values: []uint64{4},
},
"set": {
RecordIDs: []uint64{1, 2},
Values: []uint64{2, 2},
},
"tq": {
RecordIDs: []uint64{1, 1, 2},
Values: []uint64{3, 4, 6},
Signed: []int64{0, 0, 1136214245999999999},
},
"mutexkeys": {
RecordIDs: []uint64{2},
Values: []uint64{0},
},
"int": {
RecordIDs: []uint64{2},
Signed: []int64{3},
},
"intkeys": {
RecordIDs: []uint64{2},
Signed: []int64{0},
},
"setkeys": {
RecordIDs: []uint64{1, 1},
Values: []uint64{0, 1},
},
"dec": {
RecordIDs: []uint64{2},
Signed: []int64{102},
},
"bool": {
RecordIDs: []uint64{1, 2, 3},
Values: []uint64{0, 1, 1},
},
"ts": {
RecordIDs: []uint64{3},
Signed: []int64{27},
},
},
},
{
OpType: OpClear,
Seq: 1,
ClearRecordIDs: []uint64{5, 6, 7},
ClearFields: []string{"tq"},
},
{
OpType: OpWrite,
Seq: 2,
ClearRecordIDs: []uint64{3, 4, 5},
ClearFields: []string{"int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "ts"},
FieldOps: map[string]*FieldOperation{
"mutex": {
RecordIDs: []uint64{3},
Values: []uint64{4},
},
"set": {
RecordIDs: []uint64{3, 5, 4},
Values: []uint64{2, 2, 3},
},
"tq": {
RecordIDs: []uint64{5, 5, 3},
Values: []uint64{3, 4, 6},
Signed: []int64{0, 0, 1234567890e9},
},
"mutexkeys": {
RecordIDs: []uint64{3},
Values: []uint64{0},
},
"int": {
RecordIDs: []uint64{3},
Signed: []int64{3},
},
"intkeys": {
RecordIDs: []uint64{3},
Signed: []int64{0},
},
"setkeys": {
RecordIDs: []uint64{5, 5},
Values: []uint64{0, 1},
},
"ts": {
RecordIDs: []uint64{3, 4},
Signed: []int64{60000, 1577836860000},
},
},
},
{
OpType: OpDelete,
Seq: 3,
ClearRecordIDs: []uint64{9},
},
{
OpType: OpSet,
Seq: 4,
FieldOps: map[string]*FieldOperation{
"mutex": {
RecordIDs: []uint64{2},
Values: []uint64{5},
},
"mutexkeys": {
RecordIDs: []uint64{2},
Values: []uint64{1},
},
},
},
},
1: {
{
OpType: OpSet,
Seq: 0,
FieldOps: map[string]*FieldOperation{
"set": {
RecordIDs: []uint64{nextShard},
Values: []uint64{3},
},
"bool": {
RecordIDs: []uint64{nextShard},
Values: []uint64{1},
},
},
},
},
},
}
req, err := codec.ParseBytes(sampleJson)
if err != nil {
t.Fatalf("parsing sample buffer: %v", err)
}
// req.Dump(t.Logf)
fieldTypes := codec.FieldTypes()
sharded, err := req.ByShard(fieldTypes)
if err != nil {
t.Errorf("sharding err: %v", err)
}
for shard, ops := range sharded.Ops {
for i, op := range ops {
op.Sort()
for field, fop := range op.FieldOps {
sorter := fieldTypeSorts[fieldTypes[field]]
if sorter == nil {
sorter = (*FieldOperation).SortByRecords
}
sorter(fop)
}
var expectedOp *Operation
if i < len(expected.Ops[shard]) {
expectedOp = expected.Ops[shard][i]
}
if err = op.Compare(expectedOp); err != nil {
t.Errorf("shard %d, op %d: %v", shard, i, err)
}
}
}
}
func expectEqualFieldOp(t *testing.T, fo1, fo2 *FieldOperation) {
if err := fo1.Compare(fo2); err != nil {
t.Fatalf("unexpected fieldOp mismatch: %v", err)
}
if err := fo2.Compare(fo1); err != nil {
t.Fatalf("unexpected fieldOp mismatch (inverted): %v", err)
}
}
func expectUnequalFieldOp(t *testing.T, msg string, fo1, fo2 *FieldOperation) {
if err := fo1.Compare(fo2); err == nil {
t.Fatalf("unexpected fieldOp equality %s", msg)
}
if err := fo2.Compare(fo1); err == nil {
t.Fatalf("unexpected fieldOp equality %s", msg)
}
}
func expectEqualOp(t *testing.T, fo1, fo2 *Operation) {
if err := fo1.Compare(fo2); err != nil {
t.Fatalf("unexpected Op mismatch: %v", err)
}
if err := fo2.Compare(fo1); err != nil {
t.Fatalf("unexpected Op mismatch (inverted): %v", err)
}
}
func expectUnequalOp(t *testing.T, msg string, fo1, fo2 *Operation) {
if err := fo1.Compare(fo2); err == nil {
t.Fatalf("unexpected Op equality %s", msg)
}
if err := fo2.Compare(fo1); err == nil {
t.Fatalf("unexpected Op equality %s", msg)
}
}
func expectEqualReq(t *testing.T, fo1, fo2 *Request) {
if err := fo1.Compare(fo2); err != nil {
t.Fatalf("unexpected Request mismatch: %v", err)
}
if err := fo2.Compare(fo1); err != nil {
t.Fatalf("unexpected Request mismatch (inverted): %v", err)
}
}
func expectUnequalReq(t *testing.T, msg string, fo1, fo2 *Request) {
if err := fo1.Compare(fo2); err == nil {
t.Fatalf("unexpected Request equality %s", msg)
}
if err := fo2.Compare(fo1); err == nil {
t.Fatalf("unexpected Request equality %s", msg)
}
}
func expectEqualShardedReq(t *testing.T, fo1, fo2 *ShardedRequest) {
if err := fo1.Compare(fo2); err != nil {
t.Fatalf("unexpected Request mismatch: %v", err)
}
if err := fo2.Compare(fo1); err != nil {
t.Fatalf("unexpected Request mismatch (inverted): %v", err)
}
}
func expectUnequalShardedReq(t *testing.T, msg string, fo1, fo2 *ShardedRequest) {
if err := fo1.Compare(fo2); err == nil {
t.Fatalf("unexpected Request equality %s", msg)
}
if err := fo2.Compare(fo1); err == nil {
t.Fatalf("unexpected Request equality %s", msg)
}
}
func testCompareFieldOpMutate(t *testing.T, fo1 *FieldOperation) {
fo2 := fo1.clone()
expectEqualFieldOp(t, fo1, fo2)
fo1.RecordIDs = fo1.RecordIDs[:1]
expectUnequalFieldOp(t, "short record IDs", fo1, fo2)
fo1.RecordIDs = fo1.RecordIDs[:2]
fo1.RecordIDs[1] = 2
expectUnequalFieldOp(t, "mismatched record IDs", fo1, fo2)
fo1.RecordIDs[1] = fo2.RecordIDs[1]
if len(fo1.Values) != 0 {
fo1.Values = fo1.Values[:1]
expectUnequalFieldOp(t, "short values", fo1, fo2)
fo1.Values = fo1.Values[:2]
fo1.Values[1]++
expectUnequalFieldOp(t, "mismatched values", fo1, fo2)
fo1.Values[1] = fo2.Values[1]
}
if len(fo1.Signed) != 0 {
fo1.Signed = fo1.Signed[:1]
expectUnequalFieldOp(t, "short signed values", fo1, fo2)
fo1.Signed = fo1.Signed[:2]
fo1.Signed[1]++
expectUnequalFieldOp(t, "mismatched signed values", fo1, fo2)
fo1.Signed[1] = fo2.Signed[1]
}
}
func TestCompareFieldOperation(t *testing.T) {
fo := &FieldOperation{
RecordIDs: []uint64{0, 1},
Values: []uint64{0, 1},
}
testCompareFieldOpMutate(t, fo)
fo.Signed = []int64{-3, 5}
testCompareFieldOpMutate(t, fo)
fo.Values = nil
testCompareFieldOpMutate(t, fo)
// after this, fo has only RecordIDs
fo.Signed = nil
var fNil *FieldOperation
expectUnequalFieldOp(t, "nil and non-empty", fo, fNil)
fo.RecordIDs = fo.RecordIDs[:0]
expectEqualFieldOp(t, fo, fNil)
if err := fNil.Compare(fNil); err != nil {
t.Fatalf("expected nil and nil to be equal: %v", err)
}
}
func TestCompareOperation(t *testing.T) {
op1 := &Operation{
OpType: OpWrite,
ClearRecordIDs: []uint64{0, 1},
ClearFields: []string{"a", "b"},
Seq: 1,
FieldOps: map[string]*FieldOperation{
"c": {
RecordIDs: []uint64{0},
Values: []uint64{0},
},
},
}
op2 := op1.clone()
expectEqualOp(t, op1, op2)
op1.Seq++
expectUnequalOp(t, "seq mismatch", op1, op2)
op1.Seq = op2.Seq
op1.OpType++
expectUnequalOp(t, "opType mismatch", op1, op2)
op1.OpType = op2.OpType
op1.ClearRecordIDs = op1.ClearRecordIDs[:1]
expectUnequalOp(t, "short clearRecordIDs", op1, op2)
op1.ClearRecordIDs = op1.ClearRecordIDs[:2]
op1.ClearRecordIDs[1]++
expectUnequalOp(t, "mismatched clearRecordIDs", op1, op2)
op1.ClearRecordIDs[1] = op2.ClearRecordIDs[1]
op1.ClearFields = op1.ClearFields[:1]
expectUnequalOp(t, "short clearFields", op1, op2)
op1.ClearFields = op1.ClearFields[:2]
op1.ClearFields[1] = "z"
expectUnequalOp(t, "mismatched clearFields", op1, op2)
op1.ClearFields[1] = op2.ClearFields[1]
op1.FieldOps["d"] = op1.FieldOps["c"]
expectUnequalOp(t, "extra fieldOp", op1, op2)
op2.FieldOps["d"] = op2.FieldOps["c"]
expectEqualOp(t, op1, op2)
op1.FieldOps["c"].RecordIDs[0]++
expectUnequalOp(t, "mismatched fieldOp", op1, op2)
expectUnequalOp(t, "nil Op", nil, op1)
expectEqualOp(t, nil, nil)
}
func TestCompareRequest(t *testing.T) {
r1 := &Request{
Ops: []*Operation{
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
}
r2 := &Request{
Ops: []*Operation{
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
}
expectEqualReq(t, r1, r2)
r1.Ops = r1.Ops[:1]
expectUnequalReq(t, "short ops", r1, r2)
r1.Ops = r1.Ops[:2]
r1.Ops[1].Seq = 2
expectUnequalReq(t, "ops mismatch", r1, r2)
r1.Ops = r1.Ops[:2]
expectUnequalReq(t, "non-empty and nil", r1, nil)
r1.Ops = r1.Ops[:0]
expectEqualReq(t, r1, nil)
}
func TestCompareShardedRequest(t *testing.T) {
r1 := &ShardedRequest{
Ops: map[uint64][]*Operation{
1: {
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
2: {
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
},
}
r2 := &ShardedRequest{
Ops: map[uint64][]*Operation{
1: {
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
2: {
{OpType: OpSet},
{OpType: OpSet, Seq: 1},
},
},
}
expectEqualShardedReq(t, r1, r2)
stash := r1.Ops[2]
delete(r1.Ops, 2)
expectUnequalShardedReq(t, "short opmap", r1, r2)
expectUnequalShardedReq(t, "nil vs nonempty", r1, nil)
delete(r1.Ops, 1)
expectEqualShardedReq(t, r1, nil)
r1.Ops[2] = stash[:1]
expectUnequalShardedReq(t, "short ops", r1, r2)
r1.Ops[2] = stash
r1.Ops[2][1].Seq = 2
expectUnequalShardedReq(t, "mismatched ops", r1, r2)
}

17
ingest/doc.go
View file

@ -1,17 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Package ingest provides tooling for accepting record-oriented data updates
// and converting them to data that can be efficiently merged into stored
// data. Nia's original description:
//
// but the overall pipeline is:
// 1. fetch the schema and use it to configure the codec
// 2. parse the data with the codec into vectors, while stuffing temp record key mappings into a string table
// 3. call *CreateKeys on the cluster for all of the things
// 4. generate an ID remapping table for record keys and apply it to all of the vectors
// 5. remap the string keys
// 6. group each vector by shard
// 7. convert the shard vectors into matrix updates
// 8. combine those matrix updates into a shard update
// 9. send the shard updates out over the internal client
// 10. the nodes apply them to RBF
package ingest

826
ingest/op.go
View file

@ -1,826 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"errors"
"fmt"
"math/bits"
"sort"
"github.com/molecula/featurebase/v3/shardwidth"
)
type OpType uint8
const (
OpNone = OpType(iota)
OpSet
OpRemove
OpClear
OpWrite
OpDelete
)
var opNames = []string{
"none",
"set",
"remove",
"clear",
"write",
"delete",
}
type FieldType string
const (
FieldTypeSet = "set"
FieldTypeInt = "int"
FieldTypeTimeQuantum = "time"
FieldTypeTimeStamp = "timestamp"
FieldTypeDecimal = "decimal"
FieldTypeMutex = "mutex"
FieldTypeBool = "bool"
)
var fieldTypeSorts = map[FieldType]func(*FieldOperation){
FieldTypeSet: (*FieldOperation).SortByValues,
FieldTypeInt: (*FieldOperation).SortByRecords,
FieldTypeTimeQuantum: (*FieldOperation).SortByValues,
FieldTypeDecimal: (*FieldOperation).SortByRecords,
FieldTypeMutex: (*FieldOperation).SortByValues,
FieldTypeTimeStamp: (*FieldOperation).SortByRecords,
FieldTypeBool: (*FieldOperation).SortByValues,
}
func (o OpType) String() string {
if int(o) < len(opNames) {
return opNames[o]
}
return fmt.Sprintf("invalid-optype-%d", o)
}
func ParseOpType(s string) (OpType, error) {
for i, v := range opNames[1:] {
if s == v {
return OpType(i + 1), nil
}
}
return 0, fmt.Errorf("unknown operation type %q", s)
}
// Operation represents a single set of changes to make to
// the stored data, which means some combination of clearing
// columns, clearing individual bits, or setting bits or values.
// The same data structure can be used whether this represents the
// whole database operation or a single shard's values.
//
// Operations can specify individual per-field operations, which
// have maps of record IDs to values. They can also have a set of
// record IDs and fields to clear. A Clear operation will have only
// record IDs and fields, a Set or Remove will have only FieldOps,
// and a Write will have both -- populating the record IDs and fields
// from the fieldops.
//
// When we parse an operation, we assign each op a sequential ID
// within the overall request. We keep these IDs associated with ops
// when splitting them up across shards, so we can reverse this even
// if some shards don't get some ops.
type Operation struct {
OpType OpType
Seq int // sequence position within a chain of ops
ClearRecordIDs []uint64
ClearFields []string
FieldOps map[string]*FieldOperation
}
// Compare reports whether two Operations seem to be the same. While
// a FieldOperation can be "empty" and compare-equal-to a nil FieldOperation,
// no Operation is considered empty even if it has no FieldOps.
func (got *Operation) Compare(expected *Operation) error {
if got == nil && expected == nil {
return nil
}
if got == nil {
return fmt.Errorf("expected %q op, got nil", expected.OpType)
}
if expected == nil {
return fmt.Errorf("expected no op, got %q", got.OpType)
}
if got.OpType != expected.OpType {
return fmt.Errorf("operation type mismatch: expected %q, got %q", expected.OpType, got.OpType)
}
if len(got.ClearRecordIDs) != len(expected.ClearRecordIDs) {
return fmt.Errorf("clear record counts differ: expected %d, got %d", len(expected.ClearRecordIDs), len(got.ClearRecordIDs))
}
for i, v1 := range got.ClearRecordIDs {
v2 := expected.ClearRecordIDs[i]
if v1 != v2 {
return fmt.Errorf("clear record id %d differs: expected %d, got %d", i, v2, v1)
}
}
// don't assume consistent ordering for the fields, because they're
// coming out in arbitrary hash order
seenFields := make(map[string]struct{}, len(expected.ClearFields))
for _, v1 := range expected.ClearFields {
seenFields[v1] = struct{}{}
}
for _, v2 := range got.ClearFields {
if _, ok := seenFields[v2]; !ok {
return fmt.Errorf("field %q cleared unexpectedly", v2)
}
delete(seenFields, v2)
}
for v1 := range seenFields {
return fmt.Errorf("field %q should be cleared but wasn't", v1)
}
// We check compare even if the op we find on one side is nil, so
// a round-trip test will consider a missing op and an empty op to
// be interchangeable.
for k, fo1 := range got.FieldOps {
fo2 := expected.FieldOps[k]
if err := fo1.Compare(fo2); err != nil {
return fmt.Errorf("field %q mismatch: %w", k, err)
}
}
for k, fo2 := range expected.FieldOps {
fo1 := got.FieldOps[k]
if err := fo1.Compare(fo2); err != nil {
return fmt.Errorf("field %q mismatch: %w", k, err)
}
}
if expected.Seq != got.Seq {
return fmt.Errorf("sequence mismatch: expected %d, got %d", expected.Seq, got.Seq)
}
return nil
}
// FieldOperation is the specific set of changes to make to a given
// field.
//
// For a Clear operation, values can be an empty array. For Set or Remove
// operations on sets, RecordIDs can contain duplicates. Times should
// be empty except for time-quantum fields.
type FieldOperation struct {
RecordIDs []uint64
Values []uint64
// For int/timestamp/decimal, this is the value
// For time-quantum, this is the timestamp
// No field has both signed values and timestamps.
// This is not a place of honor.
Signed []int64
}
// Sort sorts the clear record IDs and field list.
func (o *Operation) Sort() {
// I am aware that this is a crime, but it avoids rewriting
// the code and justifies FieldOperation handling the "only record
// IDs" case.
f := FieldOperation{RecordIDs: o.ClearRecordIDs}
f.SortByRecords()
sort.Strings(o.ClearFields)
}
// clone makes a duplicate of the operation without shared storage
func (o *Operation) clone() *Operation {
o2 := &Operation{
OpType: o.OpType,
Seq: o.Seq,
ClearRecordIDs: append([]uint64{}, o.ClearRecordIDs...),
ClearFields: append([]string{}, o.ClearFields...),
FieldOps: make(map[string]*FieldOperation, len(o.FieldOps)),
}
for k, v := range o.FieldOps {
o2.FieldOps[k] = v.clone()
}
return o2
}
// merge merges the fieldops of the provided operation into this operation.
func (o *Operation) merge(o2 *Operation) {
o.ClearRecordIDs = append(o.ClearRecordIDs, o2.ClearRecordIDs...)
o.ClearFields = append(o.ClearFields, o2.ClearFields...)
for f, op := range o2.FieldOps {
dst := o.FieldOps[f]
if dst == nil {
o.FieldOps[f] = op
continue
}
// otherwise append any contents. dst is a pointer-to, so this
// updates the thing the map points to, we don't have to write back
// into the map.
dst.RecordIDs = append(dst.RecordIDs, op.RecordIDs...)
dst.Values = append(dst.Values, op.Values...)
dst.Signed = append(dst.Signed, op.Signed...)
}
}
type ShardedFieldOperation map[uint64]*FieldOperation
// ByShard() divides the FieldOperation's values up into corresponding chunks
// based on the shards of record IDs. Does not further sort IDs within those
// chunks.
func (f *FieldOperation) ByShard() ShardedFieldOperation {
if len(f.RecordIDs) == 0 {
return nil
}
return f.SortToShards()
}
// clone makes a duplicate of the operation without shared storage
func (f *FieldOperation) clone() *FieldOperation {
f2 := &FieldOperation{
RecordIDs: append([]uint64{}, f.RecordIDs...),
Values: append(([]uint64)(nil), f.Values...),
Signed: append(([]int64)(nil), f.Signed...),
}
return f2
}
func ShardIDs(ids []uint64) (out map[uint64][]uint64) {
shards, ends := shardwidth.FindShards(ids)
prev := 0
out = make(map[uint64][]uint64, len(shards))
for i, shard := range shards {
endIndex := ends[i]
out[shard] = ids[prev:endIndex]
prev = endIndex
}
return out
}
// SortToShards() uses a pseudo-radix-sort to divide inputs into
// shards; the individual shards are not sorted.
func (f *FieldOperation) SortToShards() ShardedFieldOperation {
if len(f.RecordIDs) == 0 {
return nil
}
diffMask := uint64(0)
prev := f.RecordIDs[0]
for _, r := range f.RecordIDs[1:] {
diffMask |= r ^ prev
prev = r
}
bitsRemaining := bits.Len64(diffMask)
if bitsRemaining <= shardwidth.Exponent {
return map[uint64]*FieldOperation{f.RecordIDs[0] >> shardwidth.Exponent: f}
}
output := make(ShardedFieldOperation)
sortToShardsInto(f, bitsRemaining-8, output)
return output
}
// sortToShardsInto puts the shards it finds into the given map, so that
// as we split off buckets, they can be inserted into the same map.
func sortToShardsInto(f *FieldOperation, shift int, into ShardedFieldOperation) {
if shift < shardwidth.Exponent {
shift = shardwidth.Exponent
}
nextShift := shift - 8
if nextShift < shardwidth.Exponent {
nextShift = shardwidth.Exponent
}
// count things that belong in each of the 256 buckets
var buckets [256]int
var starts [256]int
// compute the buckets ourselves
for _, r := range f.RecordIDs {
b := (r >> shift) & 0xFF
buckets[b]++
}
total := 0
// compute starting points of each bucket, converting the
// bucket counts into ends
for i := range buckets {
starts[i] = total
total += buckets[i]
buckets[i] = total
}
// starts[n] is the index of the first thing that should
// go in that bucket, buckets[n] is the index of the first
// thing that shouldn't
var bucketOp FieldOperation
for bucket, start := range starts {
end := buckets[bucket]
if end <= start {
continue
}
for j := start; j < end; j++ {
want := int((f.RecordIDs[j] >> shift) & 0xFF)
for want != bucket {
// move this to the beginning of the
// bucket it wants to be in, swapping
// the thing there here
dst := starts[want]
f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j]
if f.Values != nil {
f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j]
}
if f.Signed != nil {
f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j]
}
starts[want]++
want = int((f.RecordIDs[j] >> shift) & 0xFF)
}
}
// If shift == shardwidth.Exponent, then this is a completed
// shard and can go into the sharded output. otherwise, we
// can subdivide it.
bucketOp.RecordIDs = f.RecordIDs[start:end]
if f.Values != nil {
bucketOp.Values = f.Values[start:end]
}
if f.Signed != nil {
bucketOp.Signed = f.Signed[start:end]
}
if shift == shardwidth.Exponent {
x := bucketOp
into[f.RecordIDs[start]>>shardwidth.Exponent] = &x
} else {
sortToShardsInto(&bucketOp, nextShift, into)
}
}
}
const shardMask = ((uint64(1) << shardwidth.Exponent) - 1)
// SortByValues sorts the operation by values first, then by record
// ID within each value. This is the best ordering for set/mutex fields,
// where we'll want to generate positions in that order. For these
// purposes, a time quantum or bool counts as a kind of a set.
func (f *FieldOperation) SortByValues() {
keys := make([]uint64, len(f.RecordIDs))
for i, v := range f.RecordIDs {
keys[i] = (f.Values[i] << shardwidth.Exponent) | (v & shardMask)
}
f.SortByKeys(keys)
}
// SortByRecords sorts the operation by record ID, and not by value at
// all. This makes the most sense for int fields and the like.
func (f *FieldOperation) SortByRecords() {
f.SortByKeys(f.RecordIDs)
}
// SortByKeys reorganizes the record IDs and values of f according to the
// corresponding members of keys.
func (f *FieldOperation) SortByKeys(keys []uint64) {
if len(f.RecordIDs) < 2 {
return
}
diffMask := uint64(0)
prev := keys[0]
for _, r := range keys[1:] {
diffMask |= r ^ prev
prev = r
}
bitsRemaining := bits.Len64(diffMask)
sortPartialByKeys(f, keys, bitsRemaining-8)
}
// simpleSort sorts a FieldOperation by external keys, or record IDs. It's a
// horribly naive bubble sort because N is small and a more complex algorithm
// doesn't help as much as you'd hope. This beats using stdlib sort by about
// a factor of two for those small N, for larger N we're using the radix sort
// that calls this.
//
// External keys exist only when we are sorting by value, which is to say,
// when we're using row-oriented formats (set, mutex, time quantum).
// For int/decimal/timestamp fields, we're sorting by record only.
// So, if keys is the same as f.RecordIDs, we're looking at an int field
// or equivalent, so Signed exists and Values doesn't exist.
// Otherwise, we might be looking at a time quantum field (both exist)
// or set/mutex (only Values exist).
func simpleSort(f *FieldOperation, keys []uint64) {
// keys might actually just point to record IDs, in which case, we don't
// want to shuffle the corresponding RecordIDs too, because that would just
// reverse our swaps. If they're different, we actually need to swap them
// both.
if &keys[0] != &f.RecordIDs[0] {
// sorting by record IDs
if f.Values != nil && f.Signed != nil { // time quantum field
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else if f.Values != nil { // set/mutex/bool
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
}
}
} else if f.Signed != nil { // can't-happen, we think
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else { // can't-happen, we think
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
}
}
}
} else {
if f.Values == nil && f.Signed != nil {
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else if f.Values != nil && f.Signed != nil { // can't happen, we think
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else if f.Values != nil { // only happens during testing
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
}
}
} else { // should definitely not happen
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
}
}
}
}
}
func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) {
if shift < 0 {
shift = 0
}
externalKeys := &f.RecordIDs[0] != &keys[0]
nextShift := shift - 8
if nextShift < 0 {
nextShift = 0
}
// count things that belong in each of the 256 buckets
var buckets [256]int
var starts [256]int
// compute the buckets ourselves
for _, r := range keys {
b := (r >> shift) & 0xFF
buckets[b]++
}
total := 0
// compute starting points of each bucket, converting the
// bucket counts into ends
for i := range buckets {
starts[i] = total
total += buckets[i]
buckets[i] = total
}
// starts[n] is the index of the first thing that should
// go in that bucket, buckets[n] is the index of the first
// thing that shouldn't
// var newbuckets [256]int
var bucketOp FieldOperation
for bucket, start := range starts {
end := buckets[bucket]
if end <= start {
continue
}
for j := start; j < end; j++ {
want := int((keys[j] >> shift) & 0xFF)
for want != bucket {
// move this to the beginning of the
// bucket it wants to be in, swapping
// the thing there here
dst := starts[want]
keys[j], keys[dst] = keys[dst], keys[j]
// we do this to allow you to just pass in the records as keys
if externalKeys {
f.RecordIDs[j], f.RecordIDs[dst] = f.RecordIDs[dst], f.RecordIDs[j]
}
if f.Values != nil {
f.Values[j], f.Values[dst] = f.Values[dst], f.Values[j]
}
if f.Signed != nil {
f.Signed[j], f.Signed[dst] = f.Signed[dst], f.Signed[j]
}
starts[want]++
want = int((keys[j] >> shift) & 0xFF)
}
}
// If shift == shardwidth.Exponent, then this is a completed
// shard and can go into the sharded output. otherwise, we
// can subdivide it.
if shift > 0 {
bucketOp.RecordIDs = f.RecordIDs[start:end]
if f.Values != nil {
bucketOp.Values = f.Values[start:end]
}
if f.Signed != nil {
bucketOp.Signed = f.Signed[start:end]
}
// if there's not very many, sort naively instead
if end-start > 32 {
sortPartialByKeys(&bucketOp, keys[start:end], nextShift)
} else {
simpleSort(&bucketOp, keys[start:end])
}
}
}
}
// AddPair adds a record ID/value pair where the value is unsigned, as
// when used with set/mutex/time quantum fields.
func (f *FieldOperation) AddPair(rec uint64, value uint64) {
f.RecordIDs = append(f.RecordIDs, rec)
f.Values = append(f.Values, value)
}
// AddSignedPair adds a record ID/value pair where the value is signed,
// as when used with int/decimal/timestamp fields.
func (f *FieldOperation) AddSignedPair(rec uint64, value int64) {
f.RecordIDs = append(f.RecordIDs, rec)
f.Signed = append(f.Signed, value)
}
// AddStampedPair adds a record/value pair plus a time, which is just
// a Unix time in seconds. (Note, no scaling here; timestamp fields are
// scaled int fields, this is for time quantums.)
func (f *FieldOperation) AddStampedPair(rec uint64, value uint64, stamp int64) {
f.RecordIDs = append(f.RecordIDs, rec)
f.Values = append(f.Values, value)
f.Signed = append(f.Signed, stamp)
}
// Compare returns a diagnostic if the field operations do not seem
// equivalent.
func (got *FieldOperation) Compare(expected *FieldOperation) error {
if got == nil {
if expected == nil {
return nil
}
// We don't worry about non-empty Values or Signed here, because in theory
// RecordIDs are the Source of Truth as to what's in the op.
if len(expected.RecordIDs) == 0 {
return nil
}
return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs))
}
if expected == nil {
if len(got.RecordIDs) == 0 {
return nil
}
return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs))
}
if len(got.RecordIDs) != len(expected.RecordIDs) {
return fmt.Errorf("record counts differ: expected %d, got %d", len(expected.RecordIDs), len(got.RecordIDs))
}
for i, v1 := range got.RecordIDs {
v2 := expected.RecordIDs[i]
if v1 != v2 {
return fmt.Errorf("record id %d differs: expected %d, got %d", i, v2, v1)
}
}
if len(got.Values) != len(expected.Values) {
return fmt.Errorf("value counts differ: expected %d, got %d", len(expected.Values), len(got.Values))
}
for i, v1 := range got.Values {
v2 := expected.Values[i]
if v1 != v2 {
return fmt.Errorf("value %d differs: expected %d, got %d", i, v2, v1)
}
}
if len(got.Signed) != len(expected.Signed) {
return fmt.Errorf("signed value counts differ: expected %d, got %d", len(expected.Signed), len(got.Signed))
}
for i, v1 := range got.Signed {
v2 := expected.Signed[i]
if v1 != v2 {
return fmt.Errorf("signed value %d differs: expected %d, got %d", i, v2, v1)
}
}
return nil
}
// ShardOperations is a set of Operations associated with a specific shard.
type ShardOperations struct {
Shard uint64
Ops []*Operation
}
// Request is a complete ingest request, which may be any combination
// of operations, which may apply to multiple shards.
type Request struct {
Ops []*Operation
}
// ShardedRequest is an ingest request, split up into individual per-shard
// operations.
type ShardedRequest struct {
Ops map[uint64][]*Operation
}
// ByShard converts a request into the same request, only sharded.
func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) {
if len(r.Ops) == 0 {
return &ShardedRequest{Ops: nil}, nil
}
req := make(map[uint64][]*Operation)
shards := make(map[uint64]*Operation)
// we're getting per-field things, which we want to divide per-shard,
// and return to per-shard sets of per-field things, so we're inverting
// the structure.
for _, op := range r.Ops {
// for clear and write ops, we also need to split up the
// ClearRecords values, which may be distinct from the set of
// records for any given field. For Write ops, we'll then end
// up adding in field values for some fields.
if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete {
sharded := ShardIDs(op.ClearRecordIDs)
for shard, data := range sharded {
shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}}
}
}
for field, fieldOp := range op.FieldOps {
sharded := fieldOp.ByShard()
sorter := fieldTypeSorts[fields[field]]
if sorter == nil {
sorter = (*FieldOperation).SortByRecords
}
for shard, data := range sharded {
sorter(data)
shardOp, ok := shards[shard]
if !ok {
if op.OpType == OpWrite {
return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard)
}
shardOp = &Operation{OpType: op.OpType, Seq: op.Seq}
shards[shard] = shardOp
shardOp.FieldOps = map[string]*FieldOperation{field: data}
} else {
shardOp.FieldOps[field] = data
}
}
}
for shard, shardOp := range shards {
req[shard] = append(req[shard], shardOp)
}
for k := range shards {
delete(shards, k)
}
}
return &ShardedRequest{Ops: req}, nil
}
// merge combines the components of a sharded request back into a single
// unsharded request, processing shards in numerical order.
func (s *ShardedRequest) merge() *Request {
req := &Request{}
if s == nil || len(s.Ops) == 0 {
return req
}
shards := make([]uint64, 0, len(s.Ops))
for shard := range s.Ops {
shards = append(shards, shard)
}
sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] })
for _, shard := range shards {
ops := s.Ops[shard]
for _, op := range ops {
var _ *Operation
if op.Seq >= len(req.Ops) {
// Pad out with nil *Operations to the required length
req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...)
}
if req.Ops[op.Seq] == nil {
req.Ops[op.Seq] = op.clone()
continue
}
req.Ops[op.Seq].merge(op)
}
}
return req
}
func (r *Request) Dump(logf func(string, ...interface{})) {
logf("req: %#v", r)
for _, op := range r.Ops {
logf("op: %#v", op)
if len(op.ClearRecordIDs) > 0 {
if len(op.ClearRecordIDs) > 8 {
logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8)
} else {
logf(" clearRecordIDs: %d", op.ClearRecordIDs)
}
}
if len(op.ClearFields) > 0 {
if len(op.ClearFields) > 8 {
logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8)
} else {
logf(" clearFields: %s", op.ClearFields)
}
}
for field, fieldOp := range op.FieldOps {
if fieldOp != nil {
logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed))
if len(fieldOp.RecordIDs) > 0 {
if len(fieldOp.RecordIDs) > 8 {
logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8)
} else {
logf(" records %d", fieldOp.RecordIDs)
}
}
} else {
logf(" field %q: nil op", field)
}
}
}
}
func (r *Request) Compare(other *Request) error {
if other == nil {
if r != nil && len(r.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
if r == nil {
if other != nil && len(other.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
ops := r.Ops
ops2 := other.Ops
if len(ops2) != len(ops) {
return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2))
}
for i, op := range ops {
if err := op.Compare(ops2[i]); err != nil {
return fmt.Errorf("op %d: %v", i, err)
}
}
return nil
}
// Compare checks whether two ShardedRequest objects represent the same
// data. Empty shards shouldn't have entries in the map in the first place,
// so we don't accept a nil or 0-length slice of ops as equal to the
// shard key not existing, but we do accept nil or empty requests as
// equal to each other.
func (s *ShardedRequest) Compare(other *ShardedRequest) error {
if other == nil {
if s != nil && len(s.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
if s == nil {
if other != nil && len(other.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
for shard, ops := range s.Ops {
ops2, ok := other.Ops[shard]
if !ok {
return fmt.Errorf("shard %d missing in other", shard)
}
if len(ops2) != len(ops) {
return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2))
}
for i, op := range ops {
if err := op.Compare(ops2[i]); err != nil {
return fmt.Errorf("shard %d, op %d: %v", shard, i, err)
}
}
}
if len(other.Ops) != len(s.Ops) {
for shard := range other.Ops {
if _, ok := s.Ops[shard]; !ok {
return fmt.Errorf("shard %d missing in self", shard)
}
}
}
return nil
}

275
ingest/op_test.go
View file

@ -1,275 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"math/rand"
"testing"
"github.com/molecula/featurebase/v3/shardwidth"
)
type opShardingTestCase struct {
name string
input *Request
output *ShardedRequest
}
var opShardingTestCases = []opShardingTestCase{
{
name: "sample",
input: &Request{
Ops: []*Operation{
{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0": {
RecordIDs: []uint64{0, 1},
},
"shard0-1": {
RecordIDs: []uint64{
0,
1 << shardwidth.Exponent,
},
},
"shard1": {
RecordIDs: []uint64{
1 << shardwidth.Exponent,
1<<shardwidth.Exponent + 1,
},
},
},
},
{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{1, 2<<shardwidth.Exponent + 1},
},
},
},
},
},
output: &ShardedRequest{
Ops: map[uint64][]*Operation{
0: {
{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0": {
RecordIDs: []uint64{0, 1},
},
"shard0-1": {
RecordIDs: []uint64{
0,
},
},
},
},
{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{1},
},
},
},
},
1: {
{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0-1": {
RecordIDs: []uint64{
1 << shardwidth.Exponent,
},
},
"shard1": {
RecordIDs: []uint64{
1 << shardwidth.Exponent,
1<<shardwidth.Exponent + 1,
},
},
},
},
},
2: {
{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{2<<shardwidth.Exponent + 1},
},
},
},
},
},
},
},
}
func TestOpShardingSmall(t *testing.T) {
codec, _ := NewJSONCodec(nil)
_ = codec.AddIntField("shard0", nil)
_ = codec.AddIntField("shard1", nil)
_ = codec.AddIntField("shard0-1", nil)
_ = codec.AddIntField("shard0-2", nil)
fieldTypes := codec.FieldTypes()
for _, c := range opShardingTestCases {
sharded, err := c.input.ByShard(fieldTypes)
if err != nil {
t.Errorf("sharding: unexpected error %v", err)
}
if err := sharded.Compare(c.output); err != nil {
t.Fatalf("%s: shard: %v", c.name, err)
}
merged := sharded.merge()
if err := c.input.Compare(merged); err != nil {
t.Fatalf("%s: merge: %v", c.name, err)
}
}
}
func TestOpShardingLarge(t *testing.T) {
codec, err := NewJSONCodec(nil)
if err != nil {
t.Fatalf("creating codec: %v", err)
}
_ = codec.AddSetField("set", nil)
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddIntField("int", nil)
// and now we populate encodeTests[1] with a larger pool of data
// if this isn't large enough, the scattering across shards means
// we coincidentally end up with the time quantum field not getting
// tested on simpleSort.
const dataSize = 120000
shardCount := uint64(300) // shards we want to target
passes := uint64(0)
recordIDs := make([]uint64, dataSize)
values := make([]uint64, dataSize)
timeStamps := make([]int64, dataSize)
signedValues := make([]int64, dataSize)
for i := uint64(0); i < dataSize; i++ {
if (i % shardCount) == 0 {
passes++
}
recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes
values[i] = (i % 4)
timeStamps[i] = int64(1234567890e9 + (i * 100e9))
signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
}
op := &Operation{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{},
}
req := &Request{
Ops: []*Operation{op},
}
op.FieldOps["tq"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Values: append([]uint64{}, values...),
Signed: timeStamps,
}
op.FieldOps["int"] = &FieldOperation{
RecordIDs: recordIDs,
Signed: signedValues,
}
// for sets, we want to shuffle things into fewer shards, and ensure
// non-duplication of values within each record, but also have lots
// of duplication of record IDs in the low shards
recordIDs = make([]uint64, dataSize)
values = make([]uint64, dataSize)
valuesPerRecord := uint64(5)
recordsPerShard := dataSize / valuesPerRecord / 30
if recordsPerShard < 1 {
recordsPerShard = 1
}
shard := uint64(0)
nextID := uint64(0)
nextValue := uint64(0)
for i := uint64(0); i < dataSize; i++ {
recordIDs[i] = nextID
values[i] = nextValue + (i % valuesPerRecord)
nextValue++
if nextValue == valuesPerRecord {
nextValue = 0
nextID++
if nextID%(1<<shardwidth.Exponent) == recordsPerShard {
shard++
nextID = (shard << shardwidth.Exponent)
if valuesPerRecord > 1 {
valuesPerRecord--
}
}
}
}
op.FieldOps["set"] = &FieldOperation{
RecordIDs: recordIDs,
Values: values,
}
fieldTypes := codec.FieldTypes()
sharded, err := req.ByShard(fieldTypes)
if err != nil {
t.Errorf("sharding: unexpected error %v", err)
}
merged := sharded.merge()
if err := req.Compare(merged); err != nil {
t.Fatalf("merge comparison: %v", err)
}
}
func TestFancySharding(t *testing.T) {
const shardLimit = 700
const recordCount = 5000
grr := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)}
shards := make([]int, shardLimit)
for j := range f.RecordIDs {
v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent))
f.RecordIDs[j] = v
f.Values[j] = uint64(grr.Int63n(8))
shards[v>>shardwidth.Exponent]++
}
sharded := f.SortToShards()
for shard, data := range sharded {
if len(data.RecordIDs) != shards[shard] {
t.Errorf("shard %d: expected %d items, got %d", shard, shards[shard], len(data.RecordIDs))
}
for _, v := range data.RecordIDs {
if (v >> shardwidth.Exponent) != shard {
t.Errorf("shard %d: got %x, which should be in %d", shard, v, v>>shardwidth.Exponent)
}
}
// expect sorted-ness
data.SortByRecords()
prev := data.RecordIDs[0]
for i, next := range data.RecordIDs[1:] {
if next < prev {
t.Errorf("index %d: prev %d, next %d", i+1, prev, next)
}
prev = next
}
data.SortByValues()
prevV, prevRec := data.Values[0], data.RecordIDs[0]
for i, nextRec := range data.RecordIDs[1:] {
nextV := data.Values[i+1]
if nextV < prevV {
t.Errorf("index %d: prev value %d, next value %d", i+1, prevV, nextV)
}
if nextV == prevV {
if nextRec < prevRec {
t.Errorf("index %d, value %d: prev rec %d, next rec %d", i+1, nextV, prevRec, nextRec)
}
}
prevV = nextV
prevRec = nextRec
}
}
}
}

2
ingest/shard.go
View file

@ -1,2 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest

232
ingest/sort.go
View file

@ -1,232 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
// "math/bits"
// HERE THERE BE DRAGONS
// This is some sorting logic Nia was experimenting with, which we aren't currently
// using, but which beat stdlib sort by a factor-of-several on at least some test
// data, so we aren't deleting it just yet.
// groupIDPairsByShard destructively groups ID pairs by shard.
// The returned slices reference the original pairs slice.
// func groupIDPairsByShard(pairs []IDPair) map[uint64][]IDPair {
// if len(pairs) == 0 {
// return nil
// }
//
// // Sort pairs by shard (in-place radix sort).
// // This may also change the order of pairs within a shard, but that should not matter.
// for {
// // Find the highest bit which needs to be sorted.
// var diffMask uint64
// prev := pairs[0].RecordID
// for _, v := range pairs[1:] {
// if v.RecordID < prev {
// diffMask |= v.RecordID ^ prev
// }
//
// prev = v.RecordID
// }
// diffLen := bits.Len64(diffMask)
// if diffLen <= shardwidth.Exponent {
// // The pairs are sorted by shard.
// break
// }
//
// // Select a right bit shift index such that the highest unsorted bit moves to the 128's place.
// shift := uint(diffLen) - 8
//
// // Create a mask that can be used to group values by sorted bits.
// sortedMask := ^uint64(0) << bits.Len64(diffMask)
//
// for i := 0; i < len(pairs); {
// // Select a group of pairs to sort.
// // While doing so, count the pairs within each bucket.
// j := i
// var buckets [256]struct {
// start, end uint
// }
// for group := pairs[i].RecordID & sortedMask; i < len(pairs) && pairs[i].RecordID&sortedMask == group; i++ {
// buckets[uint8(pairs[i].RecordID>>uint64(shift))].end++
// }
// group := pairs[j:i]
//
// // Assign indices within the group to the buckets.
// {
// var start uint
// for i := range buckets {
// bucket := &buckets[i]
// bucket.start = start
// bucket.end += start
// start = bucket.end
// }
// }
//
// // Split the group into the buckets.
// for i, b := range buckets {
// // There is no need to update the state of the current bucket - we will never reference it again after this.
// i := uint8(i)
// for j := b.start; j < b.end; j++ {
// // This inner loop may run quite a few times for the first few buckets, but no element will be moved more than twice per byte.
// for uint8(group[j].RecordID>>shift) != i {
// // This pair is in the wrong bucket.
// // Swap it into the correct bucket.
// dstBucket := &buckets[uint8(group[j].RecordID>>shift)]
// k := dstBucket.start
// dstBucket.start++
// group[j], group[k] = group[k], group[j]
// }
// }
// }
// }
// }
//
// // Split the pairs by shard.
// shards := make(map[uint64][]IDPair)
// for i := 0; i < len(pairs); {
// // Select the shard.
// shard := pairs[i].RecordID >> shardwidth.Exponent
//
// // Find all pairs in the shard.
// j := i
// incr := 1
// for i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard {
// i += incr
// incr *= 2
// }
// for ; incr > 0; incr /= 2 {
// if i+incr < len(pairs) && pairs[i+incr].RecordID>>shardwidth.Exponent == shard {
// i += incr
// }
// }
// // that found us the last thing in this shard, so...
// i++
//
// // Add the shard, referencing the original slice.
// // This sets the cap so that we dont accidentally overwrite other shards data.
// shards[shard] = pairs[j:i:i]
// }
//
// return shards
// }
// pairsToZigZag converts a set of record-value pairs to Pilosa's zig-zag format.
// This assumes that all pairs are within the same shard.
// func pairsToZigZag(pairs []IDPair) []uint64 {
// const recordMask = (1 << shardwidth.Exponent) - 1
//
// dst := make([]uint64, len(pairs))
// for i, p := range pairs {
// dst[i] = (p.ID << shardwidth.Exponent) | (p.RecordID & recordMask)
// }
//
// return dst
// }
// radixSort64 sorts the data with radix-sort.
// The "shift" is the highest differing bit position, rounded down to a multiple of 8.
// If there are duplicates, this may change the duplicate count for some values.
// func radixSort64(data []uint64, shift uint) {
// if len(data) < 2 {
// return
// }
// if shift <= 8 {
// // The data falls into a 16-bit span, so the remaining digits can be sorted simultaneously with a bitmask.
// maskSort(data)
// return
// }
//
// // Count the values within each bucket.
// var buckets [256]struct {
// start, end uint
// }
// for _, v := range data {
// buckets[uint8(v>>shift)].end++
// }
//
// // Assign indices within the group to the buckets.
// {
// var start uint
// for i := range buckets {
// bucket := &buckets[i]
// bucket.start = start
// bucket.end += start
// start = bucket.end
// }
// }
//
// // Split the data into the buckets.
// var start uint
// for i, b := range buckets {
// // Replace misplaced values until the contents of the bucket all have the correct digit.
// i := uint8(i)
// for j := b.start; j < b.end; j++ {
// // This inner loop may run quite a few times for the first few buckets, but it will never run more than once-per-element-per-byte.
// for uint8(data[j]>>shift) != i {
// // This pair is in the wrong bucket.
// // Swap it into the correct bucket.
// dstBucket := &buckets[uint8(data[j]>>shift)]
// k := dstBucket.start
// dstBucket.start++
// data[j], data[k] = data[k], data[j]
// }
// }
//
// // Sort the contents of the bucket.
// data := data[start:b.end]
// switch {
// case len(data) < 64:
// // Use insertion-sort because the data is too small for a more complex algorithm to be efficient.
// for i := 0; i < len(data); i++ {
// for j := i; j > 0 && data[j-1] > data[j]; j-- {
// data[j-1], data[j] = data[j], data[j-1]
// }
// }
//
// default:
// // Sort the next byte recursively.
// radixSort64(data, shift-8)
// }
// start = b.end
// }
// }
// maskSort sorts integers using a bitmask.
// The values must all fall within one 16-bit span.
// If there are duplicates, this may change the duplicate count for some values.
// func maskSort(data []uint64) {
// if len(data) < 2 {
// return
// }
//
// base := data[0] &^ ((1 << 16) - 1)
//
// // Dump everything into the mask.
// var mask [(1 << 16) / 64]uint64
// for _, v := range data {
// mask[uint16(v)/64] |= 1 << (v % 64)
// }
//
// // Scan through the set bits in the mask.
// k := 0
// for i, w := range mask {
// for w != 0 {
// j := bits.TrailingZeros64(w)
// w &^= 1 << j
// data[k] = 64*uint64(i) + uint64(j) + base
// k++
// }
// }
//
// if k < len(data) {
// // Copy the ending value to fill up the rest of the space.
// // This happens once for each duplicate value.
// endVal := data[k-1]
// for k < len(data) {
// data[k] = endVal
// k++
// }
// }
// }

218
ingest/sort_test.go
View file

@ -1,218 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"testing"
)
const sampleSize = 1000000
var sampleSortingData = createSampleFieldData()
func createSampleFieldData() *FieldOperation {
fo := &FieldOperation{}
fo.RecordIDs = make([]uint64, sampleSize)
fo.Values = make([]uint64, sampleSize)
fo.Signed = make([]int64, sampleSize)
for i := range fo.RecordIDs {
fo.RecordIDs[i] = (uint64(i) * 63 * 3456789) % 50000000
fo.Values[i] = (uint64(i) * 6) % 8
fo.Signed[i] = ((int64(i) * 17) % 15) - 8
}
return fo
}
func benchmarkOneSort(b *testing.B, fo *FieldOperation) {
for i := 0; i < b.N; i++ {
b.StopTimer()
sortable := fo.clone()
b.StartTimer()
_ = sortable.SortToShards()
}
}
func BenchmarkSortFieldOp(b *testing.B) {
b.Run("full", func(b *testing.B) {
f2 := *sampleSortingData
benchmarkOneSort(b, &f2)
})
b.Run("nosign", func(b *testing.B) {
f2 := *sampleSortingData
f2.Signed = nil
benchmarkOneSort(b, &f2)
})
b.Run("signonly", func(b *testing.B) {
f2 := *sampleSortingData
f2.Values = nil
benchmarkOneSort(b, &f2)
})
}
// func BenchmarkSort64(b *testing.B) {
// gen := func(n, width uint64) func() []uint64 {
// var data []uint64
// var once sync.Once
// return func() []uint64 {
// once.Do(func() {
// data = make([]uint64, n)
// var rng rand.PCGSource
// rng.Seed(9001)
// for i := range data {
// data[i] = rng.Uint64() % width
// }
// })
//
// return data
// }
// }
//
// algos := []struct {
// name string
// maxn uint64
// fn func([]uint64) []uint64
// }{
// {
// name: "stdlib",
// maxn: 1024 * 1024 * 1024,
// fn: stdSort,
// },
// {
// name: "heap",
// maxn: 1024 * 1024 * 1024,
// fn: heapSort,
// },
// {
// name: "radix-insertion-mask",
// maxn: 1024 * 1024 * 1024,
// fn: dedupSort64,
// },
// }
//
// widths := []struct {
// name string
// width uint64
// }{
// {"64", 64},
// {"1K", 1024},
// {"64K", 64 * 1024},
// {"1M", 1024 * 1024},
// {"16M", 16 * 1024 * 1024},
// {"128M", 128 * 1024 * 1024},
// {"1B", 1024 * 1024 * 1024},
// }
//
// counts := []struct {
// name string
// n uint64
// }{
// {"64", 64},
// {"1K", 1024},
// {"64K", 64 * 1024},
// {"1M", 1024 * 1024},
// {"4M", 4 * 1024 * 1024},
// {"16M", 16 * 1024 * 1024},
// {"64M", 64 * 1024 * 1024},
// {"256M", 256 * 1024 * 1024},
// }
//
// for _, width := range widths {
// width := width
// b.Run(width.name, func(b *testing.B) {
// for _, count := range counts {
// if count.n > width.width {
// continue
// }
//
// count := count
// b.Run(count.name, func(b *testing.B) {
// datasrc := gen(count.n, width.width)
// for _, alg := range algos {
// if count.n > alg.maxn {
// continue
// }
//
// alg := alg
// b.Run(alg.name, func(b *testing.B) {
// data := datasrc()
// buf := make([]uint64, len(data))
// b.SetBytes(8 * int64(len(buf)))
//
// b.StopTimer()
// b.ResetTimer()
//
// for i := 0; i < b.N; i++ {
// copy(buf, data)
// b.StartTimer()
// alg.fn(buf)
// b.StopTimer()
// }
// })
// }
// })
// }
// })
// }
// }
//
// func heapSort(data []uint64) []uint64 {
// for i, v := range data {
// for i > 0 && v > data[(i-1)/2] {
// data[i] = data[(i-1)/2]
// i = (i - 1) / 2
// }
// data[i] = v
// }
// {
// heap := data
// for len(heap) > 1 {
// heap[0], heap[len(heap)-1] = heap[len(heap)-1], heap[0]
// heap = heap[:len(heap)-1]
// i := 0
// for {
// max := i
// if r := 2*i + 1; r < len(heap) && heap[r] > heap[max] {
// max = r
// }
// if l := 2*i + 2; l < len(heap) && heap[l] > heap[max] {
// max = l
// }
// if max == i {
// break
// }
//
// heap[max], heap[i] = heap[i], heap[max]
// i = max
// }
// }
// }
//
// j := 1
// prev := data[0]
// for _, v := range data[1:] {
// if v == prev {
// continue
// }
//
// data[j] = v
// prev = v
// }
//
// return data[:j]
// }
//
// func stdSort(data []uint64) []uint64 {
// sort.Slice(data, func(i, j int) bool { return data[i] < data[j] })
//
// j := 1
// prev := data[0]
// for _, v := range data[1:] {
// if v == prev {
// continue
// }
//
// data[j] = v
// prev = v
// }
//
// return data[:j]
// }

124
ingest/translate_test.go
View file

@ -1,124 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"fmt"
"testing"
)
// stableTranslator implements a key translator that can be reused and
// will continue to give the same keys for the same values. Possibly
// surprisingly, it will invent new keys for IDs it is asked about but
// hasn't seen. This allows us to give a codec which would use keys on
// translation a request which contains arbitrary numbers, and request
// text that would parse into that request.
type stableTranslator struct {
in map[string]uint64
out map[uint64]string
next uint64
}
func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
ret := make(map[string]uint64, len(keys))
for _, key := range keys {
if existing, ok := s.in[key]; ok {
ret[key] = existing
continue
}
id := s.next
// but what if someone already translated that ID, so now it already
// exists?
if _, ok := s.out[id]; ok {
for k := range s.out {
if k > id {
id = k
}
}
// one larger than the largest we already have. this could
// wrap around, in which case, it's your own fault.
id++
}
s.next = id + 1
s.in[key] = id
s.out[id] = key
ret[key] = id
}
return ret, nil
}
func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
ret := make(map[uint64]string, len(ids))
for _, id := range ids {
if existing, ok := s.out[id]; ok {
ret[id] = existing
continue
}
key := fmt.Sprintf("k-%d", id)
s.in[key] = id
s.out[id] = key
ret[id] = key
if id >= s.next {
s.next = id + 1
}
}
return ret, nil
}
// newStableTranslator produces a translator which can translate forwards
// and backwards and invent new things if it needs to. Don't use this.
func newStableTranslator() *stableTranslator {
return &stableTranslator{
in: make(map[string]uint64),
out: make(map[uint64]string),
}
}
func TestTranslateReuse(t *testing.T) {
// the original stable-translator design had a flaw in that it
// assumed that each new ID would always come from a string translation,
// never from a key translation, and that they'd show up sequentially.
tr := newStableTranslator()
orig, err := tr.TranslateIDs(1, 2)
tr.next = 1 // intentionally break the translation logic for testing purposes
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var s [6]string
stash := s[:0]
for _, v := range orig {
stash = append(stash, v)
}
for i := range s[len(orig):] {
stash = append(stash, fmt.Sprintf("key-%d", i))
}
keys, err := tr.TranslateKeys(stash...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ids := make([]uint64, 0, len(keys))
for _, v := range keys {
ids = append(ids, v)
}
idMap, err := tr.TranslateIDs(ids...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for k, v := range keys {
if idMap[v] != k {
t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q",
k, v, v, idMap[v])
}
}
for id, key := range idMap {
if keys[key] != id {
t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d",
id, key, key, keys[key])
}
}
for id, key := range orig {
if keys[key] != id {
t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d",
id, key, key, keys[key])
}
}
}

243
ingest/update.go
View file

@ -1,243 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"github.com/molecula/featurebase/v3/roaring"
)
// ShardUpdate is an update request for a shard.
type ShardUpdate struct {
// TODO: include schema version
// Sets are the updates to set fields.
Sets map[string]SetMatrixUpdate
// Mutexes are the updates to mutex fields.
Mutexes map[string]MutexMatrixUpdate
// TimeTensors are the updates to time fields.
TimeTensors map[string]TimeTensorUpdate
// Ints are the updates to int fields.
Ints map[string]IntUpdate
}
// Convert the ID vector to a raw update that can be imported.
// All records are assumed to fall within a shard.
// func (vec IDVector) Convert() (MutexMatrixUpdate, error) {
// // Before converting the vector, do a sanity-check for duplicates.
// dedup := make(map[uint64]struct{}, len(vec.Updates)+len(vec.Clears))
// for _, p := range vec.Updates {
// if _, ok := dedup[p.RecordID]; !ok {
// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates")
// }
//
// dedup[p.RecordID] = struct{}{}
// }
// for _, p := range vec.Clears {
// if _, ok := dedup[p]; !ok {
// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates")
// }
//
// dedup[p] = struct{}{}
// }
//
// // Convert the updates to a bitmap.
// updates, err := idPairsToBitmap(vec.Updates, false)
// if err != nil {
// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex updates")
// }
//
// // Convert the clears to a bitmap.
// clears, err := idListToBitmap(vec.Clears, false)
// if err != nil {
// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex clears")
// }
//
// // Realign clears bitmap to start of shard.
// if min, ok := clears.Min(); ok {
// min &^= (1 << shardwidth.Exponent) - 1
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
// }
//
// return MutexMatrixUpdate{
// Update: updates,
// Clear: clears,
// }, nil
// }
// MutexMatrixUpdate is an encoded update request for a mutex-type view.
type MutexMatrixUpdate struct {
// Update is a bitmap of new mutex values.
// This bitmap will not include any records in the clear bitmap.
Update *roaring.Bitmap
// Clear is a set of records for which all values should be removed.
Clear *roaring.Bitmap
}
// Convert the time vectors to a raw update that can be imported.
// All records are assumed to fall within a shard.
// func (vec TimeIDSetsVector) Convert() (TimeTensorUpdate, error) {
// // Convert the clears to a bitmap.
// clears, err := idListToBitmap(vec.Clears, false)
// if err != nil {
// return TimeTensorUpdate{}, errors.Wrap(err, "encoding time tensor clears")
// }
//
// // Realign clears bitmap to start of shard.
// if min, ok := clears.Min(); ok {
// min &^= (1 << shardwidth.Exponent) - 1
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
// }
//
// // Convert the adds to bitmaps.
// quantums := make(map[string]*roaring.Bitmap)
// for t, vec := range vec.Quantums {
// adds, err := idPairsToBitmap(vec, true)
// if err != nil {
// return TimeTensorUpdate{}, errors.Wrap(err, "encoding time adds")
// }
//
// quantums[t] = adds
// }
//
// return TimeTensorUpdate{
// Quantums: quantums,
// Clear: clears,
// }, nil
// }
// TimeTensorUpdate is an encoded update request for a time field within a shard.
type TimeTensorUpdate struct {
// Quantums are the component set matrix adds, grouped by time quantum.
Quantums map[string]*roaring.Bitmap
// Clear is a set of records for which all values should be removed.
// This must not overlap with any values in the remove or clear sections of any quantum.
Clear *roaring.Bitmap
}
// Convert the ID set vector to a raw update that can be imported.
// All records are assumed to fall within a shard.
// func (vec IDSetVector) Convert() (SetMatrixUpdate, error) {
// // Convert the clears to a bitmap.
// clears, err := idListToBitmap(vec.Clears, false)
// if err != nil {
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set clears")
// }
//
// // Realign clears bitmap to start of shard.
// if min, ok := clears.Min(); ok {
// min &^= (1 << shardwidth.Exponent) - 1
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
// }
//
// // Convert the adds to a bitmap.
// adds, err := idPairsToBitmap(vec.Adds, false)
// if err != nil {
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set adds")
// }
//
// // Verify that the removes do not include the cleared records.
// if clears.Any() {
// for i := 0; i < len(vec.Removes); {
// recID := vec.Removes[i].RecordID
// if clears.Contains(recID) {
// return SetMatrixUpdate{}, errors.New("removed element duplicated with a clear")
// }
//
// i++
// for i < len(vec.Removes) && vec.Removes[i].RecordID == recID {
// i++
// }
// }
// }
//
// // Convert the removes to a bitmap.
// removes, err := idPairsToBitmap(vec.Removes, false)
// if err != nil {
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set removes")
// }
//
// // Check that no bits are both added and removed.
// if adds.IntersectionCount(removes) > 0 {
// return SetMatrixUpdate{}, errors.New("set adds and removes overlap")
// }
//
// return SetMatrixUpdate{
// Add: adds,
// Remove: removes,
// Clear: clears,
// }, nil
// }
// SetMatrixUpdate is an encoded update request for a set-type (set/mutex) view.
type SetMatrixUpdate struct {
// Add is a bitmap to union the matrix against.
Add *roaring.Bitmap
// Remove is a bitmap to difference out of the matrix.
// No values will be present in both add and remove.
// This bitmap will not include any records in the clear bitmap.
Remove *roaring.Bitmap
// Clear is a set of records for which all values should be removed.
// This is applied before add operations.
Clear *roaring.Bitmap
}
// func idPairsToBitmap(pairs []IDPair, allowDup bool) (*roaring.Bitmap, error) {
// return idListToBitmap(pairsToZigZag(pairs), allowDup)
// }
//
// func idListToBitmap(ids []uint64, allowDup bool) (*roaring.Bitmap, error) {
// vals := dedupSort64(ids)
// if len(vals) != len(ids) && !allowDup {
// return nil, errors.New("input contains duplicate values")
// }
//
// // TODO: make the roaring package actually handle this well
// return roaring.NewBitmap(vals...), nil
// }
// Convert the int vector to a raw update that can be imported.
// All records are assumed to fall within a shard.
// func (vec IntVector) Convert() (IntUpdate, error) {
// // Convert the clears to a bitmap.
// clears, err := idListToBitmap(vec.Clears, false)
// if err != nil {
// return IntUpdate{}, errors.Wrap(err, "encoding int clears")
// }
//
// // Realign clears bitmap to start of shard.
// if min, ok := clears.Min(); ok {
// min &^= (1 << shardwidth.Exponent) - 1
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
// }
//
// // Convert updates to a bitmap.
// updates, err := idListToBitmap(intToBSI(vec.Updates), false)
// if err != nil {
// return IntUpdate{}, errors.Wrap(err, "encoding int updates")
// }
//
// // Check that no updated records are also being cleared.
// if updates.IntersectionCount(clears) > 0 {
// return IntUpdate{}, errors.New("update duplicated with a clear")
// }
//
// return IntUpdate{
// BSI: updates,
// Clear: clears,
// }, nil
// }
// IntUpdate is an encoded update request for a BSI (int/timestamp/etc.) view.
type IntUpdate struct {
// BSI is a bitmap of new BSI data to overwrite existing values.
BSI *roaring.Bitmap
// Clear is a set of records to assign null.
Clear *roaring.Bitmap
}

122
ingest/vec.go
View file

@ -1,122 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"fmt"
"reflect"
"strconv"
"unsafe"
)
// StringTable is a mapping of strings to temporary IDs.
// All mapped-to IDs fall in the range [0, len). The zero value,
// a nil map, instead just parses the numbers.
//
// We keep the array of names in creation order because we want reproducibility;
// the first key we see is always key 0. Otherwise, the keys are created in
// arbitrary orders.
type StringTable struct {
names []string
values map[string]uint64
}
// NewStringTable just creates a string table with a non-nil map.
func NewStringTable() *StringTable {
return &StringTable{values: map[string]uint64{}}
}
// unsafe is
func pretendByteIsString(data []byte) (result string) {
dH := (*reflect.SliceHeader)(unsafe.Pointer(&data))
sH := (*reflect.StringHeader)(unsafe.Pointer(&result))
sH.Data = dH.Data
sH.Len = dH.Len
return result
}
// ID returns an ID associated to the string, adding it to the table if it is not already present,
// or parsing an integer if there's no table.
func (tbl *StringTable) ID(in []byte) (uint64, error) {
str := pretendByteIsString(in)
if tbl != nil {
id, ok := tbl.values[str]
if !ok {
id = uint64(len(tbl.values))
tbl.values[str] = id
tbl.names = append(tbl.names, str)
}
return id, nil
}
return strconv.ParseUint(str, 10, 64)
}
// SignedID returns an ID associated to the string, adding it to the table if it is not already present,
// or parsing an integer if there's no table. It yields signed values only.
func (tbl *StringTable) IntID(in []byte) (int64, error) {
str := pretendByteIsString(in)
if tbl != nil {
id, ok := tbl.values[str]
if !ok {
id = uint64(len(tbl.values))
tbl.values[str] = id
tbl.names = append(tbl.names, str)
}
return int64(id), nil
}
return strconv.ParseInt(str, 10, 64)
}
// MapForStringTable, given a string table mapping strings to consecutive
// integers and a translation function from strings to "real" keys, yields
// a translation/lookup slice. If it cannot translate all the keys, it
// returns an error.
func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) {
lookedUp, err := keys.TranslateKeys(tbl.names...)
if err != nil {
return nil, err
}
if len(lookedUp) != len(tbl.names) {
return nil, fmt.Errorf("missing keys: expected %d keys, got %d", len(tbl.values), len(lookedUp))
}
out := make([]uint64, len(tbl.names))
for i, v := range tbl.names {
out[i] = lookedUp[v]
}
return out, nil
}
// translateSigned replaces values from 0 to len(mapping)-1 with the
// elements of mapping. It yields an error if any values aren't
// mapped.
func translateSigned(mapping []uint64, values []int64) error {
oops := 0
for i, v := range values {
if v >= int64(len(mapping)) {
oops++
} else {
values[i] = int64(mapping[v])
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
}
return nil
}
// translateUnsigned replaces values from 0 to len(mapping)-1 with the
// elements of mapping. It yields an error if any values aren't
// mapped.
func translateUnsigned(mapping []uint64, values []uint64) error {
oops := 0
for i, v := range values {
if v >= uint64(len(mapping)) {
oops++
} else {
values[i] = mapping[v]
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
}
return nil
}

101
ingest/vec_test.go
View file

@ -1,101 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ingest
import (
"errors"
"testing"
)
type badTranslator struct{}
func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
if len(keys) == 0 {
return nil, errors.New("no keys")
}
m := make(map[string]uint64)
skip := true
for i, k := range keys {
if skip {
skip = false
continue
}
m[k] = uint64(i)
}
out := make([]uint64, len(keys)-1)
for i := range out {
out[i] = uint64(i)
}
return nil, nil
}
func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) {
return nil, nil
}
func TestStringTableErrors(t *testing.T) {
tbl := NewStringTable()
btr := badTranslator{}
_, keyErr := tbl.MakeIDMap(btr)
if keyErr == nil {
t.Fatalf("expected error passed up from failed translate, didn't get it")
}
a1, err := tbl.ID([]byte("a"))
if err != nil {
t.Fatalf("getting translation for key: %v", err)
}
b1, err := tbl.ID([]byte("b"))
if err != nil {
t.Fatalf("getting translation for key: %v", err)
}
_, keyErr = tbl.MakeIDMap(btr)
if keyErr == nil {
t.Fatalf("expected error for short translate, didn't get it")
}
tr := newStableTranslator()
_, err = tr.TranslateKeys("c", "d")
if err != nil {
t.Fatalf("translating stray keys: %v", err)
}
m, err := tbl.MakeIDMap(tr)
if err != nil {
t.Fatalf("creating lookup: %v", err)
}
var y = []uint64{a1, b1}
err = translateUnsigned(m, y)
if err != nil {
t.Fatalf("unexpected unsigned translation error: %v", err)
}
trResults, err := tr.TranslateKeys("a", "b")
if err != nil {
t.Fatalf("unexpected translation error: %v", err)
}
if y[0] != trResults["a"] {
t.Fatalf("expected %d, got %d", trResults["a"], y[0])
}
if y[1] != trResults["b"] {
t.Fatalf("expected %d, got %d", trResults["b"], y[1])
}
y[0] = a1
y[1] = (a1 + b1 + 1) // assumed not to be any of them
err = translateUnsigned(m, y)
if err == nil {
t.Fatalf("no error from translating invalid table")
}
z := []int64{int64(a1), int64(b1)}
err = translateSigned(m, z)
if err != nil {
t.Fatalf("unexpected unsigned translation error: %v", err)
}
if uint64(z[0]) != trResults["a"] {
t.Fatalf("expected %d, got %d", trResults["a"], z[0])
}
if uint64(z[1]) != trResults["b"] {
t.Fatalf("expected %d, got %d", trResults["b"], z[1])
}
z[0] = int64(a1)
z[1] = int64(a1 + b1 + 1) // assumed not to be any of them
err = translateSigned(m, z)
if err == nil {
t.Fatalf("no error from translating invalid table")
}
}

448
ingest_test.go
View file

@ -1,448 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa_test
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/test"
"github.com/pkg/errors"
)
// For ingest API testing, we want to do tests which have a known
// schema, no existing data before we start, and perform ingests and
// then do queries.
//
// A good starting point for this would be a fairly simple file
// divided into sections which are just the JSON text of the data
// we want to be working with, or the PQL queries we want to run,
// or their expected results.
//
// So, roughly like this:
//
// schema:
// {
// "index-name": "example",
// "primary-key-type": "string",
// "fields": [
// {
// "field-name": "set",
// "field-type": "id",
// "field-options": { "cache-type": "none" }
// }
// ]
// }
// ingest:
// [
// {
// "action": "set",
// "records": {
// "1": {
// "set": [ 2 ],
// }
// }
// }
// ]
// queries:
// Row(set=2):
// [1]
//
// Additionally, the names "schema-error" and "ingest-error" are taken to
// represent a schema, or data set, which is expected to produce an error.
// For instance:
//
// ingest-error:
// [ { "action": puppy }
//
// In this case, it would be considered a test failure if an ingest request
// did NOT fail.
// Lines starting with #
// ingestSchemaPartial represents the only part of a schema we need to
// know about in order to undo its creation of a schema for use in a
// test case.
type ingestSchemaPartial struct {
IndexName string `json:"index-name"`
}
type ingestActionKind int
const (
ingestActionNone = ingestActionKind(iota)
ingestActionSchema
ingestActionIngest
ingestActionSchemaError
ingestActionIngestError
ingestActionQueries
)
var ingestActionKinds = map[string]ingestActionKind{
"schema": ingestActionSchema,
"ingest": ingestActionIngest,
"schema-error": ingestActionSchemaError,
"ingest-error": ingestActionIngestError,
"queries": ingestActionQueries,
}
var ingestActionKindNames = map[ingestActionKind]string{}
type testCaseAction struct {
kind ingestActionKind
comment []byte
lineStart int
lineEnd int
data []byte
}
type liner struct {
data []byte
at int
start int
remaining []byte
line []byte
}
func newLiner(data []byte) *liner {
return &liner{data: data, at: 0, remaining: data}
}
func (l *liner) next() bool {
if len(l.remaining) == 0 {
return false
}
foundNL := true
nextNL := bytes.IndexByte(l.remaining, '\n')
if nextNL == -1 {
foundNL = false
nextNL = len(l.remaining)
}
l.line = l.remaining[:nextNL]
// move past the newline we found, if we found one
if foundNL {
nextNL++
}
l.start, l.at = l.at, l.at+nextNL
l.remaining = l.data[l.at:]
return true
}
func (l *liner) text() (line []byte, start int, end int) {
return l.line, l.start, l.at
}
// parseExpectedResults handles something that looks like
// [1, 2, 3] or ["a", "b", "c"]. It does not handle things like
// quotes within strings, etcetera.
func parseExpectedResults(data []byte) (ints []uint64, keys []string, err error) {
if len(data) < 2 || data[0] != '[' || data[len(data)-1] != ']' {
return nil, nil, errors.New("expecting [] results")
}
words := bytes.Split(data[1:len(data)-1], []byte{','})
if len(words) == 1 && len(words[0]) == 0 {
return nil, nil, nil
}
for _, word := range words {
word = bytes.TrimSpace(word)
if len(word) == 0 {
return nil, nil, errors.New("found empty word expecting result")
}
if word[0] == '"' {
keys = append(keys, string(word[1:len(word)-1]))
continue
}
v, err := strconv.ParseInt(string(word), 10, 64)
if err != nil {
return nil, nil, err
}
ints = append(ints, uint64(v))
}
if len(ints) > 0 && len(keys) > 0 {
return nil, nil, errors.New("mixed integers and strings are invalid")
}
return ints, keys, err
}
func testQueries(t *testing.T, ctx context.Context, cmd *test.Command, index string, action testCaseAction) {
qcx := cmd.API.Txf().NewQcx()
defer func() {
if err := qcx.Finish(); err != nil {
t.Fatalf("finishing qcx: %v", err)
}
}()
l := newLiner(action.data)
for l.next() {
query, _, _ := l.text()
if !l.next() {
t.Fatalf("processing query list: no expected after %q", query)
}
expected, _, _ := l.text()
ints, keys, err := parseExpectedResults(expected)
if err != nil {
t.Fatalf("processing query list: invalid expected results %q", expected)
}
t.Logf("expecting %q -> %s", query, expected)
res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: string(query)})
if err != nil {
t.Errorf("query: %v", err)
}
if len(res.Results) != 1 {
t.Fatalf("expected one result per query, got %d results", len(res.Results))
}
var row *pilosa.Row
var ok bool
if row, ok = res.Results[0].(*pilosa.Row); !ok {
t.Fatalf("expected results to be a row")
}
if ints != nil {
cols := row.Columns()
if len(cols) != len(ints) {
t.Fatalf("wrong number of values, expected %d, got %d", len(ints), len(cols))
}
for i := range cols {
if ints[i] != cols[i] {
t.Fatalf("result %d: expected %d, got %d", i, ints[i], cols[i])
}
}
}
// key return value is unpredictable, so...
if keys != nil {
seen := make(map[string]struct{})
for _, k := range keys {
seen[k] = struct{}{}
}
for _, k := range row.Keys {
if _, ok := seen[k]; !ok {
t.Fatalf("unexpected result key %q", k)
}
delete(seen, k)
}
for k := range seen {
t.Fatalf("expected result to contain %q, but did not get it", k)
}
}
}
}
// testOneIngestTestcase runs a set of actions, then cleans up after itself
func testOneIngestTestcase(t *testing.T, ctx context.Context, cmd *test.Command, tcpath string) {
data, err := os.ReadFile(tcpath)
if err != nil {
t.Fatalf("reading %q: %v", tcpath, err)
}
var actions []testCaseAction
var action testCaseAction
l := newLiner(data)
var line []byte
var start, lineStart, lineEnd int
lineCount := 0
for l.next() {
line, lineStart, lineEnd = l.text()
lineCount++
if colon := bytes.IndexByte(line, ':'); colon != -1 {
if kind, ok := ingestActionKinds[string(line[:colon])]; ok {
if action.kind != ingestActionNone {
action.data = data[start:lineStart]
actions = append(actions, action)
action.lineEnd = lineCount - 1
} else {
if lineStart != 0 {
t.Logf("warning: %d bytes with no action type before first action", lineStart)
}
}
start = lineEnd
action.data = nil
action.kind = kind
if line[len(line)-1] == ':' {
action.comment = line[:len(line)-1]
} else {
action.comment = line
}
action.lineStart = lineCount
}
}
}
if action.kind != ingestActionNone {
action.lineEnd = lineCount - 1
action.data = data[start:]
actions = append(actions, action)
}
var mostRecentIndex string
seenIndexes := map[string]struct{}{}
cli := cmd.Client()
created := map[string][]string{}
defer func() {
t.Logf("deleting created indexes/fields:")
for k, v := range created {
if len(v) == 0 {
t.Logf(" index: %q", k)
if err := cmd.API.DeleteIndex(ctx, k); err != nil {
t.Errorf("deleting index %q: %v", k, err)
}
} else {
t.Logf(" fields in %q: %q", k, v)
for _, field := range v {
if err := cmd.API.DeleteField(ctx, k, field); err != nil {
t.Errorf("deleting field %q from %q: %v", field, k, err)
}
}
}
}
}()
noticeCreation := func(newlyCreated map[string][]string) {
for k, v := range newlyCreated {
if len(v) == 0 {
if existing, ok := created[k]; ok {
if len(existing) > 0 {
t.Fatalf("creation reports index %q newly created, but we created fields %q in it previously",
k, existing)
}
}
// create an empty list, indicating that the whole index is
// believed nil
created[k] = nil
continue
}
if existing, ok := created[k]; ok {
if len(existing) == 0 {
// we'll delete this index anyway, don't need to delete fields in it
continue
}
created[k] = append(existing, v...)
continue
}
created[k] = v
}
}
for _, action := range actions {
t.Logf("%s, lines %d-%d", action.comment, action.lineStart, action.lineEnd)
switch action.kind {
case ingestActionSchema:
var scratch ingestSchemaPartial
err = json.Unmarshal(action.data, &scratch)
if err != nil {
t.Fatalf("couldn't parse schema data: %v", err)
}
if scratch.IndexName == "" {
t.Fatalf("test case must provide an index name")
}
// stash the string from the schema, because we
// might need it later
mostRecentIndex = scratch.IndexName
seenIndexes[mostRecentIndex] = struct{}{}
var newlyCreated map[string][]string
newlyCreated, err = cli.IngestSchema(ctx, nil, action.data)
if err != nil {
t.Fatalf("executing schema: %v", err)
}
noticeCreation(newlyCreated)
case ingestActionSchemaError:
var scratch ingestSchemaPartial
err = json.Unmarshal(action.data, &scratch)
if err != nil {
t.Logf("got expected error from schema: %v", err)
break
}
var newlyCreated map[string][]string
newlyCreated, err = cli.IngestSchema(ctx, nil, action.data)
if err != nil {
t.Logf("got expected error from schema: %v", err)
break
}
noticeCreation(newlyCreated)
t.Fatalf("expected error from schema, didn't get it")
case ingestActionIngest:
func() {
qcx := cmd.API.Txf().NewQcx()
var err error
defer func() {
if err == nil {
qcx.Abort()
return
}
if err := qcx.Finish(); err != nil {
t.Fatalf("finishing qcx: %v", err)
}
}()
err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data))
if err != nil {
t.Fatalf("importing data: %v", err)
}
}()
case ingestActionIngestError:
func() {
qcx := cmd.API.Txf().NewQcx()
var err error
defer func() {
if err == nil {
qcx.Abort()
return
}
if err := qcx.Finish(); err != nil {
t.Fatalf("finishing qcx: %v", err)
}
}()
err = cmd.API.IngestOperations(ctx, qcx, mostRecentIndex, bytes.NewBuffer(action.data))
if err != nil {
t.Logf("got expected error from ingest: %v", err)
return
}
t.Fatalf("expected error from ingest, didn't get it")
}()
case ingestActionQueries:
testQueries(t, ctx, cmd, mostRecentIndex, action)
}
}
}
// TestIngestTestcases reads sample test cases from a test data directory
// and evaluates them.
func TestIngestTestcases(t *testing.T) {
_ = &ingest.Operation{}
var testcases []string
if len(ingestActionKindNames) == 0 {
for k, v := range ingestActionKinds {
ingestActionKindNames[v] = k
}
}
err := filepath.Walk("ingest_testdata", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if strings.HasSuffix(path, ".tc") {
testcases = append(testcases, path)
}
return nil
})
if err != nil {
t.Fatalf("looking for test cases: %v", err)
}
if len(testcases) == 0 {
t.Fatalf("no ingest test cases found")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 3)
defer c.Close()
coord := c.GetPrimary()
for _, tc := range testcases {
t.Run(strings.TrimSuffix(tc, ".tc"), func(t *testing.T) {
testOneIngestTestcase(t, ctx, coord, tc)
})
}
}

17
ingest_testdata/bool.tc
View file

@ -1,17 +0,0 @@
schema:
{
"index-name": "example",
"index-action": "create",
"primary-key-type": "uint",
"fields": [
{
"field-name": "tf",
"field-type": "bool"
}
]
}
ingest:
[{"action": "write", "records": {"2": { "tf": true }}}]
queries:
Row(tf=true)
[2]

101
ingest_testdata/expect_errors.tc
View file

@ -1,101 +0,0 @@
schema-error: fail to create field
{
"index-name": "examplekeys",
"index-action": "create",
"primary-key-type": "string",
"fields": [
{
"field-name": "cookie",
"field-type": "id",
"field-options": { "cache-type": "none" }
},
{
"field-name": "set",
"field-type": "id",
"field-options": { "cache-type": "nun" }
}
]
}
schema: confirm index was not created because field failed
{
"index-name": "examplekeys",
"index-action": "create",
"primary-key-type": "string",
"fields": [
{
"field-name": "set",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
schema-error: can't recreate index
{
"index-name": "examplekeys",
"index-action": "create",
"primary-key-type": "string",
"fields": [
{
"field-name": "newfield",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
schema: can add field to existing index
{
"index-name": "examplekeys",
"index-action": "ensure",
"primary-key-type": "string",
"fields": [
{
"field-name": "newfield",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
schema-error: second field failing in existing index
{
"index-name": "examplekeys",
"index-action": "ensure",
"primary-key-type": "string",
"fields": [
{
"field-name": "addokay",
"field-type": "id",
"field-options": { "cache-type": "none" }
},
{
"field-name": "addfail",
"field-type": "id",
"field-options": { "cache-type": "nun" }
}
]
}
schema: verify that fields we think exist do exist
{
"index-name": "examplekeys",
"index-action": "require",
"primary-key-type": "string",
"fields": [
{
"field-name": "newfield",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
schema-error: successful field deleted anyway because second field failed
{
"index-name": "examplekeys",
"index-action": "require",
"primary-key-type": "string",
"fields": [
{
"field-name": "addokay",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}

32
ingest_testdata/keyed.tc
View file

@ -1,32 +0,0 @@
schema:
{
"index-name": "examplekeys",
"index-action": "create",
"primary-key-type": "string",
"fields": [
{
"field-name": "set",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
ingest:
[
{
"action": "set",
"records": {
"a": {
"set": [ 2 ],
},
"b": {
"set": 3,
}
}
}
]
queries:
Row(set=2)
["a"]
Union(Row(set=3),Row(set=2))
["a","b"]

99
ingest_testdata/sample.tc
View file

@ -1,99 +0,0 @@
schema:
{
"index-name": "example",
"index-action": "create",
"primary-key-type": "uint",
"fields": [
{
"field-name": "set",
"field-type": "id",
"field-options": { "cache-type": "none" }
}
]
}
ingest:
[
{
"action": "set",
"records": {
"1": {
"set": [ 2 ],
},
"2": {
"set": 3,
}
}
}
]
queries:
Row(set=2)
[1]
Union(Row(set=3),Row(set=2))
[1,2]
schema-error:
{
"index-name": "example",
"primary-key-type": "uint",
"index-action": "require",
"fields": [
{
"field-name": "setkey",
"field-type": "string",
"field-options": { "cache-type": "none" }
}
]
}
schema:
{
"index-name": "example",
"primary-key-type": "uint",
"index-action": "ensure",
"fields": [
{
"field-name": "setkey",
"field-type": "string",
"field-options": { "cache-type": "none" }
}
]
}
ingest:
[
{
"action": "set",
"records": {
"1": {
"setkey": [ "a" ],
},
"2": {
"setkey": "b",
}
}
}
]
queries:
Row(setkey="a")
[1]
ingest-error:
[
{
"action": "setkeys",
"records": {
"a": {
"setkey": [ "a" ],
},
"b": {
"setkey": "b",
}
}
}
]
ingest:
[
{
"action": "delete",
"record_ids": [ 1 ]
}
]
queries:
Row(setkey="a")
[]

119
internal_client.go
View file

@ -21,7 +21,6 @@ import (
"github.com/hashicorp/go-retryablehttp"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/logger"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/tracing"
@ -392,124 +391,6 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
return rsp.Indexes, nil
}
// IngestSchema uses the new schema ingest endpoint. It returns a
// map from index names to fields created within them; note that if the
// entire index was created, the list of fields is empty. The intended
// usage is cleaning up after creating the indexes, so if you create the
// index, you don't need to delete the fields, but if you created fields
// within an existing index, you should delete those fields but not the
// whole index.
func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []byte) (created map[string][]string, err error) {
if uri == nil {
uri = c.defaultURI
}
u := uri.Path("/internal/schema")
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
AddAuthToken(ctx, &req.Header)
resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
if err != nil {
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
buf, err = io.ReadAll(resp.Body)
if resp.StatusCode != 200 {
if err != nil {
return nil, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status)
}
var msg string
// try to decode a JSON response
var sr successResponse
qr := &QueryResponse{}
if err = json.Unmarshal(buf, &sr); err == nil {
msg = sr.Error.Error()
} else if err := c.serializer.Unmarshal(buf, qr); err == nil {
msg = qr.Err.Error()
} else {
msg = string(buf)
}
return nil, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg)
}
// this is the err from io.ReadAll, but in the case where resp.StatusCode
// was 2xx, so we don't have a bad status.
if err != nil {
return nil, errors.Wrapf(err, "error reading response body")
}
if err = json.Unmarshal(buf, &created); err != nil {
return nil, errors.Wrapf(err, "error interpreting response body")
}
return created, nil
}
// IngestOperations uses the new ingest endpoint for ingest data
func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, indexName string, buf []byte) error {
if uri == nil {
uri = c.defaultURI
}
u := uri.Path(fmt.Sprintf("/internal/ingest/%s", indexName))
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "pilosa/"+Version)
AddAuthToken(ctx, &req.Header)
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("unexpected status code: %s", resp.Status)
}
return nil
}
// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data
func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
if uri == nil {
uri = c.defaultURI
}
u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName))
buf, err := c.serializer.Marshal(ireq)
if err != nil {
return errors.Wrap(err, "marshalling")
}
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("User-Agent", "pilosa/"+Version)
AddAuthToken(ctx, &req.Header)
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("unexpected status code: %s", resp.Status)
}
return nil
}
// MutexCheck uses the mutex-check endpoint to request mutex collision data
// from a single node. It produces per-shard results, and does not translate
// them.

34
translate.go
View file

@ -10,7 +10,6 @@ import (
"sync"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
@ -89,39 +88,6 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul
Delete(records *roaring.Bitmap) (Commitor, error)
}
// This implements ingest's key translator interface, which differs
// slightly because we want to be able to do fast lookups on arbitrary
// IDs which are not necessarily contiguous small values, so the []string
// from TranslateIDs isn't a good fit.
type ingestKeyTranslator struct {
store TranslateStore
}
var _ ingest.KeyTranslator = &ingestKeyTranslator{}
func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
return i.store.CreateKeys(keys...)
}
func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
keys, err := i.store.TranslateIDs(ids)
if err != nil {
return nil, err
}
if len(keys) != len(ids) {
return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
}
out := make(map[uint64]string, len(keys))
for i, id := range ids {
out[id] = keys[i]
}
return out, nil
}
func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator {
return &ingestKeyTranslator{store: s}
}
// TranslatorSummary is returned, for example from the boltdb string key translators,
// by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method.
type TranslatorSummary struct {