mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
016765d8a2
This partially-implemented prototype of the ingest API is based on our programmatic ingest API reference. It has noticable limitations, most crucially that it doesn't handle multi-node clusters right now. However, it basically implements the expected semantics. There's some noticeable performance issues to do with the high overhead of sorting bits in order to import them efficiently, but this is fixable. We also add the hooks to the internal client, and make the finisher logic a bit smarter. Much of this code was originally by Nia Weiss, but it's been merged and restructured a bit to get things broken into logical commits.
256 lines
8 KiB
Go
256 lines
8 KiB
Go
// Copyright 2021 Molecula Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ingest
|
|
|
|
import (
|
|
"github.com/molecula/featurebase/v2/roaring"
|
|
)
|
|
|
|
// ShardUpdate is an update request for a shard.
|
|
type ShardUpdate struct {
|
|
// TODO: include schema version
|
|
|
|
// Sets are the updates to set fields.
|
|
Sets map[string]SetMatrixUpdate
|
|
|
|
// Mutexes are the updates to mutex fields.
|
|
Mutexes map[string]MutexMatrixUpdate
|
|
|
|
// TimeTensors are the updates to time fields.
|
|
TimeTensors map[string]TimeTensorUpdate
|
|
|
|
// Ints are the updates to int fields.
|
|
Ints map[string]IntUpdate
|
|
}
|
|
|
|
// Convert the ID vector to a raw update that can be imported.
|
|
// All records are assumed to fall within a shard.
|
|
// func (vec IDVector) Convert() (MutexMatrixUpdate, error) {
|
|
// // Before converting the vector, do a sanity-check for duplicates.
|
|
// dedup := make(map[uint64]struct{}, len(vec.Updates)+len(vec.Clears))
|
|
// for _, p := range vec.Updates {
|
|
// if _, ok := dedup[p.RecordID]; !ok {
|
|
// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates")
|
|
// }
|
|
//
|
|
// dedup[p.RecordID] = struct{}{}
|
|
// }
|
|
// for _, p := range vec.Clears {
|
|
// if _, ok := dedup[p]; !ok {
|
|
// return MutexMatrixUpdate{}, errors.New("input contains conflicting updates")
|
|
// }
|
|
//
|
|
// dedup[p] = struct{}{}
|
|
// }
|
|
//
|
|
// // Convert the updates to a bitmap.
|
|
// updates, err := idPairsToBitmap(vec.Updates, false)
|
|
// if err != nil {
|
|
// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex updates")
|
|
// }
|
|
//
|
|
// // Convert the clears to a bitmap.
|
|
// clears, err := idListToBitmap(vec.Clears, false)
|
|
// if err != nil {
|
|
// return MutexMatrixUpdate{}, errors.Wrap(err, "encoding mutex clears")
|
|
// }
|
|
//
|
|
// // Realign clears bitmap to start of shard.
|
|
// if min, ok := clears.Min(); ok {
|
|
// min &^= (1 << shardwidth.Exponent) - 1
|
|
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
|
|
// }
|
|
//
|
|
// return MutexMatrixUpdate{
|
|
// Update: updates,
|
|
// Clear: clears,
|
|
// }, nil
|
|
// }
|
|
|
|
// MutexMatrixUpdate is an encoded update request for a mutex-type view.
|
|
type MutexMatrixUpdate struct {
|
|
// Update is a bitmap of new mutex values.
|
|
// This bitmap will not include any records in the clear bitmap.
|
|
Update *roaring.Bitmap
|
|
|
|
// Clear is a set of records for which all values should be removed.
|
|
Clear *roaring.Bitmap
|
|
}
|
|
|
|
// Convert the time vectors to a raw update that can be imported.
|
|
// All records are assumed to fall within a shard.
|
|
// func (vec TimeIDSetsVector) Convert() (TimeTensorUpdate, error) {
|
|
// // Convert the clears to a bitmap.
|
|
// clears, err := idListToBitmap(vec.Clears, false)
|
|
// if err != nil {
|
|
// return TimeTensorUpdate{}, errors.Wrap(err, "encoding time tensor clears")
|
|
// }
|
|
//
|
|
// // Realign clears bitmap to start of shard.
|
|
// if min, ok := clears.Min(); ok {
|
|
// min &^= (1 << shardwidth.Exponent) - 1
|
|
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
|
|
// }
|
|
//
|
|
// // Convert the adds to bitmaps.
|
|
// quantums := make(map[string]*roaring.Bitmap)
|
|
// for t, vec := range vec.Quantums {
|
|
// adds, err := idPairsToBitmap(vec, true)
|
|
// if err != nil {
|
|
// return TimeTensorUpdate{}, errors.Wrap(err, "encoding time adds")
|
|
// }
|
|
//
|
|
// quantums[t] = adds
|
|
// }
|
|
//
|
|
// return TimeTensorUpdate{
|
|
// Quantums: quantums,
|
|
// Clear: clears,
|
|
// }, nil
|
|
// }
|
|
|
|
// TimeTensorUpdate is an encoded update request for a time field within a shard.
|
|
type TimeTensorUpdate struct {
|
|
// Quantums are the component set matrix adds, grouped by time quantum.
|
|
Quantums map[string]*roaring.Bitmap
|
|
|
|
// Clear is a set of records for which all values should be removed.
|
|
// This must not overlap with any values in the remove or clear sections of any quantum.
|
|
Clear *roaring.Bitmap
|
|
}
|
|
|
|
// Convert the ID set vector to a raw update that can be imported.
|
|
// All records are assumed to fall within a shard.
|
|
// func (vec IDSetVector) Convert() (SetMatrixUpdate, error) {
|
|
// // Convert the clears to a bitmap.
|
|
// clears, err := idListToBitmap(vec.Clears, false)
|
|
// if err != nil {
|
|
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set clears")
|
|
// }
|
|
//
|
|
// // Realign clears bitmap to start of shard.
|
|
// if min, ok := clears.Min(); ok {
|
|
// min &^= (1 << shardwidth.Exponent) - 1
|
|
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
|
|
// }
|
|
//
|
|
// // Convert the adds to a bitmap.
|
|
// adds, err := idPairsToBitmap(vec.Adds, false)
|
|
// if err != nil {
|
|
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set adds")
|
|
// }
|
|
//
|
|
// // Verify that the removes do not include the cleared records.
|
|
// if clears.Any() {
|
|
// for i := 0; i < len(vec.Removes); {
|
|
// recID := vec.Removes[i].RecordID
|
|
// if clears.Contains(recID) {
|
|
// return SetMatrixUpdate{}, errors.New("removed element duplicated with a clear")
|
|
// }
|
|
//
|
|
// i++
|
|
// for i < len(vec.Removes) && vec.Removes[i].RecordID == recID {
|
|
// i++
|
|
// }
|
|
// }
|
|
// }
|
|
//
|
|
// // Convert the removes to a bitmap.
|
|
// removes, err := idPairsToBitmap(vec.Removes, false)
|
|
// if err != nil {
|
|
// return SetMatrixUpdate{}, errors.Wrap(err, "encoding set removes")
|
|
// }
|
|
//
|
|
// // Check that no bits are both added and removed.
|
|
// if adds.IntersectionCount(removes) > 0 {
|
|
// return SetMatrixUpdate{}, errors.New("set adds and removes overlap")
|
|
// }
|
|
//
|
|
// return SetMatrixUpdate{
|
|
// Add: adds,
|
|
// Remove: removes,
|
|
// Clear: clears,
|
|
// }, nil
|
|
// }
|
|
|
|
// SetMatrixUpdate is an encoded update request for a set-type (set/mutex) view.
|
|
type SetMatrixUpdate struct {
|
|
// Add is a bitmap to union the matrix against.
|
|
Add *roaring.Bitmap
|
|
|
|
// Remove is a bitmap to difference out of the matrix.
|
|
// No values will be present in both add and remove.
|
|
// This bitmap will not include any records in the clear bitmap.
|
|
Remove *roaring.Bitmap
|
|
|
|
// Clear is a set of records for which all values should be removed.
|
|
// This is applied before add operations.
|
|
Clear *roaring.Bitmap
|
|
}
|
|
|
|
// func idPairsToBitmap(pairs []IDPair, allowDup bool) (*roaring.Bitmap, error) {
|
|
// return idListToBitmap(pairsToZigZag(pairs), allowDup)
|
|
// }
|
|
//
|
|
// func idListToBitmap(ids []uint64, allowDup bool) (*roaring.Bitmap, error) {
|
|
// vals := dedupSort64(ids)
|
|
// if len(vals) != len(ids) && !allowDup {
|
|
// return nil, errors.New("input contains duplicate values")
|
|
// }
|
|
//
|
|
// // TODO: make the roaring package actually handle this well
|
|
// return roaring.NewBitmap(vals...), nil
|
|
// }
|
|
|
|
// Convert the int vector to a raw update that can be imported.
|
|
// All records are assumed to fall within a shard.
|
|
// func (vec IntVector) Convert() (IntUpdate, error) {
|
|
// // Convert the clears to a bitmap.
|
|
// clears, err := idListToBitmap(vec.Clears, false)
|
|
// if err != nil {
|
|
// return IntUpdate{}, errors.Wrap(err, "encoding int clears")
|
|
// }
|
|
//
|
|
// // Realign clears bitmap to start of shard.
|
|
// if min, ok := clears.Min(); ok {
|
|
// min &^= (1 << shardwidth.Exponent) - 1
|
|
// clears = clears.OffsetRange(-min, 0, 1<<shardwidth.Exponent)
|
|
// }
|
|
//
|
|
// // Convert updates to a bitmap.
|
|
// updates, err := idListToBitmap(intToBSI(vec.Updates), false)
|
|
// if err != nil {
|
|
// return IntUpdate{}, errors.Wrap(err, "encoding int updates")
|
|
// }
|
|
//
|
|
// // Check that no updated records are also being cleared.
|
|
// if updates.IntersectionCount(clears) > 0 {
|
|
// return IntUpdate{}, errors.New("update duplicated with a clear")
|
|
// }
|
|
//
|
|
// return IntUpdate{
|
|
// BSI: updates,
|
|
// Clear: clears,
|
|
// }, nil
|
|
// }
|
|
|
|
// IntUpdate is an encoded update request for a BSI (int/timestamp/etc.) view.
|
|
type IntUpdate struct {
|
|
// BSI is a bitmap of new BSI data to overwrite existing values.
|
|
BSI *roaring.Bitmap
|
|
|
|
// Clear is a set of records to assign null.
|
|
Clear *roaring.Bitmap
|
|
}
|