Merge pull request #702 from benbjohnson/617-bsi-internal-set-field-value

Set & retrieve field values.

fragment.go

@@ -472,6 +472,69 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
 	return changed, nil
 }
 
+func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
+	pos, err := f.pos(rowID, columnID)
+	if err != nil {
+		return false, err
+	}
+	return f.storage.Contains(pos), nil
+}
+
+// FieldValue uses a column of bits to read a multi-bit value.
+func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// If existance bit is unset then ignore remaining bits.
+	if v, err := f.bit(uint64(bitDepth), columnID); err != nil {
+		return 0, false, err
+	} else if !v {
+		return 0, false, nil
+	}
+
+	// Compute other bits into a value.
+	for i := uint(0); i < bitDepth; i++ {
+		if v, err := f.bit(uint64(i), columnID); err != nil {
+			return 0, false, err
+		} else if !v {
+			value |= (1 << i)
+		}
+	}
+
+	return value, true, nil
+}
+
+// SetFieldValue uses a column of bits to set a multi-bit value.
+func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for i := uint(0); i < bitDepth; i++ {
+		if value&(1<<i) != 0 {
+			if c, err := f.clearBit(uint64(i), columnID); err != nil {
+				return changed, err
+			} else if c {
+				changed = true
+			}
+		} else {
+			if c, err := f.setBit(uint64(i), columnID); err != nil {
+				return changed, err
+			} else if c {
+				changed = true
+			}
+		}
+	}
+
+	// Mark value as set.
+	if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
+		return changed, err
+	} else if c {
+		changed = true
+	}
+
+	return changed, nil
+}
+
 // pos translates the row ID and column ID into a position in the storage bitmap.
 func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
 	// Return an error if the column ID is out of the range of the fragment's slice.

fragment_test.go

@@ -20,6 +20,7 @@ import (
 	"math"
 	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pilosa/pilosa"
@@ -92,6 +93,128 @@ func TestFragment_ClearBit(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can set & read a field value.
+func TestFragment_SetFieldValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, 16); err != nil {
+			t.Fatal(err)
+		} else if value != 3829 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected to exist")
+		}
+
+		// Setting value should return no change.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if changed {
+			t.Fatal("expected no change")
+		}
+	})
+
+	t.Run("Overwrite", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Overwriting value should overwrite all bits.
+		if changed, err := f.SetFieldValue(100, 16, 2028); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, 16); err != nil {
+			t.Fatal(err)
+		} else if value != 2028 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected to exist")
+		}
+	})
+
+	t.Run("NotExists", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 10, 20); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Non-existant value.
+		if value, exists, err := f.FieldValue(100, 11); err != nil {
+			t.Fatal(err)
+		} else if value != 0 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if exists {
+			t.Fatal("expected to not exist")
+		}
+	})
+
+	t.Run("QuickCheck", func(t *testing.T) {
+		if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
+			// Limit bit depth & maximum values.
+			bitDepth = (bitDepth % 62) + 1
+			columnN = (columnN % 100)
+			for i := range values {
+				values[i] = values[i] % (1 << bitDepth)
+			}
+
+			f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+			defer f.Close()
+
+			// Set values.
+			m := make(map[uint64]int64)
+			for _, value := range values {
+				columnID := value % columnN
+
+				m[columnID] = int64(value)
+
+				if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Ensure values are set.
+			for columnID, value := range m {
+				v, exists, err := f.FieldValue(columnID, bitDepth)
+				if err != nil {
+					t.Fatal(err)
+				} else if value != int64(v) {
+					t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
+				} else if !exists {
+					t.Fatalf("value should exist: column=%d", columnID)
+				}
+			}
+
+			return true
+		}, nil); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
 // Ensure a fragment can snapshot correctly.
 func TestFragment_Snapshot(t *testing.T) {
 	f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")

frame.go

@@ -404,6 +404,16 @@ func (f *Frame) Schema() *FrameSchema {
 	return f.schema
 }
 
+// Field returns a field from the schema by name.
+func (f *Frame) Field(name string) *Field {
+	for _, field := range f.Schema().Fields {
+		if field.Name == name {
+			return field
+		}
+	}
+	return nil
+}
+
 // TimeQuantum returns the time quantum for the frame.
 func (f *Frame) TimeQuantum() TimeQuantum {
 	f.mu.Lock()
@@ -576,6 +586,52 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
 	return changed, nil
 }
 
+// FieldValue reads a field value for a column.
+func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) {
+	field := f.Field(name)
+	if field == nil {
+		return 0, false, ErrFieldNotFound
+	}
+
+	// Fetch target view.
+	view := f.View(ViewFieldPrefix + name)
+	if view == nil {
+		return 0, false, nil
+	}
+
+	v, exists, err := view.FieldValue(columnID, field.BitDepth())
+	if err != nil {
+		return 0, false, err
+	} else if !exists {
+		return 0, false, nil
+	}
+	return int64(v) + field.Min, true, nil
+}
+
+// SetFieldValue sets a field value for a column.
+func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) {
+	// Fetch field and validate value.
+	field := f.Field(name)
+	if field == nil {
+		return false, ErrFieldNotFound
+	} else if value < field.Min {
+		return false, ErrFieldValueTooLow
+	} else if value > field.Max {
+		return false, ErrFieldValueTooHigh
+	}
+
+	// Fetch target view.
+	view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name)
+	if err != nil {
+		return false, err
+	}
+
+	// Determine base value to store.
+	baseValue := uint64(value - field.Min)
+
+	return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
+}
+
 // Import bulk imports data.
 func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
 	// Determine quantum if timestamps are set.
@@ -761,8 +817,18 @@ func IsValidFieldType(v string) bool {
 type Field struct {
 	Name string `json:"name,omitempty"`
 	Type string `json:"type,omitempty"`
-	Min  int    `json:"min,omitempty"`
-	Max  int    `json:"max,omitempty"`
+	Min  int64  `json:"min,omitempty"`
+	Max  int64  `json:"max,omitempty"`
+}
+
+// BitDepth returns the number of bits required to store a value between min & max.
+func (f *Field) BitDepth() uint {
+	for i := uint(0); i < 63; i++ {
+		if f.Max-f.Min < (1 << i) {
+			return i
+		}
+	}
+	return 63
 }
 
 func ValidateField(f *Field) error {
@@ -817,8 +883,8 @@ func decodeField(f *internal.Field) *Field {
 	return &Field{
 		Name: f.Name,
 		Type: f.Type,
-		Min:  int(f.Min),
-		Max:  int(f.Max),
+		Min:  f.Min,
+		Max:  f.Max,
 	}
 }
 

frame_test.go

@@ -68,6 +68,153 @@ func TestFrame_SetTimeQuantum(t *testing.T) {
 	}
 }
 
+// Ensure a frame can set & read a field value.
+func TestFrame_SetFieldValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on first field.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Set value on same column but different field.
+		if changed, err := f.SetFieldValue(100, "field1", 25); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		// Setting value should return no change.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if changed {
+			t.Fatal("expected no change")
+		}
+	})
+
+	t.Run("Overwrite", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Set different value.
+		if changed, err := f.SetFieldValue(100, "field0", 23); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
+			t.Fatal(err)
+		} else if value != 23 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+	})
+
+	t.Run("ErrFieldNotFound", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrFieldValueTooLow", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrFieldValueTooHigh", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}
+
 func TestFrame_NameRestriction(t *testing.T) {
 	path, err := ioutil.TempDir("", "pilosa-frame-")
 	if err != nil {

pilosa.go

@@ -36,12 +36,15 @@ var (
 	ErrFrameInverseDisabled = errors.New("frame inverse disabled")
 	ErrColumnRowLabelEqual  = errors.New("column and row labels cannot be equal")
 
+	ErrFieldNotFound          = errors.New("field not found")
 	ErrFieldNameRequired      = errors.New("field name required")
 	ErrInvalidFieldType       = errors.New("invalid field type")
 	ErrInvalidFieldRange      = errors.New("invalid field range")
 	ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
 	ErrRangeCacheNotAllowed   = errors.New("range cache not allowed")
 	ErrFrameFieldsNotAllowed  = errors.New("frame fields not allowed")
+	ErrFieldValueTooLow       = errors.New("field value too low")
+	ErrFieldValueTooHigh      = errors.New("field value too high")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")

view.go

@@ -31,6 +31,8 @@ import (
 const (
 	ViewStandard = "standard"
 	ViewInverse  = "inverse"
+
+	ViewFieldPrefix = "field_"
 )
 
 // IsValidView returns true if name is valid.
@@ -278,6 +280,26 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
 	return frag.ClearBit(rowID, columnID)
 }
 
+// FieldValue uses a column of bits to read a multi-bit value.
+func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
+	slice := columnID / SliceWidth
+	frag, err := v.CreateFragmentIfNotExists(slice)
+	if err != nil {
+		return value, exists, err
+	}
+	return frag.FieldValue(columnID, bitDepth)
+}
+
+// SetFieldValue uses a column of bits to set a multi-bit value.
+func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
+	slice := columnID / SliceWidth
+	frag, err := v.CreateFragmentIfNotExists(slice)
+	if err != nil {
+		return changed, err
+	}
+	return frag.SetFieldValue(columnID, bitDepth, value)
+}
+
 // IsInverseView returns true if the view is used for storing an inverted representation.
 func IsInverseView(name string) bool {
 	return strings.HasPrefix(name, ViewInverse)