Merge remote-tracking branch 'upstream/master' into input-definition

Conflicts:
	pilosa.go

docs/query-language.md

@@ -4,6 +4,8 @@ title = "Query Language"
 
 ## Query Language
 
+### Overview
+
 This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always:
 
 ```
@@ -43,9 +45,9 @@ curl localhost:10101/index/repository/query \
 * `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range`
 * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`)
 
-#### Write Operations
+### Write Operations
 
-##### SetBit
+#### SetBit
 
 **Spec:**
 
@@ -83,7 +85,7 @@ Setting multiple bits in a single request:
 SetBit(frame="stargazer", repo_id=10, stargazer_id=1) SetBit(frame="stargazer", repo_id=10, stargazer_id=2) SetBit(frame="stargazer", repo_id=20, stargazer_id=1) SetBit(frame="stargazer", repo_id=30, stargazer_id=2)
 ```
 
-##### SetRowAttrs
+#### SetRowAttrs
 **Spec:**
 
 ```
@@ -114,7 +116,7 @@ SetRowAttrs(frame="stargazer", stargazer_id=10, username=null)
 
 Delete username value for user 10.
 
-##### SetColumnAttrs
+#### SetColumnAttrs
 
 **Spec:**
 
@@ -147,7 +149,7 @@ SetColumnAttrs(frame="stargazer", repo_id=10, url=null)
 Delete url value for repo 10.
 
 
-##### ClearBit
+#### ClearBit
 
 **Spec:**
 
@@ -175,9 +177,9 @@ ClearBit(frame="stargazer", repo_id=10, stargazer_id=1)
 Remove relationship between stargazer_id 1 and repo_id 10  from the stargazer frame.
 
 
-#### Read Operations
+### Read Operations
 
-##### Bitmap
+#### Bitmap
 
 **Spec:**
 
@@ -205,7 +207,7 @@ Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}`
 * attrs are the attributes for user 1 
 * bits are the repositories which user 1 has starred.
 
-##### Union
+#### Union
 
 **Spec:**
 
@@ -232,7 +234,7 @@ Returns `{"attrs":{},"bits":[10, 20, 30]}`.
 
 * bits are repositories that were starred by user 1 OR user 2
 
-##### Intersect
+#### Intersect
 
 
 **Spec:**
@@ -261,7 +263,7 @@ Returns `{"attrs":{},"bits":[10]}`.
 
 * bits are repositories that were starred by user 1 AND user 2
 
-##### Difference
+#### Difference
 
 **Spec:**
 
@@ -296,7 +298,7 @@ Return `{"attrs":{},"bits":[30]}`
 
 * Bits are repositories that were starred by user 2 BUT NOT user 1
 
-##### Count
+#### Count
 **Spec:**
 
 ```
@@ -320,7 +322,7 @@ Return `2`
 
 * Result is the number of repositories that user 1 has starred.
 
-##### TopN
+#### TopN
 
 **Spec:**
 
@@ -377,7 +379,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]`
 
 * Results are the top two users sorted by the number of repositories that they've starred which are written in language 1.
 
-##### Range Queries
+#### Range Queries
 
 **Spec:**
 

fragment.go

@@ -472,6 +472,69 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
 	return changed, nil
 }
 
+func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
+	pos, err := f.pos(rowID, columnID)
+	if err != nil {
+		return false, err
+	}
+	return f.storage.Contains(pos), nil
+}
+
+// FieldValue uses a column of bits to read a multi-bit value.
+func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	// If existance bit is unset then ignore remaining bits.
+	if v, err := f.bit(uint64(bitDepth), columnID); err != nil {
+		return 0, false, err
+	} else if !v {
+		return 0, false, nil
+	}
+
+	// Compute other bits into a value.
+	for i := uint(0); i < bitDepth; i++ {
+		if v, err := f.bit(uint64(i), columnID); err != nil {
+			return 0, false, err
+		} else if !v {
+			value |= (1 << i)
+		}
+	}
+
+	return value, true, nil
+}
+
+// SetFieldValue uses a column of bits to set a multi-bit value.
+func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for i := uint(0); i < bitDepth; i++ {
+		if value&(1<<i) != 0 {
+			if c, err := f.clearBit(uint64(i), columnID); err != nil {
+				return changed, err
+			} else if c {
+				changed = true
+			}
+		} else {
+			if c, err := f.setBit(uint64(i), columnID); err != nil {
+				return changed, err
+			} else if c {
+				changed = true
+			}
+		}
+	}
+
+	// Mark value as set.
+	if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
+		return changed, err
+	} else if c {
+		changed = true
+	}
+
+	return changed, nil
+}
+
 // pos translates the row ID and column ID into a position in the storage bitmap.
 func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
 	// Return an error if the column ID is out of the range of the fragment's slice.

fragment_test.go

@@ -20,6 +20,7 @@ import (
 	"math"
 	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pilosa/pilosa"
@@ -92,6 +93,128 @@ func TestFragment_ClearBit(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can set & read a field value.
+func TestFragment_SetFieldValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, 16); err != nil {
+			t.Fatal(err)
+		} else if value != 3829 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected to exist")
+		}
+
+		// Setting value should return no change.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if changed {
+			t.Fatal("expected no change")
+		}
+	})
+
+	t.Run("Overwrite", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Overwriting value should overwrite all bits.
+		if changed, err := f.SetFieldValue(100, 16, 2028); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, 16); err != nil {
+			t.Fatal(err)
+		} else if value != 2028 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected to exist")
+		}
+	})
+
+	t.Run("NotExists", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, 10, 20); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Non-existant value.
+		if value, exists, err := f.FieldValue(100, 11); err != nil {
+			t.Fatal(err)
+		} else if value != 0 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if exists {
+			t.Fatal("expected to not exist")
+		}
+	})
+
+	t.Run("QuickCheck", func(t *testing.T) {
+		if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
+			// Limit bit depth & maximum values.
+			bitDepth = (bitDepth % 62) + 1
+			columnN = (columnN % 100)
+			for i := range values {
+				values[i] = values[i] % (1 << bitDepth)
+			}
+
+			f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+			defer f.Close()
+
+			// Set values.
+			m := make(map[uint64]int64)
+			for _, value := range values {
+				columnID := value % columnN
+
+				m[columnID] = int64(value)
+
+				if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Ensure values are set.
+			for columnID, value := range m {
+				v, exists, err := f.FieldValue(columnID, bitDepth)
+				if err != nil {
+					t.Fatal(err)
+				} else if value != int64(v) {
+					t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
+				} else if !exists {
+					t.Fatalf("value should exist: column=%d", columnID)
+				}
+			}
+
+			return true
+		}, nil); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
 // Ensure a fragment can snapshot correctly.
 func TestFragment_Snapshot(t *testing.T) {
 	f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")

frame.go

@@ -402,6 +402,16 @@ func (f *Frame) Schema() *FrameSchema {
 	return f.schema
 }
 
+// Field returns a field from the schema by name.
+func (f *Frame) Field(name string) *Field {
+	for _, field := range f.Schema().Fields {
+		if field.Name == name {
+			return field
+		}
+	}
+	return nil
+}
+
 // TimeQuantum returns the time quantum for the frame.
 func (f *Frame) TimeQuantum() TimeQuantum {
 	f.mu.Lock()
@@ -574,6 +584,52 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
 	return changed, nil
 }
 
+// FieldValue reads a field value for a column.
+func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) {
+	field := f.Field(name)
+	if field == nil {
+		return 0, false, ErrFieldNotFound
+	}
+
+	// Fetch target view.
+	view := f.View(ViewFieldPrefix + name)
+	if view == nil {
+		return 0, false, nil
+	}
+
+	v, exists, err := view.FieldValue(columnID, field.BitDepth())
+	if err != nil {
+		return 0, false, err
+	} else if !exists {
+		return 0, false, nil
+	}
+	return int64(v) + field.Min, true, nil
+}
+
+// SetFieldValue sets a field value for a column.
+func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) {
+	// Fetch field and validate value.
+	field := f.Field(name)
+	if field == nil {
+		return false, ErrFieldNotFound
+	} else if value < field.Min {
+		return false, ErrFieldValueTooLow
+	} else if value > field.Max {
+		return false, ErrFieldValueTooHigh
+	}
+
+	// Fetch target view.
+	view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name)
+	if err != nil {
+		return false, err
+	}
+
+	// Determine base value to store.
+	baseValue := uint64(value - field.Min)
+
+	return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
+}
+
 // Import bulk imports data.
 func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
 	// Determine quantum if timestamps are set.
@@ -759,8 +815,18 @@ func IsValidFieldType(v string) bool {
 type Field struct {
 	Name string `json:"name,omitempty"`
 	Type string `json:"type,omitempty"`
-	Min  int    `json:"min,omitempty"`
-	Max  int    `json:"max,omitempty"`
+	Min  int64  `json:"min,omitempty"`
+	Max  int64  `json:"max,omitempty"`
+}
+
+// BitDepth returns the number of bits required to store a value between min & max.
+func (f *Field) BitDepth() uint {
+	for i := uint(0); i < 63; i++ {
+		if f.Max-f.Min < (1 << i) {
+			return i
+		}
+	}
+	return 63
 }
 
 func ValidateField(f *Field) error {
@@ -815,8 +881,8 @@ func decodeField(f *internal.Field) *Field {
 	return &Field{
 		Name: f.Name,
 		Type: f.Type,
-		Min:  int(f.Min),
-		Max:  int(f.Max),
+		Min:  f.Min,
+		Max:  f.Max,
 	}
 }
 

frame_test.go

@@ -68,6 +68,153 @@ func TestFrame_SetTimeQuantum(t *testing.T) {
 	}
 }
 
+// Ensure a frame can set & read a field value.
+func TestFrame_SetFieldValue(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+				{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value on first field.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Set value on same column but different field.
+		if changed, err := f.SetFieldValue(100, "field1", 25); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
+			t.Fatal(err)
+		} else if value != 21 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+
+		// Setting value should return no change.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if changed {
+			t.Fatal("expected no change")
+		}
+	})
+
+	t.Run("Overwrite", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Set different value.
+		if changed, err := f.SetFieldValue(100, "field0", 23); err != nil {
+			t.Fatal(err)
+		} else if !changed {
+			t.Fatal("expected change")
+		}
+
+		// Read value.
+		if value, exists, err := f.FieldValue(100, "field0"); err != nil {
+			t.Fatal(err)
+		} else if value != 23 {
+			t.Fatalf("unexpected value: %d", value)
+		} else if !exists {
+			t.Fatal("expected value to exist")
+		}
+	})
+
+	t.Run("ErrFieldNotFound", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrFieldValueTooLow", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+
+	t.Run("ErrFieldValueTooHigh", func(t *testing.T) {
+		idx := test.MustOpenIndex()
+		defer idx.Close()
+
+		f, err := idx.CreateFrame("f", pilosa.FrameOptions{
+			RangeEnabled: true,
+			Fields: []*pilosa.Field{
+				{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set value.
+		if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	})
+}
+
 func TestFrame_NameRestriction(t *testing.T) {
 	path, err := ioutil.TempDir("", "pilosa-frame-")
 	if err != nil {

pilosa.go

@@ -36,6 +36,12 @@ var (
 	ErrFrameInverseDisabled = errors.New("frame inverse disabled")
 	ErrColumnRowLabelEqual  = errors.New("column and row labels cannot be equal")
 
+	ErrFieldNameRequired      = errors.New("field name required")
+	ErrInvalidFieldType       = errors.New("invalid field type")
+	ErrInvalidFieldRange      = errors.New("invalid field range")
+	ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
+	ErrRangeCacheNotAllowed   = errors.New("range cache not allowed")
+	ErrFrameFieldsNotAllowed  = errors.New("frame fields not allowed")
 	ErrInputDefinitionExists         = errors.New("input-definition already exists")
 	ErrInputDefinitionHasPrimaryKey  = errors.New("input-definition must contain one PrimaryKey")
 	ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey")
@@ -52,6 +58,15 @@ var (
 	ErrRangeCacheNotAllowed    = errors.New("range cache not allowed")
 	ErrFrameFieldsNotAllowed   = errors.New("frame fields not allowed")
 	ErrInputDefinitionNotFound = errors.New("input-definition not found")
+	ErrFieldNotFound          = errors.New("field not found")
+	ErrFieldNameRequired      = errors.New("field name required")
+	ErrInvalidFieldType       = errors.New("invalid field type")
+	ErrInvalidFieldRange      = errors.New("invalid field range")
+	ErrInverseRangeNotAllowed = errors.New("inverse range not allowed")
+	ErrRangeCacheNotAllowed   = errors.New("range cache not allowed")
+	ErrFrameFieldsNotAllowed  = errors.New("frame fields not allowed")
+	ErrFieldValueTooLow       = errors.New("field value too low")
+	ErrFieldValueTooHigh      = errors.New("field value too high")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")

view.go

@@ -31,6 +31,8 @@ import (
 const (
 	ViewStandard = "standard"
 	ViewInverse  = "inverse"
+
+	ViewFieldPrefix = "field_"
 )
 
 // IsValidView returns true if name is valid.
@@ -278,6 +280,26 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
 	return frag.ClearBit(rowID, columnID)
 }
 
+// FieldValue uses a column of bits to read a multi-bit value.
+func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
+	slice := columnID / SliceWidth
+	frag, err := v.CreateFragmentIfNotExists(slice)
+	if err != nil {
+		return value, exists, err
+	}
+	return frag.FieldValue(columnID, bitDepth)
+}
+
+// SetFieldValue uses a column of bits to set a multi-bit value.
+func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
+	slice := columnID / SliceWidth
+	frag, err := v.CreateFragmentIfNotExists(slice)
+	if err != nil {
+		return changed, err
+	}
+	return frag.SetFieldValue(columnID, bitDepth, value)
+}
+
 // IsInverseView returns true if the view is used for storing an inverted representation.
 func IsInverseView(name string) bool {
 	return strings.HasPrefix(name, ViewInverse)