From 6621a90599800dc5b011bd5c875d7381b9891a6c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 28 Jun 2017 16:50:43 -0500 Subject: [PATCH 1/2] Increase header sizes --- docs/query-language.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 1d0e01ac4..5400c8ead 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -4,6 +4,8 @@ title = "Query Language" ## Query Language +### Overview + This section will provide a detailed reference and examples for the Pilosa Query Language (PQL). All PQL queries operate on a single [index]({{< ref "glossary.md#index" >}}) and are passed to Pilosa through the `/index/*index_name*/query` endpoint. You may pass multiple PQL queries in a single request by simply concatenating the queries together - a space is not needed. The results format is always: ``` @@ -43,9 +45,9 @@ curl localhost:10101/index/repository/query \ * `BITMAP_CALL` Any query which returns a bitmap, such as `Bitmap`, `Union`, `Difference`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) -#### Write Operations +### Write Operations -##### SetBit +#### SetBit **Spec:** @@ -83,7 +85,7 @@ Setting multiple bits in a single request: SetBit(frame="stargazer", repo_id=10, stargazer_id=1) SetBit(frame="stargazer", repo_id=10, stargazer_id=2) SetBit(frame="stargazer", repo_id=20, stargazer_id=1) SetBit(frame="stargazer", repo_id=30, stargazer_id=2) ``` -##### SetRowAttrs +#### SetRowAttrs **Spec:** ``` @@ -114,7 +116,7 @@ SetRowAttrs(frame="stargazer", stargazer_id=10, username=null) Delete username value for user 10. -##### SetColumnAttrs +#### SetColumnAttrs **Spec:** @@ -147,7 +149,7 @@ SetColumnAttrs(frame="stargazer", repo_id=10, url=null) Delete url value for repo 10. -##### ClearBit +#### ClearBit **Spec:** @@ -175,9 +177,9 @@ ClearBit(frame="stargazer", repo_id=10, stargazer_id=1) Remove relationship between stargazer_id 1 and repo_id 10 from the stargazer frame. -#### Read Operations +### Read Operations -##### Bitmap +#### Bitmap **Spec:** @@ -205,7 +207,7 @@ Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` * attrs are the attributes for user 1 * bits are the repositories which user 1 has starred. -##### Union +#### Union **Spec:** @@ -232,7 +234,7 @@ Returns `{"attrs":{},"bits":[10, 20, 30]}`. * bits are repositories that were starred by user 1 OR user 2 -##### Intersect +#### Intersect **Spec:** @@ -261,7 +263,7 @@ Returns `{"attrs":{},"bits":[10]}`. * bits are repositories that were starred by user 1 AND user 2 -##### Difference +#### Difference **Spec:** @@ -296,7 +298,7 @@ Return `{"attrs":{},"bits":[30]}` * Bits are repositories that were starred by user 2 BUT NOT user 1 -##### Count +#### Count **Spec:** ``` @@ -320,7 +322,7 @@ Return `2` * Result is the number of repositories that user 1 has starred. -##### TopN +#### TopN **Spec:** @@ -377,7 +379,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` * Results are the top two users sorted by the number of repositories that they've starred which are written in language 1. -##### Range Queries +#### Range Queries **Spec:** From 800e3844f9b3104301481b2351bed153fb6c310b Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 29 Jun 2017 21:35:01 -0600 Subject: [PATCH 2/2] Set & retrieve field values. This commit adds support for set/get field values at the `Frame`, `View`, and `Fragment` levels. --- fragment.go | 63 ++++++++++++++++++++ fragment_test.go | 123 +++++++++++++++++++++++++++++++++++++++ frame.go | 74 ++++++++++++++++++++++-- frame_test.go | 147 +++++++++++++++++++++++++++++++++++++++++++++++ pilosa.go | 3 + view.go | 22 +++++++ 6 files changed, 428 insertions(+), 4 deletions(-) diff --git a/fragment.go b/fragment.go index 278a23bcf..64c31434b 100644 --- a/fragment.go +++ b/fragment.go @@ -472,6 +472,69 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } +func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { + pos, err := f.pos(rowID, columnID) + if err != nil { + return false, err + } + return f.storage.Contains(pos), nil +} + +// FieldValue uses a column of bits to read a multi-bit value. +func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + // If existance bit is unset then ignore remaining bits. + if v, err := f.bit(uint64(bitDepth), columnID); err != nil { + return 0, false, err + } else if !v { + return 0, false, nil + } + + // Compute other bits into a value. + for i := uint(0); i < bitDepth; i++ { + if v, err := f.bit(uint64(i), columnID); err != nil { + return 0, false, err + } else if !v { + value |= (1 << i) + } + } + + return value, true, nil +} + +// SetFieldValue uses a column of bits to set a multi-bit value. +func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + for i := uint(0); i < bitDepth; i++ { + if value&(1< field.Max { + return false, ErrFieldValueTooHigh + } + + // Fetch target view. + view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name) + if err != nil { + return false, err + } + + // Determine base value to store. + baseValue := uint64(value - field.Min) + + return view.SetFieldValue(columnID, field.BitDepth(), baseValue) +} + // Import bulk imports data. func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. @@ -761,8 +817,18 @@ func IsValidFieldType(v string) bool { type Field struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` - Min int `json:"min,omitempty"` - Max int `json:"max,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` +} + +// BitDepth returns the number of bits required to store a value between min & max. +func (f *Field) BitDepth() uint { + for i := uint(0); i < 63; i++ { + if f.Max-f.Min < (1 << i) { + return i + } + } + return 63 } func ValidateField(f *Field) error { @@ -817,8 +883,8 @@ func decodeField(f *internal.Field) *Field { return &Field{ Name: f.Name, Type: f.Type, - Min: int(f.Min), - Max: int(f.Max), + Min: f.Min, + Max: f.Max, } } diff --git a/frame_test.go b/frame_test.go index 70e2a8b0b..8cb48cca8 100644 --- a/frame_test.go +++ b/frame_test.go @@ -68,6 +68,153 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } } +// Ensure a frame can set & read a field value. +func TestFrame_SetFieldValue(t *testing.T) { + t.Run("OK", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, + {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set value on first field. + if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Set value on same column but different field. + if changed, err := f.SetFieldValue(100, "field1", 25); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.FieldValue(100, "field0"); err != nil { + t.Fatal(err) + } else if value != 21 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + + // Setting value should return no change. + if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { + t.Fatal(err) + } else if changed { + t.Fatal("expected no change") + } + }) + + t.Run("Overwrite", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Set different value. + if changed, err := f.SetFieldValue(100, "field0", 23); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.FieldValue(100, "field0"); err != nil { + t.Fatal(err) + } else if value != 23 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFieldValueTooLow", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrFieldValueTooHigh", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, + }, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh { + t.Fatalf("unexpected error: %s", err) + } + }) +} + func TestFrame_NameRestriction(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-frame-") if err != nil { diff --git a/pilosa.go b/pilosa.go index 978ebaf30..a334a2be1 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,12 +36,15 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFieldNotFound = errors.New("field not found") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") ErrInvalidFieldRange = errors.New("invalid field range") ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") ErrRangeCacheNotAllowed = errors.New("range cache not allowed") ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrFieldValueTooLow = errors.New("field value too low") + ErrFieldValueTooHigh = errors.New("field value too high") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/view.go b/view.go index 367a69b5a..2f47fae46 100644 --- a/view.go +++ b/view.go @@ -31,6 +31,8 @@ import ( const ( ViewStandard = "standard" ViewInverse = "inverse" + + ViewFieldPrefix = "field_" ) // IsValidView returns true if name is valid. @@ -278,6 +280,26 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { return frag.ClearBit(rowID, columnID) } +// FieldValue uses a column of bits to read a multi-bit value. +func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { + slice := columnID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return value, exists, err + } + return frag.FieldValue(columnID, bitDepth) +} + +// SetFieldValue uses a column of bits to set a multi-bit value. +func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { + slice := columnID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetFieldValue(columnID, bitDepth, value) +} + // IsInverseView returns true if the view is used for storing an inverted representation. func IsInverseView(name string) bool { return strings.HasPrefix(name, ViewInverse)