From 7a2d90ade8d753bc634d7d76776bf00bf71f0998 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 9 Dec 2019 16:28:03 -0600 Subject: [PATCH 1/5] add support to clear value for column --- executor.go | 41 ++++++++++++++++++++++++++++++++++++++++- field.go | 24 +++++++++++++++++++++++- field_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ view.go | 10 ++++++++++ 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 5d8c94c75..adbd78f41 100644 --- a/executor.go +++ b/executor.go @@ -2479,7 +2479,6 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if f == nil { return false, ErrFieldNotFound } - // Read fields using labels. rowID, ok, err := c.UintArg(fieldName) if err != nil { @@ -2495,6 +2494,11 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("column argument to Clear(, =) required") } + // Int field. + if f.Type() == FieldTypeInt || f.Type() == FieldTypeDecimal { + return e.executeClearValueField(ctx, index, c, f, colID, opt) + } + return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } @@ -2842,6 +2846,41 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq return ret, nil } +// executeSetValueField executes a Set() call for a specific int field. +func (e *executor) executeClearValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, opt *execOptions) (bool, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearValueField") + defer span.Finish() + + shard := colID / ShardWidth + ret := false + + for _, node := range e.Cluster.shardNodes(index, shard) { + // Update locally if host matches. + if node.ID == e.Node.ID { + val, err := f.ClearValue(colID) + if err != nil { + return false, err + } else if val { + ret = true + } + continue + } + + // Do not forward call if this is already being forwarded. + if opt.Remote { + continue + } + + // Forward call to remote node otherwise. + res, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil) + if err != nil { + return false, err + } + ret = res[0].(bool) + } + return ret, nil +} + // executeSetRowAttrs executes a SetRowAttrs() call. func (e *executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *execOptions) error { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetRowAttrs") diff --git a/field.go b/field.go index b6047e3a8..079c0d7d3 100644 --- a/field.go +++ b/field.go @@ -1183,10 +1183,32 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) if err != nil { return false, errors.Wrap(err, "creating view") } - return view.setValue(columnID, bsig.BitDepth, baseValue) } +// SetValue sets a field value for a column. +func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { + // Fetch bsiGroup & validate min/max. + bsig := f.bsiGroup(f.name) + if bsig == nil { + return false, ErrBSIGroupNotFound + } + + // Fetch target view. + view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + value, exists, err := view.value(columnID, bsig.BitDepth) + if err != nil { + return false, err + } + if exists { + return view.clearValue(columnID, bsig.BitDepth, value) + } + return false, nil +} + // FloatSum performs a Sum query and converts the result to a float // based on the field's configured scale. func (f *Field) FloatSum(filter *Row, name string) (sum float64, count int64, err error) { diff --git a/field_test.go b/field_test.go index 6e7a5d353..79b14e91f 100644 --- a/field_test.go +++ b/field_test.go @@ -227,3 +227,43 @@ func TestField_AvailableShards(t *testing.T) { t.Fatal(diff) } } + +func TestField_ClearValue(t *testing.T) { + t.Run("OK", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) + if err != nil { + t.Fatal(err) + } + + // Set value on field. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if value != 21 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + + if changed, err := f.ClearValue(100); err != nil { + t.Fatal(err) + } else if !changed { + } + + // Read value. + if _, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if exists { + t.Fatal("expected value to not exist") + } + }) +} diff --git a/view.go b/view.go index e1648a838..079a20fac 100644 --- a/view.go +++ b/view.go @@ -405,6 +405,16 @@ func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bo return frag.setValue(columnID, bitDepth, value) } +// clearValue uses a column of bits to set a multi-bit value. +func (v *view) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) + if err != nil { + return changed, err + } + return frag.clearValue(columnID, bitDepth, value) +} + // sum returns the sum & count of a field. func (v *view) sum(filter *Row, bitDepth uint) (sum int64, count uint64, err error) { for _, f := range v.allFragments() { From 95f2864abe99854e5f6ae990c7195e17fcf7248a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 11 Dec 2019 12:13:57 -0600 Subject: [PATCH 2/5] applied suggestions by @travisturner --- executor.go | 2 +- field_test.go | 1 + view.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index adbd78f41..d84563a41 100644 --- a/executor.go +++ b/executor.go @@ -2846,7 +2846,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetValueField executes a Set() call for a specific int field. +// executeSetValueField removes value for colID if present func (e *executor) executeClearValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearValueField") defer span.Finish() diff --git a/field_test.go b/field_test.go index 79b14e91f..398002ca2 100644 --- a/field_test.go +++ b/field_test.go @@ -257,6 +257,7 @@ func TestField_ClearValue(t *testing.T) { if changed, err := f.ClearValue(100); err != nil { t.Fatal(err) } else if !changed { + t.Fatal(err) } // Read value. diff --git a/view.go b/view.go index 079a20fac..334d55d34 100644 --- a/view.go +++ b/view.go @@ -405,7 +405,7 @@ func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bo return frag.setValue(columnID, bitDepth, value) } -// clearValue uses a column of bits to set a multi-bit value. +// clearValue removes a specific value assigned to columnID func (v *view) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth frag, err := v.CreateFragmentIfNotExists(shard) From d843959904cfc7eb0bec21215071e887bfe31900 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 7 Jan 2020 10:39:56 -0600 Subject: [PATCH 3/5] fixed comments; removed create fragment --- executor.go | 2 +- view.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 33c7c6cd8..abd557837 100644 --- a/executor.go +++ b/executor.go @@ -3031,7 +3031,7 @@ func (e *executor) executeSetValueField(ctx context.Context, index string, c *pq return ret, nil } -// executeSetValueField removes value for colID if present +// executeClearValueField removes value for colID if present func (e *executor) executeClearValueField(ctx context.Context, index string, c *pql.Call, f *Field, colID uint64, opt *execOptions) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearValueField") defer span.Finish() diff --git a/view.go b/view.go index 334d55d34..815e2a71e 100644 --- a/view.go +++ b/view.go @@ -408,9 +408,9 @@ func (v *view) setValue(columnID uint64, bitDepth uint, value int64) (changed bo // clearValue removes a specific value assigned to columnID func (v *view) clearValue(columnID uint64, bitDepth uint, value int64) (changed bool, err error) { shard := columnID / ShardWidth - frag, err := v.CreateFragmentIfNotExists(shard) - if err != nil { - return changed, err + frag := v.Fragment(shard) + if frag == nil { + return false, nil } return frag.clearValue(columnID, bitDepth, value) } From 26e3460413e7bd36dec4d2199c771b13b5e3565c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 7 Jan 2020 10:55:45 -0600 Subject: [PATCH 4/5] removed view creation from ClearValue --- field.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/field.go b/field.go index 079c0d7d3..06bb2ca1d 100644 --- a/field.go +++ b/field.go @@ -1186,7 +1186,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) return view.setValue(columnID, bsig.BitDepth, baseValue) } -// SetValue sets a field value for a column. +// ClearValue removes a field value for a column. func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) @@ -1195,9 +1195,9 @@ func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { } // Fetch target view. - view, err := f.createViewIfNotExists(viewBSIGroupPrefix + f.name) - if err != nil { - return false, errors.Wrap(err, "creating view") + view := f.view(viewBSIGroupPrefix + f.name) + if view == nil { + return false, nil } value, exists, err := view.value(columnID, bsig.BitDepth) if err != nil { From 643884aeb3dd5fe9b506eaabb1849912f0dd40fd Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 7 Jan 2020 12:55:32 -0600 Subject: [PATCH 5/5] refactored clear to fix q2;removed unused comment --- executor.go | 32 ++++++++++++++++++++------------ field.go | 2 -- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/executor.go b/executor.go index abd557837..db53dad27 100644 --- a/executor.go +++ b/executor.go @@ -2650,6 +2650,14 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeClearBit") defer span.Finish() + // Read colID + colID, ok, err := c.UintArg("_" + columnLabel) + if err != nil { + return false, fmt.Errorf("reading Clear() column: %v", err) + } else if !ok { + return false, fmt.Errorf("column argument to Clear(, =) required") + } + // Read field name. fieldName, err := c.FieldArg() if err != nil { return false, errors.New("Clear() argument required: field") @@ -2664,19 +2672,12 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal if f == nil { return false, ErrFieldNotFound } - // Read fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Clear() row: %v", err) - } else if !ok { - return false, fmt.Errorf("row= argument required to Clear() call") - } - colID, ok, err := c.UintArg("_" + columnLabel) - if err != nil { - return false, fmt.Errorf("reading Clear() column: %v", err) - } else if !ok { - return false, fmt.Errorf("column argument to Clear(, =) required") + // Clear column on existence field + if ef := idx.existenceField(); ef != nil { + if _, err := ef.ClearBit(0, colID); err != nil { + return false, errors.Wrap(err, "clearing existence column") + } } // Int field. @@ -2684,6 +2685,13 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return e.executeClearValueField(ctx, index, c, f, colID, opt) } + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Clear() row: %v", err) + } else if !ok { + return false, fmt.Errorf("row= argument required to Clear() call") + } + return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } diff --git a/field.go b/field.go index 06bb2ca1d..86c5204a8 100644 --- a/field.go +++ b/field.go @@ -1188,12 +1188,10 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) // ClearValue removes a field value for a column. func (f *Field) ClearValue(columnID uint64) (changed bool, err error) { - // Fetch bsiGroup & validate min/max. bsig := f.bsiGroup(f.name) if bsig == nil { return false, ErrBSIGroupNotFound } - // Fetch target view. view := f.view(viewBSIGroupPrefix + f.name) if view == nil {