diff --git a/executor.go b/executor.go index 610696022..51b82b5b8 100644 --- a/executor.go +++ b/executor.go @@ -1037,7 +1037,7 @@ func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pq for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.ClearBit(rowID, colID, nil) + val, err := f.ClearBit(rowID, colID) if err != nil { return false, err } else if val { diff --git a/executor_test.go b/executor_test.go index 250d8b13d..bb83faea7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -928,6 +928,18 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatalf("unexpected columns: %+v", columns) } }) + + t.Run("Clear", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Clear( 2, f=1)`), nil, nil); err != nil { + t.Fatal(err) + } + + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + t.Fatal(err) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) } // Ensure a Range(bsiGroup) query can be executed. diff --git a/field.go b/field.go index 4fd684682..f4100fa7e 100644 --- a/field.go +++ b/field.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" @@ -38,6 +39,10 @@ const ( // Default ranked field cache DefaultCacheSize = 50000 + + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 + ) // Field types. @@ -609,7 +614,6 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { if view := f.views[name]; view != nil { return view, false, nil } - view := f.newView(f.ViewPath(name), name) if err := view.open(); err != nil { @@ -715,13 +719,14 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // ClearBit clears a bit within the field. -func (f *Field) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { +func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { viewName := ViewStandard // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(viewName) - if err != nil { - return changed, errors.Wrap(err, "creating view") + view, present := f.views[viewName] + if !present { + return changed, errors.Wrap(err, "clearing missing view") + } // Clear non-time bit. @@ -730,29 +735,75 @@ func (f *Field) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err e } else if v { changed = v } - - // Exit early if no timestamp is specified. - if t == nil { + if len(f.views) == 1 { // assuming no time views return changed, nil } - - // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) + lastViewNameSize := 0 + level := 0 + skipAbove := maxInt + for _, view := range f.allTimeViewsSortedByQuantum() { + if lastViewNameSize < len(view.name) { + level++ + } else if lastViewNameSize > len(view.name) { + level-- } - - if c, err := view.clearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "clearing on view %s", subname) - } else if c { - changed = true + if level < skipAbove { + if changed, err = view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "clearing on view %s", view.name) + } + if !changed { + skipAbove = level + 1 + } else { + skipAbove = maxInt + } } + lastViewNameSize = len(view.name) } return changed, nil } +func groupCompare(a, b string, offset int) (lt, eq bool) { + if len(a) > offset { + a = a[:offset] + } + if len(b) > offset { + b = b[:offset] + } + v := strings.Compare(a, b) + return v < 0, v == 0 +} + +func (f *Field) allTimeViewsSortedByQuantum() (me []*View) { + me = make([]*View, len(f.views), len(f.views)) + prefix := ViewStandard + "_" + offset := len(ViewStandard) + 1 + i := 0 + for _, v := range f.views { + if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views + me[i] = v + i++ + } + } + me = me[:i] + year := strings.Index(me[0].name, "_") + 4 + month := year + 2 + day := month + 2 + sort.Slice(me, func(i, j int) (lt bool) { + var eq bool + // group by quantum from year to hour + if lt, eq = groupCompare(me[i].name, me[j].name, year); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, month); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, day); eq { + lt = strings.Compare(me[i].name, me[j].name) > 0 + } + } + } + return + }) + return +} + // Value reads a field value for a column. func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) diff --git a/internal/public.pb.go b/internal/public.pb.go index 3cb6fa270..ec00e2eec 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -27,7 +27,7 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" +import binary "encoding/binary" import io "io" @@ -800,7 +800,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } return i, nil @@ -2317,7 +2317,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 m.FloatValue = float64(math.Float64frombits(v)) default: diff --git a/test/holder.go b/test/holder.go index 648d910ed..1b900d2fe 100644 --- a/test/holder.go +++ b/test/holder.go @@ -152,7 +152,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.ClearBit(rowID, columnID, nil) + f.ClearBit(rowID, columnID) } // MustSetBits sets columns on a row. Panic on error. diff --git a/view.go b/view.go index 732ecd485..897c54815 100644 --- a/view.go +++ b/view.go @@ -57,9 +57,8 @@ type View struct { // prevent sending multiple `CreateSliceMessage` messages maxSlice uint64 - broadcaster Broadcaster - stats StatsClient - + broadcaster Broadcaster + stats StatsClient RowAttrStore AttrStore Logger Logger } @@ -318,9 +317,9 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { // clearBit clears a bit within the view. func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return changed, err + frag, found := v.fragments[slice] + if !found { + return false, nil } return frag.clearBit(rowID, columnID) }