From 35af5801832cdd29f1448bb5d790b5f9ce932254 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 26 Jun 2018 10:54:06 -0500 Subject: [PATCH 01/10] expanded views to contain viewType for special handling --- api.go | 2 +- cluster.go | 2 +- executor.go | 4 +- field.go | 136 ++++++++++++++++++++--------- fragment.go | 21 ++--- fragment_internal_test.go | 10 +-- holder.go | 4 +- internal/private.pb.go | 179 ++++++++++++++++++++++++-------------- internal/private.proto | 1 + internal/public.pb.go | 6 +- server.go | 2 +- test/holder.go | 4 +- time.go | 27 +++--- time_internal_test.go | 16 ++-- view.go | 14 +-- 15 files changed, 266 insertions(+), 162 deletions(-) diff --git a/api.go b/api.go index 28248230c..3b51b6ddd 100644 --- a/api.go +++ b/api.go @@ -399,7 +399,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa } // Retrieve view. - view, err := f.CreateViewIfNotExists(ViewStandard) + view, err := f.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/cluster.go b/cluster.go index 957fe20fd..1400f0daa 100644 --- a/cluster.go +++ b/cluster.go @@ -1226,7 +1226,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err } // Create view. - v, err := f.CreateViewIfNotExists(src.View) + v, err := f.CreateViewIfNotExists(viewTimeKey{name: src.View}) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/executor.go b/executor.go index 1fbf56960..f1ccd9006 100644 --- a/executor.go +++ b/executor.go @@ -785,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, fieldName, view, slice) + f := e.Holder.Fragment(index, fieldName, view.name, slice) if f == nil { continue } @@ -1037,7 +1037,7 @@ func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pq for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { - val, err := f.ClearBit(rowID, colID, nil) + val, err := f.ClearBit(rowID, colID) if err != nil { return false, err } else if val { diff --git a/field.go b/field.go index fda0ee364..3556dcd62 100644 --- a/field.go +++ b/field.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" @@ -37,6 +38,9 @@ const ( // Default ranked field cache defaultCacheSize = 50000 + + BitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + MaxInt = 1<<(BitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 ) // Field types. @@ -248,7 +252,7 @@ func (f *Field) openViews() error { } name := filepath.Base(fi.Name()) - view := f.newView(f.ViewPath(name), name) + view := f.newView(f.ViewPath(name), viewTimeKey{name: name}) if err := view.open(); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } @@ -555,9 +559,9 @@ func (f *Field) RecalculateCaches() { // CreateViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. -func (f *Field) CreateViewIfNotExists(name string) (*View, error) { +func (f *Field) CreateViewIfNotExists(vtk viewTimeKey) (*View, error) { - view, created, err := f.createViewIfNotExistsBase(name) + view, created, err := f.createViewIfNotExistsBase(vtk) if err != nil { return nil, err } @@ -568,7 +572,8 @@ func (f *Field) CreateViewIfNotExists(name string) (*View, error) { &internal.CreateViewMessage{ Index: f.index, Field: f.name, - View: name, + View: vtk.name, + Type: string(vtk.quantum), }) if err != nil { return nil, errors.Wrap(err, "sending CreateView message") @@ -580,15 +585,15 @@ func (f *Field) CreateViewIfNotExists(name string) (*View, error) { // createViewIfNotExistsBase returns the named view, creating it if necessary. // The returned bool indicates whether the view was created or not. -func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { +func (f *Field) createViewIfNotExistsBase(vtk viewTimeKey) (*View, bool, error) { f.mu.Lock() defer f.mu.Unlock() - if view := f.views[name]; view != nil { + if view := f.views[vtk.name]; view != nil { return view, false, nil } - view := f.newView(f.ViewPath(name), name) + view := f.newView(f.ViewPath(vtk.name), vtk) if err := view.open(); err != nil { return nil, false, errors.Wrap(err, "opening view") @@ -599,13 +604,14 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { return view, true, nil } -func (f *Field) newView(path, name string) *View { - view := NewView(path, f.index, f.name, name, f.options.CacheSize) +func (f *Field) newView(path string, vtk viewTimeKey) *View { + view := NewView(path, f.index, f.name, vtk.name, f.options.CacheSize) view.cacheType = f.options.CacheType view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore - view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) + view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", vtk.name)) view.broadcaster = f.broadcaster + view.viewType = vtk.quantum return view } @@ -655,7 +661,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { // SetBit sets a bit on a view within the field. func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { - viewName := ViewStandard + viewName := viewTimeKey{name: ViewStandard} // Retrieve view. Exit if it doesn't exist. view, err := f.CreateViewIfNotExists(viewName) @@ -676,14 +682,14 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // If a timestamp is specified then set bits across all views for the quantum. - for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) + for _, vtk := range viewsByTime(viewName.name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(vtk) if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) + return changed, errors.Wrapf(err, "creating view %s", vtk.name) } if c, err := view.setBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) + return changed, errors.Wrapf(err, "setting on view %s", vtk.name) } else if c { changed = true } @@ -693,44 +699,88 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // ClearBit clears a bit within the field. -func (f *Field) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { - viewName := ViewStandard +func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { + viewName := viewTimeKey{name: ViewStandard} // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(viewName) - if err != nil { - return changed, errors.Wrap(err, "creating view") + view, present := f.views[viewName.name] + if !present { + return changed, errors.Wrap(err, "clearing missing view") + } // Clear non-time bit. - if v, err := view.clearBit(rowID, colID); err != nil { + if v, _, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "clearing on view") } else if v { changed = v } - // Exit early if no timestamp is specified. - if t == nil { - return changed, nil - } - - // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) + process := true + anyRemaining := false + lastLevel := 0 + skipLevel := MaxInt //just setting to a bignum + for i, quantumView := range f.allTimeViewsSortedByQuantum() { + if process { + if changed, remainingBits, err := quantumView.clearBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "clearing on view %s", quantumView.name) + } else if remainingBits { //now empty implies that the row as just been cleared and removed + anyRemaining = true + } } + if i == 0 { + lastLevel = len(quantumView.name) + } else if lastLevel != len(quantumView.name) { + if lastLevel < len(quantumView.name) { + if anyRemaining { + skipLevel = lastLevel + process = false + anyRemaining = false + } + } else if lastLevel > len(quantumView.name) { + if len(quantumView.name) <= skipLevel { //skip no more + process = true + skipLevel = MaxInt + } + } - if c, err := view.clearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "clearing on view %s", subname) - } else if c { - changed = true } + lastLevel = len(quantumView.name) } return changed, nil } +func groupCompare(a, b string, offset int) (lt, eq bool) { + v := strings.Compare(a[:offset], b[:offset]) + return v < 0, v == 0 +} + +func (f *Field) allTimeViewsSortedByQuantum() (me []*View) { + me = make([]*View, len(f.views), len(f.views)) + for _, v := range f.views { + if v.viewType != 0 { // skip non-time views + me = append(me, v) + } + } + year := strings.Index(me[0].name, "_") + 4 + month := year + 2 + day := month + 2 + sort.Slice(me, func(i, j int) (lt bool) { + var eq bool + // ensure all catA are grouped together: + if lt, eq = groupCompare(me[i].name, me[j].name, year); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, month); eq { + if lt, eq = groupCompare(me[i].name, me[j].name, day); eq { + lt = strings.Compare(me[i].name, me[j].name) > 0 + } + } + } + return + }) + return +} + // Value reads a field value for a column. func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { bsig := f.bsiGroup(f.name) @@ -766,7 +816,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) } // Fetch target view. - view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name) + view, err := f.CreateViewIfNotExists(viewTimeKey{name: viewBSIGroupPrefix + f.name}) if err != nil { return false, errors.Wrap(err, "creating view") } @@ -900,19 +950,19 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro timestamp = timestamps[i] } - var standard []string + var standard []viewTimeKey if timestamp == nil { - standard = []string{ViewStandard} + standard = []viewTimeKey{{name: ViewStandard}} } else { standard = viewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. - standard = append(standard, ViewStandard) + standard = append(standard, viewTimeKey{name: ViewStandard}) } // Attach bit to each standard view. - for _, name := range standard { - key := importKey{View: name, Slice: columnID / SliceWidth} + for _, vtk := range standard { + key := importKey{View: vtk.name, Slice: columnID / SliceWidth} data := dataByFragment[key] data.RowIDs = append(data.RowIDs, rowID) data.ColumnIDs = append(data.ColumnIDs, columnID) @@ -922,7 +972,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Import into each fragment. for key, data := range dataByFragment { - view, err := f.CreateViewIfNotExists(key.View) + view, err := f.CreateViewIfNotExists(viewTimeKey{name: key.View}) if err != nil { return errors.Wrap(err, "creating view") } @@ -974,7 +1024,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { // The view must already exist (i.e. we can't create it) // because we need to know bitDepth (based on min/max value). - view, err := f.CreateViewIfNotExists(key.View) + view, err := f.CreateViewIfNotExists(viewTimeKey{name: key.View}) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/fragment.go b/fragment.go index 28978d337..3a828150d 100644 --- a/fragment.go +++ b/fragment.go @@ -412,28 +412,28 @@ func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err // clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (bool, bool, error) { f.mu.Lock() defer f.mu.Unlock() return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, remaining bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting bit pos") + return false, false, errors.Wrap(err, "getting bit pos") } // Write to storage. if changed, err = f.storage.Remove(pos); err != nil { - return false, errors.Wrap(err, "writing") + return false, false, errors.Wrap(err, "writing") } // Don't update the cache if nothing changed. if !changed { - return changed, nil + return changed, false, nil } // Invalidate block checksum. @@ -441,7 +441,7 @@ func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { - return false, errors.Wrap(err, "incrementing") + return false, false, errors.Wrap(err, "incrementing") } // Get the row from cache or fragment.storage. @@ -449,11 +449,12 @@ func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er row.ClearBit(columnID) // Update the cache. - f.cache.Add(rowID, row.Count()) + c := row.Count() + f.cache.Add(rowID, c) f.stats.Count("clearBit", 1, 1.0) - return changed, nil + return changed, c > 0, nil } func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { @@ -501,7 +502,7 @@ func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (chang changed = true } } else { - if c, err := f.unprotectedClearBit(uint64(i), columnID); err != nil { + if c, _, err := f.unprotectedClearBit(uint64(i), columnID); err != nil { return changed, err } else if c { changed = true @@ -1285,7 +1286,7 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Clear local bits. for i := range clears[0].columnIDs { - if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { + if _, _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 6c733ba4c..db4da534e 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -75,7 +75,7 @@ func TestFragment_ClearBit(t *testing.T) { t.Fatal(err) } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.clearBit(1000, 1); err != nil { + } else if _, _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } @@ -532,7 +532,7 @@ func TestFragment_Snapshot(t *testing.T) { t.Fatal(err) } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.clearBit(1000, 1); err != nil { + } else if _, _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } @@ -756,7 +756,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(ViewStandard) + view, err := field.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) if err != nil { t.Fatal(err) } @@ -922,7 +922,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(ViewStandard) + view, err := field.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) if err != nil { t.Fatal(err) } @@ -973,7 +973,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if _, err := f0.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f0.clearBit(1000, 1); err != nil { + } else if _, _, err := f0.clearBit(1000, 1); err != nil { t.Fatal(err) } diff --git a/holder.go b/holder.go index 316ca1244..037b27928 100644 --- a/holder.go +++ b/holder.go @@ -247,7 +247,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { } // Create views that don't exist. for _, v := range f.Views { - _, err := field.CreateViewIfNotExists(v) + _, err := field.CreateViewIfNotExists(viewTimeKey{name: v}) if err != nil { return errors.Wrap(err, "creating view") } @@ -744,7 +744,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err } // Ensure view exists locally. - v, err := f.CreateViewIfNotExists(view) + v, err := f.CreateViewIfNotExists(viewTimeKey{name: view}) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/internal/private.pb.go b/internal/private.pb.go index c3dadb455..7130a37dc 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -671,6 +671,7 @@ type CreateViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` + Type string `protobuf:"bytes,4,opt,name=Type,proto3" json:"Type,omitempty"` } func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } @@ -699,6 +700,13 @@ func (m *CreateViewMessage) GetView() string { return "" } +func (m *CreateViewMessage) GetType() string { + if m != nil { + return m.Type + } + return "" +} + type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1823,6 +1831,12 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } + if len(m.Type) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } return i, nil } @@ -2520,6 +2534,10 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -5539,6 +5557,35 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } m.View = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -6681,70 +6728,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1028 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, - 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, - 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, - 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, - 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, - 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, - 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, - 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, - 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, - 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, - 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, - 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, - 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, - 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, - 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, - 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, - 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, - 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, - 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, - 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, - 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, - 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, - 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, - 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, - 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, - 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, - 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, - 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, - 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, - 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, - 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, - 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, - 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, - 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, - 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, - 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, - 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, - 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, - 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, - 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, - 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, - 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, - 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, - 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, - 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, - 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, - 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, - 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, - 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, - 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, - 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, - 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, - 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, - 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, - 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, - 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, - 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, - 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, - 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, - 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, - 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, - 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, - 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, - 0x0a, 0x0b, 0x00, 0x00, + // 1034 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x73, 0xdb, 0x44, + 0x14, 0x47, 0x96, 0xec, 0xd8, 0x2f, 0x75, 0x48, 0xb6, 0x10, 0x54, 0x86, 0x49, 0xcd, 0x4e, 0x67, + 0x1a, 0x7a, 0xc8, 0x94, 0xf6, 0xc2, 0xbf, 0xce, 0x64, 0x62, 0x07, 0x10, 0x90, 0x00, 0xab, 0xa4, + 0xb7, 0x1e, 0xb6, 0xf6, 0x4e, 0xaa, 0x89, 0xac, 0x15, 0xd2, 0x2a, 0x89, 0x7b, 0xe0, 0x0a, 0x17, + 0xee, 0x0c, 0x9f, 0x84, 0x8f, 0xc0, 0x91, 0x8f, 0xc0, 0x84, 0x2f, 0xc2, 0xec, 0xdb, 0xd5, 0x9f, + 0xc4, 0x4e, 0xd3, 0x09, 0xbd, 0xed, 0xfb, 0xff, 0xd3, 0x7b, 0xbf, 0x7d, 0x2b, 0xe8, 0xa7, 0x59, + 0x74, 0xc2, 0x95, 0xd8, 0x4a, 0x33, 0xa9, 0x24, 0xe9, 0x46, 0x89, 0x12, 0x59, 0xc2, 0x63, 0x7a, + 0x17, 0x7a, 0x41, 0x32, 0x11, 0x67, 0x7b, 0x42, 0x71, 0x42, 0xc0, 0xfb, 0x56, 0xcc, 0x72, 0xdf, + 0x1d, 0x38, 0x9b, 0x5d, 0x86, 0x67, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8c, 0x44, 0x3c, 0xf9, 0x3e, + 0x55, 0x91, 0x4c, 0x72, 0xf2, 0x01, 0xf4, 0x86, 0x7c, 0xfc, 0x42, 0x1c, 0xcc, 0x52, 0x81, 0x9e, + 0x3d, 0x56, 0x2b, 0x2a, 0x6b, 0x18, 0xbd, 0x14, 0xbe, 0x37, 0x70, 0x36, 0xfb, 0xac, 0x56, 0x90, + 0x01, 0x2c, 0x1f, 0x44, 0x53, 0xf1, 0x63, 0xc1, 0x13, 0x55, 0x4c, 0xfd, 0x36, 0x46, 0x37, 0x55, + 0x1a, 0x02, 0x26, 0xee, 0xa2, 0x09, 0xcf, 0x64, 0x15, 0xdc, 0xbd, 0x28, 0xf1, 0x7b, 0x03, 0x67, + 0xd3, 0x65, 0xfa, 0x88, 0x1a, 0x7e, 0xe6, 0x83, 0xd5, 0xf0, 0xb3, 0x0a, 0xfa, 0x72, 0x03, 0x3a, + 0x85, 0x95, 0x60, 0x9a, 0xca, 0x4c, 0x31, 0x91, 0xa7, 0x32, 0xc9, 0x31, 0xd3, 0x6e, 0x96, 0xf9, + 0x0e, 0x26, 0xd7, 0x47, 0xfa, 0x33, 0xac, 0xee, 0xc4, 0x72, 0x7c, 0x3c, 0xe2, 0x8a, 0x33, 0xf1, + 0x53, 0x21, 0x72, 0x45, 0xde, 0x81, 0x36, 0xf6, 0xc4, 0xfa, 0x19, 0x41, 0x6b, 0xb1, 0x0f, 0x7e, + 0xcb, 0x68, 0x51, 0xd0, 0x5a, 0x8c, 0xc7, 0x4e, 0x78, 0xcc, 0x08, 0x5a, 0x1b, 0xc6, 0xd1, 0xd8, + 0x74, 0xc0, 0x63, 0x46, 0xd0, 0x18, 0x9f, 0x46, 0xe2, 0xd4, 0x7e, 0x36, 0x9e, 0x69, 0x00, 0x6b, + 0x8d, 0xfa, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, + 0x59, 0x09, 0x9b, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x53, 0x0b, 0x4d, 0xb5, 0x82, 0xde, 0x81, 0x36, + 0x76, 0x5a, 0x7f, 0x65, 0x1d, 0xab, 0x8f, 0xf4, 0x17, 0x07, 0x7a, 0x7b, 0xfc, 0x0c, 0x61, 0xe4, + 0xe4, 0x09, 0x74, 0x43, 0xc5, 0x93, 0x09, 0xcf, 0x26, 0xe8, 0xb4, 0xfc, 0xe8, 0xc3, 0xad, 0x92, + 0x10, 0x5b, 0x95, 0xdb, 0x56, 0xe9, 0xb3, 0x9b, 0xa8, 0x6c, 0xc6, 0xaa, 0x90, 0xf7, 0x3f, 0x87, + 0xfe, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x65, 0x57, 0x8f, 0xc5, 0x4c, 0x7f, 0xff, 0x09, 0x8f, + 0x0b, 0x81, 0xbd, 0xf2, 0x98, 0x11, 0x3e, 0x6b, 0x7d, 0xe2, 0xd0, 0x6d, 0x20, 0xc3, 0x4c, 0x70, + 0x25, 0xb0, 0xc8, 0x9e, 0xc8, 0x73, 0x7e, 0x24, 0xae, 0xee, 0xb8, 0xe9, 0x62, 0xab, 0xd1, 0x45, + 0xfa, 0x00, 0xc8, 0x48, 0xc4, 0x42, 0x09, 0xcb, 0xdb, 0x57, 0x64, 0xa0, 0x61, 0x59, 0xed, 0x7a, + 0x5f, 0x72, 0x1f, 0x3c, 0x7d, 0x09, 0xb0, 0xd8, 0xf2, 0xa3, 0xdb, 0x75, 0x47, 0xaa, 0xfb, 0xc1, + 0xd0, 0x81, 0xc6, 0x65, 0x52, 0x64, 0xc0, 0xb5, 0x9f, 0xb0, 0x80, 0x34, 0x0f, 0x6c, 0x29, 0x17, + 0x4b, 0xad, 0xd7, 0xa5, 0x9a, 0x17, 0xcd, 0x56, 0xdb, 0x2e, 0x3f, 0xf7, 0xa6, 0xd5, 0xe8, 0x33, + 0xab, 0xd5, 0xfc, 0xdb, 0xe7, 0x53, 0x61, 0x63, 0xf0, 0x5c, 0x41, 0x69, 0x5d, 0x0f, 0x45, 0xa7, + 0xd7, 0x9c, 0xd5, 0xfb, 0xc1, 0xd5, 0xe9, 0x51, 0xa0, 0x8f, 0xa1, 0x13, 0x8e, 0x5f, 0x88, 0x29, + 0x27, 0x1f, 0xc1, 0x12, 0xe2, 0x10, 0xb9, 0xa5, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x4a, 0x3b, 0x1d, + 0x59, 0xfc, 0x0b, 0x31, 0xdd, 0x87, 0x0e, 0x56, 0xcf, 0x7d, 0xef, 0x72, 0x1a, 0xd4, 0x33, 0x6b, + 0xa6, 0xbb, 0xe0, 0x1e, 0xb2, 0x40, 0x5f, 0x17, 0x44, 0x50, 0x66, 0xb1, 0x92, 0xce, 0xfd, 0xb5, + 0xcc, 0x95, 0xed, 0x06, 0x9e, 0xb5, 0xee, 0x07, 0x99, 0x29, 0x6c, 0x7d, 0x9f, 0xe1, 0x99, 0x3e, + 0x03, 0x6f, 0x5f, 0x4e, 0x04, 0x59, 0x81, 0x56, 0x30, 0xb2, 0x39, 0x5a, 0xc1, 0x88, 0xdc, 0xc5, + 0xf4, 0xb6, 0x35, 0xfd, 0x1a, 0xc4, 0x21, 0x0b, 0x18, 0x16, 0xbe, 0x07, 0xfd, 0x20, 0x1f, 0x4a, + 0x99, 0x4d, 0xa2, 0x84, 0x2b, 0x99, 0xd9, 0xc5, 0x79, 0x51, 0x49, 0xb7, 0x61, 0x55, 0xa7, 0x0f, + 0x15, 0x57, 0x15, 0xe1, 0xd7, 0xa1, 0xa3, 0x75, 0x55, 0x39, 0x2b, 0x21, 0xe5, 0xb5, 0x5f, 0x39, + 0x41, 0x14, 0xe8, 0x77, 0x26, 0xc3, 0xee, 0x89, 0x48, 0x54, 0x83, 0x01, 0x28, 0x63, 0x82, 0x3e, + 0x33, 0x02, 0xa1, 0xe6, 0x53, 0x2c, 0xe6, 0x95, 0x1a, 0xb3, 0xd6, 0x32, 0xb4, 0xd1, 0xdf, 0x1c, + 0x80, 0x12, 0x50, 0x91, 0x57, 0x21, 0xce, 0xd5, 0x21, 0xe4, 0xe3, 0xc6, 0xfa, 0x98, 0xbf, 0x20, + 0x95, 0x89, 0x35, 0x96, 0xcc, 0x66, 0x49, 0x0b, 0xcb, 0xf2, 0xd5, 0xda, 0xdf, 0xe8, 0xed, 0x98, + 0x38, 0x8d, 0xa0, 0x3f, 0x8c, 0x8b, 0x5c, 0x89, 0xcc, 0x22, 0xd2, 0x6b, 0xce, 0x28, 0xaa, 0xfe, + 0xd4, 0x8a, 0xc5, 0x2d, 0x22, 0xf7, 0xa0, 0xad, 0x91, 0x1a, 0x6e, 0xce, 0x7f, 0x86, 0x31, 0xd2, + 0xa7, 0xd0, 0xdd, 0x09, 0x83, 0xaf, 0x32, 0x59, 0xa4, 0x0b, 0x99, 0x57, 0xbe, 0x3e, 0xad, 0xf9, + 0xd7, 0xc7, 0x9d, 0x7b, 0x7d, 0xbc, 0xea, 0xf5, 0xa1, 0x47, 0xb0, 0x66, 0x56, 0x82, 0xbe, 0x12, + 0x37, 0xd9, 0x08, 0xe5, 0xd3, 0xe0, 0xd6, 0x4f, 0x43, 0x05, 0xc6, 0xab, 0xc1, 0xd0, 0x10, 0xd6, + 0xcc, 0x36, 0x78, 0x83, 0x85, 0xe8, 0x1f, 0x2d, 0x58, 0x63, 0x22, 0x8f, 0x5e, 0x8a, 0x20, 0xc9, + 0x55, 0x56, 0x8c, 0xf5, 0xa5, 0xd7, 0xf1, 0xdf, 0xc8, 0xe7, 0x76, 0x02, 0x2e, 0x33, 0xc2, 0xeb, + 0x10, 0x8c, 0x3c, 0x84, 0xe5, 0xcb, 0x97, 0x62, 0xde, 0xb5, 0xe9, 0x42, 0x1e, 0xc2, 0x52, 0x28, + 0x8b, 0x4c, 0xb3, 0xcb, 0x5c, 0xf9, 0xc6, 0x22, 0x32, 0xc8, 0x8c, 0x99, 0x95, 0x6e, 0x0d, 0x7a, + 0xb5, 0x5f, 0x4d, 0x2f, 0xf2, 0xe4, 0x12, 0xbd, 0xfc, 0x0e, 0x06, 0xbc, 0x57, 0x07, 0x5c, 0x30, + 0xb3, 0x8b, 0xde, 0xf4, 0x57, 0x07, 0x6e, 0x35, 0x21, 0xbc, 0xd6, 0x7d, 0xa9, 0x26, 0xd2, 0x5a, + 0x38, 0x11, 0x77, 0xd1, 0x44, 0xbc, 0xc6, 0xe8, 0xab, 0x97, 0xaf, 0xdd, 0x7c, 0xf9, 0x8e, 0xe1, + 0xce, 0xdc, 0x98, 0x86, 0x72, 0x9a, 0x6a, 0x3e, 0xfc, 0x8f, 0x71, 0xe9, 0x4d, 0x92, 0x65, 0x76, + 0x50, 0x3d, 0x66, 0x04, 0xfa, 0x29, 0xbc, 0x1b, 0x0a, 0xd5, 0x18, 0x52, 0xc9, 0xb6, 0x01, 0xb8, + 0xfb, 0xe2, 0xf4, 0x8a, 0xcf, 0xd7, 0x26, 0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0xa3, + 0xe8, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6, 0xf2, 0x68, 0x76, 0xcd, 0x26, 0xf0, 0x61, 0xc9, 0xac, + 0x4d, 0xf3, 0x33, 0xd4, 0x63, 0xa5, 0x48, 0x6f, 0x6b, 0x42, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, + 0x43, 0xff, 0x15, 0xe5, 0x3b, 0xab, 0x7f, 0x9d, 0x6f, 0x38, 0x7f, 0x9f, 0x6f, 0x38, 0xff, 0x9c, + 0x6f, 0x38, 0xbf, 0xff, 0xbb, 0xf1, 0xd6, 0xf3, 0x0e, 0xfe, 0x0d, 0x3f, 0xfe, 0x2f, 0x00, 0x00, + 0xff, 0xff, 0x41, 0xae, 0x0a, 0x68, 0x1e, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 23bb4886c..d12ee4a34 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -126,6 +126,7 @@ message CreateViewMessage { string Index = 1; string Field = 2; string View = 3; + string Type = 4; } message DeleteViewMessage { diff --git a/internal/public.pb.go b/internal/public.pb.go index 3cb6fa270..ec00e2eec 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -27,7 +27,7 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" +import binary "encoding/binary" import io "io" @@ -800,7 +800,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } return i, nil @@ -2317,7 +2317,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 m.FloatValue = float64(math.Float64frombits(v)) default: diff --git a/server.go b/server.go index 84a09a01d..6ffdeb0aa 100644 --- a/server.go +++ b/server.go @@ -458,7 +458,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) } - _, _, err := f.createViewIfNotExistsBase(obj.View) + _, _, err := f.createViewIfNotExistsBase(viewTimeKey{name: obj.View}) if err != nil { return err } diff --git a/test/holder.go b/test/holder.go index 7bae8afaa..e5acec257 100644 --- a/test/holder.go +++ b/test/holder.go @@ -96,7 +96,7 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, if err != nil { panic(err) } - v, err := f.CreateViewIfNotExists(view) + v, err := f.CreateViewIfNotExists(viewTimeKey{name: view}) if err != nil { panic(err) } @@ -152,7 +152,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.ClearBit(rowID, columnID, nil) + f.ClearBit(rowID, columnID) } // MustSetBits sets columns on a row. Panic on error. diff --git a/time.go b/time.go index def889304..95218e221 100644 --- a/time.go +++ b/time.go @@ -79,28 +79,33 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } +type viewTimeKey struct { + name string + quantum rune +} + // viewByTimeUnit returns the view name for time with a given quantum unit. -func viewByTimeUnit(name string, t time.Time, unit rune) string { +func viewByTimeUnit(name string, t time.Time, unit rune) viewTimeKey { switch unit { case 'Y': - return fmt.Sprintf("%s_%s", name, t.Format("2006")) + return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("2006")), quantum: unit} case 'M': - return fmt.Sprintf("%s_%s", name, t.Format("200601")) + return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("200601")), quantum: unit} case 'D': - return fmt.Sprintf("%s_%s", name, t.Format("20060102")) + return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("20060102")), quantum: unit} case 'H': - return fmt.Sprintf("%s_%s", name, t.Format("2006010215")) + return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("2006010215")), quantum: unit} default: - return "" + return viewTimeKey{} } } // viewsByTime returns a list of views for a given timestamp. -func viewsByTime(name string, t time.Time, q TimeQuantum) []string { - a := make([]string, 0, len(q)) +func viewsByTime(name string, t time.Time, q TimeQuantum) []viewTimeKey { + a := make([]viewTimeKey, 0, len(q)) for _, unit := range q { view := viewByTimeUnit(name, t, unit) - if view == "" { + if view.name == "" { continue } a = append(a, view) @@ -109,7 +114,7 @@ func viewsByTime(name string, t time.Time, q TimeQuantum) []string { } // viewsByTimeRange returns a list of views to traverse to query a time range. -func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []viewTimeKey { t := start // Save flags for performance. @@ -118,7 +123,7 @@ func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string hasDay := q.HasDay() hasHour := q.HasHour() - var results []string + var results []viewTimeKey // Walk up from smallest units to largest units. if hasHour || hasDay || hasMonth { diff --git a/time_internal_test.go b/time_internal_test.go index 2920685fc..75a321abf 100644 --- a/time_internal_test.go +++ b/time_internal_test.go @@ -42,23 +42,23 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { - t.Fatalf("unexpected name: %s", s) + if s := viewByTimeUnit("F", ts, 'Y'); s.name != "F_2000" { + t.Fatalf("unexpected name: %s", s.name) } }) t.Run("M", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { - t.Fatalf("unexpected name: %s", s) + if s := viewByTimeUnit("F", ts, 'M'); s.name != "F_200001" { + t.Fatalf("unexpected name: %s", s.name) } }) t.Run("D", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { - t.Fatalf("unexpected name: %s", s) + if s := viewByTimeUnit("F", ts, 'D'); s.name != "F_20000102" { + t.Fatalf("unexpected name: %s", s.name) } }) t.Run("H", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { - t.Fatalf("unexpected name: %s", s) + if s := viewByTimeUnit("F", ts, 'H'); s.name != "F_2000010203" { + t.Fatalf("unexpected name: %s", s.name) } }) } diff --git a/view.go b/view.go index 428fc6f54..7825a42e9 100644 --- a/view.go +++ b/view.go @@ -57,9 +57,9 @@ type View struct { // prevent sending multiple `CreateSliceMessage` messages maxSlice uint64 - broadcaster Broadcaster - stats StatsClient - + broadcaster Broadcaster + stats StatsClient + viewType rune RowAttrStore AttrStore Logger Logger } @@ -316,11 +316,11 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { } // clearBit clears a bit within the view. -func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (v *View) clearBit(rowID, columnID uint64) (changed bool, remaining bool, err error) { slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return changed, err + frag, found := v.fragments[slice] + if !found { + return false, false, nil } return frag.clearBit(rowID, columnID) } From 5d43d414f7f14185e029bc2ea1adf0762a7a558d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 27 Jun 2018 16:59:35 -0500 Subject: [PATCH 02/10] Fix a few data races --- http/translator_test.go | 17 +++++++++++------ translate.go | 4 +++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/http/translator_test.go b/http/translator_test.go index d317944e4..fdfc93f3b 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" gohttp "net/http" + "sync/atomic" "testing" "time" @@ -37,9 +38,10 @@ func TestTranslateStore_Reader(t *testing.T) { return 0, nil } } - var closeInvoked bool + closeInvoked := atomic.Value{} + closeInvoked.Store(false) mrc.CloseFunc = func() error { - closeInvoked = true + closeInvoked.Store(true) return nil } @@ -85,7 +87,7 @@ func TestTranslateStore_Reader(t *testing.T) { t.Fatal(err) } - if !closeInvoked { + if !closeInvoked.Load().(bool) { t.Fatal("expected server close") } }) @@ -100,9 +102,12 @@ func TestTranslateStore_Reader(t *testing.T) { <-done return 0, io.EOF } - var closeInvoked bool + + closeInvoked := atomic.Value{} + closeInvoked.Store(false) + mrc.CloseFunc = func() error { - closeInvoked = true + closeInvoked.Store(true) return nil } @@ -127,7 +132,7 @@ func TestTranslateStore_Reader(t *testing.T) { // Cancel the context and check if server is closed. cancel() time.Sleep(100 * time.Millisecond) - if !closeInvoked { + if !closeInvoked.Load().(bool) { t.Fatal("expected server-side close") } }) diff --git a/translate.go b/translate.go index 398055fdb..7c716640b 100644 --- a/translate.go +++ b/translate.go @@ -306,11 +306,13 @@ func (s *TranslateFile) replicate(ctx context.Context) error { } else if err != nil { return err } - + s.mu.Lock() // Write to local store. if err := s.appendEntry(&entry); err != nil { + s.mu.Unlock() return err } + s.mu.Unlock() } } From 4970083d4d717e07b257ac4a1ac002cdcf08c77a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 27 Jun 2018 18:15:44 -0500 Subject: [PATCH 03/10] refactored strategy for time based clearbit --- api.go | 2 +- cluster.go | 2 +- executor.go | 2 +- executor_test.go | 12 +++ field.go | 119 +++++++++++++------------ fragment.go | 21 +++-- fragment_internal_test.go | 10 +-- holder.go | 4 +- internal/private.pb.go | 179 ++++++++++++++------------------------ internal/private.proto | 1 - server.go | 2 +- test/holder.go | 2 +- time.go | 27 +++--- time_internal_test.go | 16 ++-- view.go | 6 +- 15 files changed, 183 insertions(+), 222 deletions(-) diff --git a/api.go b/api.go index 265873376..98fd5d717 100644 --- a/api.go +++ b/api.go @@ -387,7 +387,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa } // Retrieve view. - view, err := f.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) + view, err := f.CreateViewIfNotExists(ViewStandard) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/cluster.go b/cluster.go index 2feb8d677..ae34bf03c 100644 --- a/cluster.go +++ b/cluster.go @@ -1221,7 +1221,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err } // Create view. - v, err := f.CreateViewIfNotExists(viewTimeKey{name: src.View}) + v, err := f.CreateViewIfNotExists(src.View) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/executor.go b/executor.go index ed014e423..51b82b5b8 100644 --- a/executor.go +++ b/executor.go @@ -785,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, fieldName, view.name, slice) + f := e.Holder.Fragment(index, fieldName, view, slice) if f == nil { continue } diff --git a/executor_test.go b/executor_test.go index 022219235..e44d7783a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -928,6 +928,18 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatalf("unexpected columns: %+v", columns) } }) + + t.Run("Clear", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Clear( 2, f=1)`), nil, nil); err != nil { + t.Fatal(err) + } + + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + t.Fatal(err) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) } // Ensure a Range(bsiGroup) query can be executed. diff --git a/field.go b/field.go index 3556dcd62..0c137dfdc 100644 --- a/field.go +++ b/field.go @@ -252,7 +252,7 @@ func (f *Field) openViews() error { } name := filepath.Base(fi.Name()) - view := f.newView(f.ViewPath(name), viewTimeKey{name: name}) + view := f.newView(f.ViewPath(name), name) if err := view.open(); err != nil { return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) } @@ -559,9 +559,9 @@ func (f *Field) RecalculateCaches() { // CreateViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. -func (f *Field) CreateViewIfNotExists(vtk viewTimeKey) (*View, error) { +func (f *Field) CreateViewIfNotExists(name string) (*View, error) { - view, created, err := f.createViewIfNotExistsBase(vtk) + view, created, err := f.createViewIfNotExistsBase(name) if err != nil { return nil, err } @@ -572,8 +572,7 @@ func (f *Field) CreateViewIfNotExists(vtk viewTimeKey) (*View, error) { &internal.CreateViewMessage{ Index: f.index, Field: f.name, - View: vtk.name, - Type: string(vtk.quantum), + View: view.name, }) if err != nil { return nil, errors.Wrap(err, "sending CreateView message") @@ -585,15 +584,14 @@ func (f *Field) CreateViewIfNotExists(vtk viewTimeKey) (*View, error) { // createViewIfNotExistsBase returns the named view, creating it if necessary. // The returned bool indicates whether the view was created or not. -func (f *Field) createViewIfNotExistsBase(vtk viewTimeKey) (*View, bool, error) { +func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { f.mu.Lock() defer f.mu.Unlock() - if view := f.views[vtk.name]; view != nil { + if view := f.views[name]; view != nil { return view, false, nil } - - view := f.newView(f.ViewPath(vtk.name), vtk) + view := f.newView(f.ViewPath(name), name) if err := view.open(); err != nil { return nil, false, errors.Wrap(err, "opening view") @@ -604,14 +602,13 @@ func (f *Field) createViewIfNotExistsBase(vtk viewTimeKey) (*View, bool, error) return view, true, nil } -func (f *Field) newView(path string, vtk viewTimeKey) *View { - view := NewView(path, f.index, f.name, vtk.name, f.options.CacheSize) +func (f *Field) newView(path string, name string) *View { + view := NewView(path, f.index, f.name, name, f.options.CacheSize) view.cacheType = f.options.CacheType view.Logger = f.Logger view.RowAttrStore = f.rowAttrStore - view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", vtk.name)) + view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) view.broadcaster = f.broadcaster - view.viewType = vtk.quantum return view } @@ -661,7 +658,7 @@ func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { // SetBit sets a bit on a view within the field. func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { - viewName := viewTimeKey{name: ViewStandard} + viewName := ViewStandard // Retrieve view. Exit if it doesn't exist. view, err := f.CreateViewIfNotExists(viewName) @@ -682,14 +679,14 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // If a timestamp is specified then set bits across all views for the quantum. - for _, vtk := range viewsByTime(viewName.name, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(vtk) + for _, name := range viewsByTime(viewName, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(name) if err != nil { - return changed, errors.Wrapf(err, "creating view %s", vtk.name) + return changed, errors.Wrapf(err, "creating view %s", name) } if c, err := view.setBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", vtk.name) + return changed, errors.Wrapf(err, "setting on view %s", name) } else if c { changed = true } @@ -700,69 +697,75 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err // ClearBit clears a bit within the field. func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { - viewName := viewTimeKey{name: ViewStandard} + viewName := ViewStandard // Retrieve view. Exit if it doesn't exist. - view, present := f.views[viewName.name] + view, present := f.views[viewName] if !present { return changed, errors.Wrap(err, "clearing missing view") } // Clear non-time bit. - if v, _, err := view.clearBit(rowID, colID); err != nil { + if v, err := view.clearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "clearing on view") } else if v { changed = v } - - process := true - anyRemaining := false + if len(f.views) == 1 { // assuming no time views + return changed, nil + } lastLevel := 0 - skipLevel := MaxInt //just setting to a bignum - for i, quantumView := range f.allTimeViewsSortedByQuantum() { - if process { - if changed, remainingBits, err := quantumView.clearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "clearing on view %s", quantumView.name) - } else if remainingBits { //now empty implies that the row as just been cleared and removed - anyRemaining = true - } + level := 0 + skipBelow := MaxInt + for _, view := range f.allTimeViewsSortedByQuantum() { + if lastLevel < len(view.name) { + level++ + } else if lastLevel > len(view.name) { + level-- } - if i == 0 { - lastLevel = len(quantumView.name) - } else if lastLevel != len(quantumView.name) { - if lastLevel < len(quantumView.name) { - if anyRemaining { - skipLevel = lastLevel - process = false - anyRemaining = false - } - } else if lastLevel > len(quantumView.name) { - if len(quantumView.name) <= skipLevel { //skip no more - process = true - skipLevel = MaxInt - } + if level < skipBelow { + if changed, err = view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "clearing on view %s", view.name) } + if !changed { + if level < skipBelow { + skipBelow = level + 1 + } + } else { + skipBelow = MaxInt + } } - lastLevel = len(quantumView.name) + lastLevel = len(view.name) } return changed, nil } func groupCompare(a, b string, offset int) (lt, eq bool) { - v := strings.Compare(a[:offset], b[:offset]) + if len(a) > offset { + a = a[:offset] + } + if len(b) > offset { + b = b[:offset] + } + v := strings.Compare(a, b) return v < 0, v == 0 } func (f *Field) allTimeViewsSortedByQuantum() (me []*View) { me = make([]*View, len(f.views), len(f.views)) + prefix := ViewStandard + "_" + offset := len(ViewStandard) + 1 + i := 0 for _, v := range f.views { - if v.viewType != 0 { // skip non-time views - me = append(me, v) + if len(v.name) > offset && strings.Compare(v.name[:offset], prefix) == 0 { // skip non-time views + me[i] = v + i++ } } + me = me[:i] year := strings.Index(me[0].name, "_") + 4 month := year + 2 day := month + 2 @@ -816,7 +819,7 @@ func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) } // Fetch target view. - view, err := f.CreateViewIfNotExists(viewTimeKey{name: viewBSIGroupPrefix + f.name}) + view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name) if err != nil { return false, errors.Wrap(err, "creating view") } @@ -950,19 +953,19 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro timestamp = timestamps[i] } - var standard []viewTimeKey + var standard []string if timestamp == nil { - standard = []viewTimeKey{{name: ViewStandard}} + standard = []string{ViewStandard} } else { standard = viewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. - standard = append(standard, viewTimeKey{name: ViewStandard}) + standard = append(standard, ViewStandard) } // Attach bit to each standard view. - for _, vtk := range standard { - key := importKey{View: vtk.name, Slice: columnID / SliceWidth} + for _, name := range standard { + key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] data.RowIDs = append(data.RowIDs, rowID) data.ColumnIDs = append(data.ColumnIDs, columnID) @@ -972,7 +975,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Import into each fragment. for key, data := range dataByFragment { - view, err := f.CreateViewIfNotExists(viewTimeKey{name: key.View}) + view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") } @@ -1024,7 +1027,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { // The view must already exist (i.e. we can't create it) // because we need to know bitDepth (based on min/max value). - view, err := f.CreateViewIfNotExists(viewTimeKey{name: key.View}) + view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/fragment.go b/fragment.go index 3a828150d..28978d337 100644 --- a/fragment.go +++ b/fragment.go @@ -412,28 +412,28 @@ func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err // clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) clearBit(rowID, columnID uint64) (bool, bool, error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, remaining bool, err error) { +func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, false, errors.Wrap(err, "getting bit pos") + return false, errors.Wrap(err, "getting bit pos") } // Write to storage. if changed, err = f.storage.Remove(pos); err != nil { - return false, false, errors.Wrap(err, "writing") + return false, errors.Wrap(err, "writing") } // Don't update the cache if nothing changed. if !changed { - return changed, false, nil + return changed, nil } // Invalidate block checksum. @@ -441,7 +441,7 @@ func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, re // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { - return false, false, errors.Wrap(err, "incrementing") + return false, errors.Wrap(err, "incrementing") } // Get the row from cache or fragment.storage. @@ -449,12 +449,11 @@ func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, re row.ClearBit(columnID) // Update the cache. - c := row.Count() - f.cache.Add(rowID, c) + f.cache.Add(rowID, row.Count()) f.stats.Count("clearBit", 1, 1.0) - return changed, c > 0, nil + return changed, nil } func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { @@ -502,7 +501,7 @@ func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (chang changed = true } } else { - if c, _, err := f.unprotectedClearBit(uint64(i), columnID); err != nil { + if c, err := f.unprotectedClearBit(uint64(i), columnID); err != nil { return changed, err } else if c { changed = true @@ -1286,7 +1285,7 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Clear local bits. for i := range clears[0].columnIDs { - if _, _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index db4da534e..6c733ba4c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -75,7 +75,7 @@ func TestFragment_ClearBit(t *testing.T) { t.Fatal(err) } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, _, err := f.clearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } @@ -532,7 +532,7 @@ func TestFragment_Snapshot(t *testing.T) { t.Fatal(err) } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, _, err := f.clearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } @@ -756,7 +756,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -922,7 +922,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Create view. - view, err := field.CreateViewIfNotExists(viewTimeKey{name: ViewStandard}) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -973,7 +973,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if _, err := f0.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, _, err := f0.clearBit(1000, 1); err != nil { + } else if _, err := f0.clearBit(1000, 1); err != nil { t.Fatal(err) } diff --git a/holder.go b/holder.go index 037b27928..316ca1244 100644 --- a/holder.go +++ b/holder.go @@ -247,7 +247,7 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { } // Create views that don't exist. for _, v := range f.Views { - _, err := field.CreateViewIfNotExists(viewTimeKey{name: v}) + _, err := field.CreateViewIfNotExists(v) if err != nil { return errors.Wrap(err, "creating view") } @@ -744,7 +744,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err } // Ensure view exists locally. - v, err := f.CreateViewIfNotExists(viewTimeKey{name: view}) + v, err := f.CreateViewIfNotExists(view) if err != nil { return errors.Wrap(err, "creating view") } diff --git a/internal/private.pb.go b/internal/private.pb.go index 7130a37dc..c3dadb455 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -671,7 +671,6 @@ type CreateViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` - Type string `protobuf:"bytes,4,opt,name=Type,proto3" json:"Type,omitempty"` } func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } @@ -700,13 +699,6 @@ func (m *CreateViewMessage) GetView() string { return "" } -func (m *CreateViewMessage) GetType() string { - if m != nil { - return m.Type - } - return "" -} - type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -1831,12 +1823,6 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if len(m.Type) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) - i += copy(dAtA[i:], m.Type) - } return i, nil } @@ -2534,10 +2520,6 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Type) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } return n } @@ -5557,35 +5539,6 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { } m.View = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Type = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -6728,70 +6681,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1034 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x73, 0xdb, 0x44, - 0x14, 0x47, 0x96, 0xec, 0xd8, 0x2f, 0x75, 0x48, 0xb6, 0x10, 0x54, 0x86, 0x49, 0xcd, 0x4e, 0x67, - 0x1a, 0x7a, 0xc8, 0x94, 0xf6, 0xc2, 0xbf, 0xce, 0x64, 0x62, 0x07, 0x10, 0x90, 0x00, 0xab, 0xa4, - 0xb7, 0x1e, 0xb6, 0xf6, 0x4e, 0xaa, 0x89, 0xac, 0x15, 0xd2, 0x2a, 0x89, 0x7b, 0xe0, 0x0a, 0x17, - 0xee, 0x0c, 0x9f, 0x84, 0x8f, 0xc0, 0x91, 0x8f, 0xc0, 0x84, 0x2f, 0xc2, 0xec, 0xdb, 0xd5, 0x9f, - 0xc4, 0x4e, 0xd3, 0x09, 0xbd, 0xed, 0xfb, 0xff, 0xd3, 0x7b, 0xbf, 0x7d, 0x2b, 0xe8, 0xa7, 0x59, - 0x74, 0xc2, 0x95, 0xd8, 0x4a, 0x33, 0xa9, 0x24, 0xe9, 0x46, 0x89, 0x12, 0x59, 0xc2, 0x63, 0x7a, - 0x17, 0x7a, 0x41, 0x32, 0x11, 0x67, 0x7b, 0x42, 0x71, 0x42, 0xc0, 0xfb, 0x56, 0xcc, 0x72, 0xdf, - 0x1d, 0x38, 0x9b, 0x5d, 0x86, 0x67, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8c, 0x44, 0x3c, 0xf9, 0x3e, - 0x55, 0x91, 0x4c, 0x72, 0xf2, 0x01, 0xf4, 0x86, 0x7c, 0xfc, 0x42, 0x1c, 0xcc, 0x52, 0x81, 0x9e, - 0x3d, 0x56, 0x2b, 0x2a, 0x6b, 0x18, 0xbd, 0x14, 0xbe, 0x37, 0x70, 0x36, 0xfb, 0xac, 0x56, 0x90, - 0x01, 0x2c, 0x1f, 0x44, 0x53, 0xf1, 0x63, 0xc1, 0x13, 0x55, 0x4c, 0xfd, 0x36, 0x46, 0x37, 0x55, - 0x1a, 0x02, 0x26, 0xee, 0xa2, 0x09, 0xcf, 0x64, 0x15, 0xdc, 0xbd, 0x28, 0xf1, 0x7b, 0x03, 0x67, - 0xd3, 0x65, 0xfa, 0x88, 0x1a, 0x7e, 0xe6, 0x83, 0xd5, 0xf0, 0xb3, 0x0a, 0xfa, 0x72, 0x03, 0x3a, - 0x85, 0x95, 0x60, 0x9a, 0xca, 0x4c, 0x31, 0x91, 0xa7, 0x32, 0xc9, 0x31, 0xd3, 0x6e, 0x96, 0xf9, - 0x0e, 0x26, 0xd7, 0x47, 0xfa, 0x33, 0xac, 0xee, 0xc4, 0x72, 0x7c, 0x3c, 0xe2, 0x8a, 0x33, 0xf1, - 0x53, 0x21, 0x72, 0x45, 0xde, 0x81, 0x36, 0xf6, 0xc4, 0xfa, 0x19, 0x41, 0x6b, 0xb1, 0x0f, 0x7e, - 0xcb, 0x68, 0x51, 0xd0, 0x5a, 0x8c, 0xc7, 0x4e, 0x78, 0xcc, 0x08, 0x5a, 0x1b, 0xc6, 0xd1, 0xd8, - 0x74, 0xc0, 0x63, 0x46, 0xd0, 0x18, 0x9f, 0x46, 0xe2, 0xd4, 0x7e, 0x36, 0x9e, 0x69, 0x00, 0x6b, - 0x8d, 0xfa, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, - 0x59, 0x09, 0x9b, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x53, 0x0b, 0x4d, 0xb5, 0x82, 0xde, 0x81, 0x36, - 0x76, 0x5a, 0x7f, 0x65, 0x1d, 0xab, 0x8f, 0xf4, 0x17, 0x07, 0x7a, 0x7b, 0xfc, 0x0c, 0x61, 0xe4, - 0xe4, 0x09, 0x74, 0x43, 0xc5, 0x93, 0x09, 0xcf, 0x26, 0xe8, 0xb4, 0xfc, 0xe8, 0xc3, 0xad, 0x92, - 0x10, 0x5b, 0x95, 0xdb, 0x56, 0xe9, 0xb3, 0x9b, 0xa8, 0x6c, 0xc6, 0xaa, 0x90, 0xf7, 0x3f, 0x87, - 0xfe, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x65, 0x57, 0x8f, 0xc5, 0x4c, 0x7f, 0xff, 0x09, 0x8f, - 0x0b, 0x81, 0xbd, 0xf2, 0x98, 0x11, 0x3e, 0x6b, 0x7d, 0xe2, 0xd0, 0x6d, 0x20, 0xc3, 0x4c, 0x70, - 0x25, 0xb0, 0xc8, 0x9e, 0xc8, 0x73, 0x7e, 0x24, 0xae, 0xee, 0xb8, 0xe9, 0x62, 0xab, 0xd1, 0x45, - 0xfa, 0x00, 0xc8, 0x48, 0xc4, 0x42, 0x09, 0xcb, 0xdb, 0x57, 0x64, 0xa0, 0x61, 0x59, 0xed, 0x7a, - 0x5f, 0x72, 0x1f, 0x3c, 0x7d, 0x09, 0xb0, 0xd8, 0xf2, 0xa3, 0xdb, 0x75, 0x47, 0xaa, 0xfb, 0xc1, - 0xd0, 0x81, 0xc6, 0x65, 0x52, 0x64, 0xc0, 0xb5, 0x9f, 0xb0, 0x80, 0x34, 0x0f, 0x6c, 0x29, 0x17, - 0x4b, 0xad, 0xd7, 0xa5, 0x9a, 0x17, 0xcd, 0x56, 0xdb, 0x2e, 0x3f, 0xf7, 0xa6, 0xd5, 0xe8, 0x33, - 0xab, 0xd5, 0xfc, 0xdb, 0xe7, 0x53, 0x61, 0x63, 0xf0, 0x5c, 0x41, 0x69, 0x5d, 0x0f, 0x45, 0xa7, - 0xd7, 0x9c, 0xd5, 0xfb, 0xc1, 0xd5, 0xe9, 0x51, 0xa0, 0x8f, 0xa1, 0x13, 0x8e, 0x5f, 0x88, 0x29, - 0x27, 0x1f, 0xc1, 0x12, 0xe2, 0x10, 0xb9, 0xa5, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x4a, 0x3b, 0x1d, - 0x59, 0xfc, 0x0b, 0x31, 0xdd, 0x87, 0x0e, 0x56, 0xcf, 0x7d, 0xef, 0x72, 0x1a, 0xd4, 0x33, 0x6b, - 0xa6, 0xbb, 0xe0, 0x1e, 0xb2, 0x40, 0x5f, 0x17, 0x44, 0x50, 0x66, 0xb1, 0x92, 0xce, 0xfd, 0xb5, - 0xcc, 0x95, 0xed, 0x06, 0x9e, 0xb5, 0xee, 0x07, 0x99, 0x29, 0x6c, 0x7d, 0x9f, 0xe1, 0x99, 0x3e, - 0x03, 0x6f, 0x5f, 0x4e, 0x04, 0x59, 0x81, 0x56, 0x30, 0xb2, 0x39, 0x5a, 0xc1, 0x88, 0xdc, 0xc5, - 0xf4, 0xb6, 0x35, 0xfd, 0x1a, 0xc4, 0x21, 0x0b, 0x18, 0x16, 0xbe, 0x07, 0xfd, 0x20, 0x1f, 0x4a, - 0x99, 0x4d, 0xa2, 0x84, 0x2b, 0x99, 0xd9, 0xc5, 0x79, 0x51, 0x49, 0xb7, 0x61, 0x55, 0xa7, 0x0f, - 0x15, 0x57, 0x15, 0xe1, 0xd7, 0xa1, 0xa3, 0x75, 0x55, 0x39, 0x2b, 0x21, 0xe5, 0xb5, 0x5f, 0x39, - 0x41, 0x14, 0xe8, 0x77, 0x26, 0xc3, 0xee, 0x89, 0x48, 0x54, 0x83, 0x01, 0x28, 0x63, 0x82, 0x3e, - 0x33, 0x02, 0xa1, 0xe6, 0x53, 0x2c, 0xe6, 0x95, 0x1a, 0xb3, 0xd6, 0x32, 0xb4, 0xd1, 0xdf, 0x1c, - 0x80, 0x12, 0x50, 0x91, 0x57, 0x21, 0xce, 0xd5, 0x21, 0xe4, 0xe3, 0xc6, 0xfa, 0x98, 0xbf, 0x20, - 0x95, 0x89, 0x35, 0x96, 0xcc, 0x66, 0x49, 0x0b, 0xcb, 0xf2, 0xd5, 0xda, 0xdf, 0xe8, 0xed, 0x98, - 0x38, 0x8d, 0xa0, 0x3f, 0x8c, 0x8b, 0x5c, 0x89, 0xcc, 0x22, 0xd2, 0x6b, 0xce, 0x28, 0xaa, 0xfe, - 0xd4, 0x8a, 0xc5, 0x2d, 0x22, 0xf7, 0xa0, 0xad, 0x91, 0x1a, 0x6e, 0xce, 0x7f, 0x86, 0x31, 0xd2, - 0xa7, 0xd0, 0xdd, 0x09, 0x83, 0xaf, 0x32, 0x59, 0xa4, 0x0b, 0x99, 0x57, 0xbe, 0x3e, 0xad, 0xf9, - 0xd7, 0xc7, 0x9d, 0x7b, 0x7d, 0xbc, 0xea, 0xf5, 0xa1, 0x47, 0xb0, 0x66, 0x56, 0x82, 0xbe, 0x12, - 0x37, 0xd9, 0x08, 0xe5, 0xd3, 0xe0, 0xd6, 0x4f, 0x43, 0x05, 0xc6, 0xab, 0xc1, 0xd0, 0x10, 0xd6, - 0xcc, 0x36, 0x78, 0x83, 0x85, 0xe8, 0x1f, 0x2d, 0x58, 0x63, 0x22, 0x8f, 0x5e, 0x8a, 0x20, 0xc9, - 0x55, 0x56, 0x8c, 0xf5, 0xa5, 0xd7, 0xf1, 0xdf, 0xc8, 0xe7, 0x76, 0x02, 0x2e, 0x33, 0xc2, 0xeb, - 0x10, 0x8c, 0x3c, 0x84, 0xe5, 0xcb, 0x97, 0x62, 0xde, 0xb5, 0xe9, 0x42, 0x1e, 0xc2, 0x52, 0x28, - 0x8b, 0x4c, 0xb3, 0xcb, 0x5c, 0xf9, 0xc6, 0x22, 0x32, 0xc8, 0x8c, 0x99, 0x95, 0x6e, 0x0d, 0x7a, - 0xb5, 0x5f, 0x4d, 0x2f, 0xf2, 0xe4, 0x12, 0xbd, 0xfc, 0x0e, 0x06, 0xbc, 0x57, 0x07, 0x5c, 0x30, - 0xb3, 0x8b, 0xde, 0xf4, 0x57, 0x07, 0x6e, 0x35, 0x21, 0xbc, 0xd6, 0x7d, 0xa9, 0x26, 0xd2, 0x5a, - 0x38, 0x11, 0x77, 0xd1, 0x44, 0xbc, 0xc6, 0xe8, 0xab, 0x97, 0xaf, 0xdd, 0x7c, 0xf9, 0x8e, 0xe1, - 0xce, 0xdc, 0x98, 0x86, 0x72, 0x9a, 0x6a, 0x3e, 0xfc, 0x8f, 0x71, 0xe9, 0x4d, 0x92, 0x65, 0x76, - 0x50, 0x3d, 0x66, 0x04, 0xfa, 0x29, 0xbc, 0x1b, 0x0a, 0xd5, 0x18, 0x52, 0xc9, 0xb6, 0x01, 0xb8, - 0xfb, 0xe2, 0xf4, 0x8a, 0xcf, 0xd7, 0x26, 0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0xa3, - 0xe8, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6, 0xf2, 0x68, 0x76, 0xcd, 0x26, 0xf0, 0x61, 0xc9, 0xac, - 0x4d, 0xf3, 0x33, 0xd4, 0x63, 0xa5, 0x48, 0x6f, 0x6b, 0x42, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, - 0x43, 0xff, 0x15, 0xe5, 0x3b, 0xab, 0x7f, 0x9d, 0x6f, 0x38, 0x7f, 0x9f, 0x6f, 0x38, 0xff, 0x9c, - 0x6f, 0x38, 0xbf, 0xff, 0xbb, 0xf1, 0xd6, 0xf3, 0x0e, 0xfe, 0x0d, 0x3f, 0xfe, 0x2f, 0x00, 0x00, - 0xff, 0xff, 0x41, 0xae, 0x0a, 0x68, 0x1e, 0x0b, 0x00, 0x00, + // 1028 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, + 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, + 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, + 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, + 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, + 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, + 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, + 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, + 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, + 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, + 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, + 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, + 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, + 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, + 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, + 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, + 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, + 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, + 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, + 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, + 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, + 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, + 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, + 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, + 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, + 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, + 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, + 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, + 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, + 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, + 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, + 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, + 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, + 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, + 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, + 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, + 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, + 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, + 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, + 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, + 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, + 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, + 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, + 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, + 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, + 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, + 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, + 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, + 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, + 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, + 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, + 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, + 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, + 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, + 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, + 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, + 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, + 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, + 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, + 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, + 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, + 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, + 0x0a, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index d12ee4a34..23bb4886c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -126,7 +126,6 @@ message CreateViewMessage { string Index = 1; string Field = 2; string View = 3; - string Type = 4; } message DeleteViewMessage { diff --git a/server.go b/server.go index c04e7e4fd..74bedab5b 100644 --- a/server.go +++ b/server.go @@ -453,7 +453,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if f == nil { return fmt.Errorf("Local Field not found: %s", obj.Field) } - _, _, err := f.createViewIfNotExistsBase(viewTimeKey{name: obj.View}) + _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { return err } diff --git a/test/holder.go b/test/holder.go index e5acec257..f4cb73c2e 100644 --- a/test/holder.go +++ b/test/holder.go @@ -96,7 +96,7 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, if err != nil { panic(err) } - v, err := f.CreateViewIfNotExists(viewTimeKey{name: view}) + v, err := f.CreateViewIfNotExists(view) if err != nil { panic(err) } diff --git a/time.go b/time.go index 95218e221..def889304 100644 --- a/time.go +++ b/time.go @@ -79,33 +79,28 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -type viewTimeKey struct { - name string - quantum rune -} - // viewByTimeUnit returns the view name for time with a given quantum unit. -func viewByTimeUnit(name string, t time.Time, unit rune) viewTimeKey { +func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': - return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("2006")), quantum: unit} + return fmt.Sprintf("%s_%s", name, t.Format("2006")) case 'M': - return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("200601")), quantum: unit} + return fmt.Sprintf("%s_%s", name, t.Format("200601")) case 'D': - return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("20060102")), quantum: unit} + return fmt.Sprintf("%s_%s", name, t.Format("20060102")) case 'H': - return viewTimeKey{name: fmt.Sprintf("%s_%s", name, t.Format("2006010215")), quantum: unit} + return fmt.Sprintf("%s_%s", name, t.Format("2006010215")) default: - return viewTimeKey{} + return "" } } // viewsByTime returns a list of views for a given timestamp. -func viewsByTime(name string, t time.Time, q TimeQuantum) []viewTimeKey { - a := make([]viewTimeKey, 0, len(q)) +func viewsByTime(name string, t time.Time, q TimeQuantum) []string { + a := make([]string, 0, len(q)) for _, unit := range q { view := viewByTimeUnit(name, t, unit) - if view.name == "" { + if view == "" { continue } a = append(a, view) @@ -114,7 +109,7 @@ func viewsByTime(name string, t time.Time, q TimeQuantum) []viewTimeKey { } // viewsByTimeRange returns a list of views to traverse to query a time range. -func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []viewTimeKey { +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -123,7 +118,7 @@ func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []viewTi hasDay := q.HasDay() hasHour := q.HasHour() - var results []viewTimeKey + var results []string // Walk up from smallest units to largest units. if hasHour || hasDay || hasMonth { diff --git a/time_internal_test.go b/time_internal_test.go index 75a321abf..2920685fc 100644 --- a/time_internal_test.go +++ b/time_internal_test.go @@ -42,23 +42,23 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'Y'); s.name != "F_2000" { - t.Fatalf("unexpected name: %s", s.name) + if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { + t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'M'); s.name != "F_200001" { - t.Fatalf("unexpected name: %s", s.name) + if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { + t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'D'); s.name != "F_20000102" { - t.Fatalf("unexpected name: %s", s.name) + if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { + t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := viewByTimeUnit("F", ts, 'H'); s.name != "F_2000010203" { - t.Fatalf("unexpected name: %s", s.name) + if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + t.Fatalf("unexpected name: %s", s) } }) } diff --git a/view.go b/view.go index 7825a42e9..54f6b143b 100644 --- a/view.go +++ b/view.go @@ -59,7 +59,7 @@ type View struct { broadcaster Broadcaster stats StatsClient - viewType rune + viewType string RowAttrStore AttrStore Logger Logger } @@ -316,11 +316,11 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { } // clearBit clears a bit within the view. -func (v *View) clearBit(rowID, columnID uint64) (changed bool, remaining bool, err error) { +func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, found := v.fragments[slice] if !found { - return false, false, nil + return false, nil } return frag.clearBit(rowID, columnID) } From 25be5c0f2fe0ca18605d3ac1dd587773c5d57e48 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Jun 2018 10:38:37 -0500 Subject: [PATCH 04/10] Use channel to notify on server close instead of atomic.Value. Ensure CloseFunc() only called once. --- http/translator_test.go | 46 +++++++++++++++++++++++------------------ mock/mock.go | 9 +++++++- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/http/translator_test.go b/http/translator_test.go index fdfc93f3b..1eea725a2 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -5,7 +5,6 @@ import ( "io" "io/ioutil" gohttp "net/http" - "sync/atomic" "testing" "time" @@ -16,6 +15,17 @@ import ( "github.com/pilosa/pilosa/test" ) +func newMockReadCloser() *mock.ReadCloser { + return &mock.ReadCloser{ + ReadFunc: func(p []byte) (int, error) { + return 0, io.EOF + }, + CloseFunc: func() error { + return nil + }, + } +} + func TestTranslateStore_Reader(t *testing.T) { // Ensure client can connect and stream the translate store data. t.Run("OK", func(t *testing.T) { @@ -38,10 +48,9 @@ func TestTranslateStore_Reader(t *testing.T) { return 0, nil } } - closeInvoked := atomic.Value{} - closeInvoked.Store(false) + closeInvoked := make(chan struct{}) mrc.CloseFunc = func() error { - closeInvoked.Store(true) + close(closeInvoked) return nil } @@ -57,15 +66,7 @@ func TestTranslateStore_Reader(t *testing.T) { } return &mrc, nil } - mrc2 := mock.ReadCloser{ - ReadFunc: func(p []byte) (int, error) { - return 0, io.EOF - }, - CloseFunc: func() error { - return nil - }, - } - return &mrc2, nil + return newMockReadCloser(), nil } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) @@ -87,8 +88,11 @@ func TestTranslateStore_Reader(t *testing.T) { t.Fatal(err) } - if !closeInvoked.Load().(bool) { + select { + case <-time.NewTimer(time.Millisecond * 100).C: t.Fatal("expected server close") + case <-closeInvoked: + return } }) @@ -103,15 +107,15 @@ func TestTranslateStore_Reader(t *testing.T) { return 0, io.EOF } - closeInvoked := atomic.Value{} - closeInvoked.Store(false) + closeInvoked := make(chan struct{}) mrc.CloseFunc = func() error { - closeInvoked.Store(true) + close(closeInvoked) return nil } var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { return &mrc, nil } @@ -131,9 +135,11 @@ func TestTranslateStore_Reader(t *testing.T) { // Cancel the context and check if server is closed. cancel() - time.Sleep(100 * time.Millisecond) - if !closeInvoked.Load().(bool) { - t.Fatal("expected server-side close") + select { + case <-time.NewTimer(time.Millisecond * 100).C: + t.Fatal("expected server close") + case <-closeInvoked: + return } }) }) diff --git a/mock/mock.go b/mock/mock.go index 46469c2f9..97ebf8641 100644 --- a/mock/mock.go +++ b/mock/mock.go @@ -1,8 +1,11 @@ package mock +import "sync" + type ReadCloser struct { ReadFunc func(p []byte) (int, error) CloseFunc func() error + once sync.Once } func (rc *ReadCloser) Read(p []byte) (int, error) { @@ -10,5 +13,9 @@ func (rc *ReadCloser) Read(p []byte) (int, error) { } func (rc *ReadCloser) Close() error { - return rc.CloseFunc() + var err error = nil + rc.once.Do(func() { + err = rc.CloseFunc() + }) + return err } From fcecb871cfb20cc5184894181c6d9007952d49d3 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 28 Jun 2018 12:41:39 -0500 Subject: [PATCH 05/10] naming adjustments; code cleanup --- field.go | 37 +++++++++++++++++-------------------- view.go | 1 - 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/field.go b/field.go index 0c137dfdc..886f4a1df 100644 --- a/field.go +++ b/field.go @@ -39,8 +39,8 @@ const ( // Default ranked field cache defaultCacheSize = 50000 - BitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 - MaxInt = 1<<(BitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 + bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 + maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 ) // Field types. @@ -572,7 +572,7 @@ func (f *Field) CreateViewIfNotExists(name string) (*View, error) { &internal.CreateViewMessage{ Index: f.index, Field: f.name, - View: view.name, + View: name, }) if err != nil { return nil, errors.Wrap(err, "sending CreateView message") @@ -602,7 +602,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { return view, true, nil } -func (f *Field) newView(path string, name string) *View { +func (f *Field) newView(path, name string) *View { view := NewView(path, f.index, f.name, name, f.options.CacheSize) view.cacheType = f.options.CacheType view.Logger = f.Logger @@ -679,14 +679,14 @@ func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err err } // If a timestamp is specified then set bits across all views for the quantum. - for _, name := range viewsByTime(viewName, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(name) + for _, subname := range viewsByTime(viewName, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) if err != nil { - return changed, errors.Wrapf(err, "creating view %s", name) + return changed, errors.Wrapf(err, "creating view %s", subname) } if c, err := view.setBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", name) + return changed, errors.Wrapf(err, "setting on view %s", subname) } else if c { changed = true } @@ -715,29 +715,26 @@ func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error) { if len(f.views) == 1 { // assuming no time views return changed, nil } - lastLevel := 0 + lastViewNameSize := 0 level := 0 - skipBelow := MaxInt + skipAbove := maxInt for _, view := range f.allTimeViewsSortedByQuantum() { - if lastLevel < len(view.name) { + if lastViewNameSize < len(view.name) { level++ - } else if lastLevel > len(view.name) { + } else if lastViewNameSize > len(view.name) { level-- } - if level < skipBelow { + if level < skipAbove { if changed, err = view.clearBit(rowID, colID); err != nil { return changed, errors.Wrapf(err, "clearing on view %s", view.name) } if !changed { - if level < skipBelow { - skipBelow = level + 1 - } + skipAbove = level + 1 } else { - - skipBelow = MaxInt + skipAbove = maxInt } } - lastLevel = len(view.name) + lastViewNameSize = len(view.name) } return changed, nil @@ -771,7 +768,7 @@ func (f *Field) allTimeViewsSortedByQuantum() (me []*View) { day := month + 2 sort.Slice(me, func(i, j int) (lt bool) { var eq bool - // ensure all catA are grouped together: + // group by quantum from year to hour if lt, eq = groupCompare(me[i].name, me[j].name, year); eq { if lt, eq = groupCompare(me[i].name, me[j].name, month); eq { if lt, eq = groupCompare(me[i].name, me[j].name, day); eq { diff --git a/view.go b/view.go index 54f6b143b..1f8d10203 100644 --- a/view.go +++ b/view.go @@ -59,7 +59,6 @@ type View struct { broadcaster Broadcaster stats StatsClient - viewType string RowAttrStore AttrStore Logger Logger } From 824474160e28b29d0919f4d3882ac99f9879a3f9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 27 Jun 2018 17:11:55 -0500 Subject: [PATCH 06/10] Enhance test utilities (introduce Cluster type, improve naming) --- ctl/export_test.go | 2 +- ctl/import_test.go | 8 +- executor_test.go | 4 +- http/client_test.go | 6 +- http/translator_test.go | 6 +- server/cluster_test.go | 30 +++---- server/handler_test.go | 4 +- server/server_test.go | 8 +- server_test.go | 2 +- stats_test.go | 2 +- test/pilosa.go | 168 ++++++++++++++++++++++++---------------- test/pilosa_test.go | 4 +- 12 files changed, 141 insertions(+), 103 deletions(-) diff --git a/ctl/export_test.go b/ctl/export_test.go index 8960702f6..e959da6bc 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -44,7 +44,7 @@ func TestExportCommand_Validation(t *testing.T) { } func TestExportCommand_Run(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) diff --git a/ctl/import_test.go b/ctl/import_test.go index 522de0eba..32792c02d 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -61,7 +61,7 @@ func TestImportCommand_Run(t *testing.T) { t.Fatal(err) } - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.Server.URI.HostPort() cm.Index = "i" @@ -86,7 +86,7 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] cm.Host = cmd.Server.URI.HostPort() http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) @@ -102,7 +102,7 @@ func TestImportCommand_RunValue(t *testing.T) { } func TestImportCommand_InvalidFile(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -176,7 +176,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { } func TestImportCommand_BugOverwriteValue(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) diff --git a/executor_test.go b/executor_test.go index 022219235..250d8b13d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -267,7 +267,7 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_SetBit(t *testing.T) { t.Run("ID", func(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} hldr.SetBit("i", "f", 1, 0) @@ -312,7 +312,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) t.Run("Keys", func(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) diff --git a/http/client_test.go b/http/client_test.go index 3ca4cde50..7363879ce 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -219,7 +219,7 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -249,7 +249,7 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -321,7 +321,7 @@ func TestClient_ImportValue(t *testing.T) { // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/http/translator_test.go b/http/translator_test.go index 1eea725a2..a3a5e8603 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -70,7 +70,7 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0] + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] defer main.Close() @@ -121,7 +121,7 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0] + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] defer main.Close() defer close(done) @@ -152,7 +152,7 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0] + main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] defer main.Close() _, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0) diff --git a/server/cluster_test.go b/server/cluster_test.go index 825911ace..d8c2d7802 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -31,7 +31,7 @@ import ( // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { - ms := test.MustRunMainWithCluster(t, 2) + ms := test.MustRunCluster(t, 2) m0, m1 := ms[0], ms[1] defer m0.Close() defer m1.Close() @@ -116,7 +116,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Ensure that an empty node comes up in a NORMAL state. func TestClusterResize_EmptyNode(t *testing.T) { - m0 := test.MustRunMain() + m0 := test.MustRunCommand() defer m0.Close() if m0.API.State() != pilosa.ClusterStateNormal { @@ -126,7 +126,7 @@ func TestClusterResize_EmptyNode(t *testing.T) { // Ensure that a cluster of empty nodes comes up in a NORMAL state. func TestClusterResize_EmptyNodes(t *testing.T) { - clus := test.MustRunMainWithCluster(t, 2) + clus := test.MustRunCluster(t, 2) defer clus[0].Close() defer clus[1].Close() @@ -140,7 +140,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNode(t *testing.T) { t.Run("NoData", func(t *testing.T) { - clus := test.MustRunMainWithCluster(t, 2) + clus := test.MustRunCluster(t, 2) if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) @@ -150,7 +150,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.MustRunMainWithCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() seed := m0.GossipAddress() @@ -166,7 +166,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -183,7 +183,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("ContinuousSlices", func(t *testing.T) { // Configure node0 - m0 := test.MustRunMainWithCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() seed := m0.GossipAddress() @@ -207,7 +207,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -224,7 +224,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("SkippedSlice", func(t *testing.T) { // Configure node0 - m0 := test.MustRunMainWithCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() seed := m0.GossipAddress() @@ -248,7 +248,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -269,7 +269,7 @@ func TestClusterResize_AddNode(t *testing.T) { func TestCluster_GossipMembership(t *testing.T) { t.Run("Node0Down", func(t *testing.T) { // Configure node0 - m0 := test.MustRunMainWithCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() seed := m0.GossipAddress() @@ -277,7 +277,7 @@ func TestCluster_GossipMembership(t *testing.T) { var eg errgroup.Group // Configure node1 - m1 := test.NewMainWithCluster(false) + m1 := test.NewCommandNode(false) defer m1.Close() eg.Go(func() error { m1.Config.Gossip.Port = "0" @@ -291,7 +291,7 @@ func TestCluster_GossipMembership(t *testing.T) { }) // Configure node1 - m2 := test.NewMainWithCluster(false) + m2 := test.NewCommandNode(false) defer m2.Close() eg.Go(func() error { m2.Config.Gossip.Port = "0" @@ -324,7 +324,7 @@ func TestCluster_GossipMembership(t *testing.T) { } func TestClusterResize_RemoveNode(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3) + cluster := test.MustRunCluster(t, 3) m0 := cluster[0] m1 := cluster[1] @@ -410,7 +410,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // checkClusterState polls a given cluster for its state until it // receives a matching state. It polls up to n times before returning. -func checkClusterState(m *test.Main, state string, n int) bool { +func checkClusterState(m *test.Command, state string, n int) bool { for i := 0; i < n; i++ { if m.API.State() == state { return true diff --git a/server/handler_test.go b/server/handler_test.go index cc21b8825..2981f375c 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -37,7 +37,7 @@ import ( // Ensure the handler returns "not found" for invalid paths. func TestHandler_Endpoints(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -566,7 +566,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) } - clus := test.MustRunMainWithCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) + clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) w = httptest.NewRecorder() h := clus[0].Handler.(*http.Handler).Handler h.ServeHTTP(w, req) diff --git a/server/server_test.go b/server/server_test.go index b3067db87..8b1e29acc 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -40,7 +40,7 @@ func TestMain_Set_Quick(t *testing.T) { } if err := quick.Check(func(cmds []SetCommand) bool { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() // Create client. @@ -116,7 +116,7 @@ func TestMain_Set_Quick(t *testing.T) { // Ensure program can set row attributes and retrieve them. func TestMain_SetRowAttrs(t *testing.T) { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() // Create fields. @@ -193,7 +193,7 @@ func TestMain_SetRowAttrs(t *testing.T) { // Ensure program can set column attributes and retrieve them. func TestMain_SetColumnAttrs(t *testing.T) { - m := test.MustRunMain() + m := test.MustRunCommand() defer m.Close() // Create fields. @@ -264,7 +264,7 @@ func tempMkdir(t *testing.T) string { func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 - cluster := test.MustRunMainWithCluster(t, clusterSize) + cluster := test.MustRunCluster(t, clusterSize) // Create the schema. client0 := cluster[0].Client() diff --git a/server_test.go b/server_test.go index 711572e25..04f8a6171 100644 --- a/server_test.go +++ b/server_test.go @@ -28,7 +28,7 @@ import ( // pilosa.Server was not having its remoteClient field set by an option and so // it was using a nil client in monitorAntiEntropy. func TestMonitorAntiEntropy(t *testing.T) { - cluster := test.MustRunMainWithCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)}) + cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)}) client := cluster[1].Client() err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) if err != nil { diff --git a/stats_test.go b/stats_test.go index 271057d9b..932672e3a 100644 --- a/stats_test.go +++ b/stats_test.go @@ -209,7 +209,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { } func TestStatsCount_APICalls(t *testing.T) { - cmd := test.MustRunMainWithCluster(t, 1)[0] + cmd := test.MustRunCluster(t, 1)[0] h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/test/pilosa.go b/test/pilosa.go index e201de283..f68082985 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -32,8 +32,8 @@ import ( ) //////////////////////////////////////////////////////////////////////////////////// -// Main represents a test wrapper for server.Command. -type Main struct { +// Command represents a test wrapper for server.Command. +type Command struct { *server.Command commandOptions []server.CommandOption @@ -57,21 +57,14 @@ func OptAllowedOrigins(origins []string) server.CommandOption { } } -// GossipAddress returns the address on which gossip is listening after a Main -// has been setup. Useful to pass as a seed to other nodes when creating and -// testing clusters. -func (m *Main) GossipAddress() string { - return m.GossipTransport().URI.String() -} - -// NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain(opts ...server.CommandOption) *Main { +// NewCommand returns a new instance of Main with a temporary data directory and random port. +func NewCommand(opts ...server.CommandOption) *Command { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) } - m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts} + m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts} m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true @@ -92,58 +85,17 @@ func NewMain(opts ...server.CommandOption) *Main { return m } -// NewMainWithCluster returns a new instance of Main with clustering enabled. -func NewMainWithCluster(isCoordinator bool, opts ...server.CommandOption) *Main { - m := NewMain(opts...) +// NewCommandNode returns a new instance of Command with clustering enabled. +func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { + m := NewCommand(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m } -// MustRunMainWithCluster ruturns a running array of *Main where -// all nodes are joined via memberlist (i.e. clustering enabled). -func MustRunMainWithCluster(t *testing.T, size int, opts ...[]server.CommandOption) []*Main { - ma, err := runMainWithCluster(size, opts...) - if err != nil { - t.Fatalf("new main array with cluster: %v", err) - } - return ma -} - -// runMainWithCluster runs an array of *Main where all nodes are -// joined via memberlist (i.e. clustering enabled). -func runMainWithCluster(size int, opts ...[]server.CommandOption) ([]*Main, error) { - if size == 0 { - return nil, errors.New("cluster must contain at least one node") - } - if len(opts) != size && len(opts) != 0 && len(opts) != 1 { - return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes") - } - - mains := make([]*Main, size) - var gossipSeeds = make([]string, size) - for i := 0; i < size; i++ { - var commandOpts []server.CommandOption - if len(opts) > 0 { - commandOpts = opts[i%len(opts)] - } - m := NewMainWithCluster(i == 0, commandOpts...) - m.Config.Gossip.Port = "0" - m.Config.Gossip.Seeds = gossipSeeds[:i] - - if err := m.Start(); err != nil { - return nil, errors.Wrapf(err, "Starting server %d", i) - } - gossipSeeds[i] = m.GossipTransport().URI.String() - mains[i] = m - } - - return mains, nil -} - -// MustRunMain returns a new, running Main. Panic on error. -func MustRunMain() *Main { - m := NewMain() +// MustRunCommand returns a new, running Main. Panic on error. +func MustRunCommand() *Command { + m := NewCommand() m.Config.Metric.Diagnostics = false // Disable diagnostics. if err := m.Start(); err != nil { panic(err) @@ -151,14 +103,21 @@ func MustRunMain() *Main { return m } +// GossipAddress returns the address on which gossip is listening after a Main +// has been setup. Useful to pass as a seed to other nodes when creating and +// testing clusters. +func (m *Command) GossipAddress() string { + return m.GossipTransport().URI.String() +} + // Close closes the program and removes the underlying data directory. -func (m *Main) Close() error { +func (m *Command) Close() error { defer os.RemoveAll(m.Config.DataDir) return m.Command.Close() } // Reopen closes the program and reopens it. -func (m *Main) Reopen() error { +func (m *Command) Reopen() error { if err := m.Command.Close(); err != nil { return err } @@ -180,10 +139,10 @@ func (m *Main) Reopen() error { } // URL returns the base URL string for accessing the running program. -func (m *Main) URL() string { return m.Server.URI.String() } +func (m *Command) URL() string { return m.Server.URI.String() } // Client returns a client to connect to the program. -func (m *Main) Client() *http.InternalClient { +func (m *Command) Client() *http.InternalClient { client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) @@ -192,7 +151,7 @@ func (m *Main) Client() *http.InternalClient { } // Query executes a query against the program through the HTTP API. -func (m *Main) Query(index, rawQuery, query string) (string, error) { +func (m *Command) Query(index, rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) @@ -200,7 +159,7 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } -func (m *Main) RecalculateCaches() error { +func (m *Command) RecalculateCaches() error { resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") if resp.StatusCode != 204 { return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) @@ -208,6 +167,85 @@ func (m *Main) RecalculateCaches() error { return nil } +// Cluster represents a Pilosa cluster (multiple Command instances) +type Cluster []*Command + +// Start runs a Cluster +func (c Cluster) Start() error { + var gossipSeeds = make([]string, len(c)) + for i, cc := range c { + cc.Config.Gossip.Port = "0" + cc.Config.Gossip.Seeds = gossipSeeds[:i] + if err := cc.Start(); err != nil { + return errors.Wrapf(err, "starting server %d", i) + } + gossipSeeds[i] = cc.GossipTransport().URI.String() + } + return nil +} + +// Stop stops a Cluster +func (c Cluster) Close() error { + for i, cc := range c { + if err := cc.Close(); err != nil { + return errors.Wrapf(err, "stopping server %d", i) + } + } + return nil +} + +// MustNewCluster creates a new cluster +func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { + c, err := newCluster(size, opts...) + if err != nil { + t.Fatalf("new cluster: %v", err) + } + return c +} + +// newCluster creates a new cluster +func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { + if size == 0 { + return nil, errors.New("cluster must contain at least one node") + } + if len(opts) != size && len(opts) != 0 && len(opts) != 1 { + return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes") + } + + cluster := make(Cluster, size) + for i := 0; i < size; i++ { + var commandOpts []server.CommandOption + if len(opts) > 0 { + commandOpts = opts[i%len(opts)] + } + m := NewCommandNode(i == 0, commandOpts...) + cluster[i] = m + } + + return cluster, nil +} + +// runCluster creates and starts a new cluster +func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { + cluster, err := newCluster(size, opts...) + if err != nil { + return nil, errors.Wrap(err, "new cluster") + } + if err = cluster.Start(); err != nil { + return nil, errors.Wrap(err, "starting cluster") + } + return cluster, nil +} + +// MustRunCluster creates and starts a new cluster +func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { + c, err := runCluster(size, opts...) + if err != nil { + t.Fatalf("run cluster: %v", err) + } + return c +} + //////////////////////////////////////////////////////////////////////////////////// // MustDo executes http.Do() with an http.NewRequest(). Panic on error. diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 2ba7504c8..25bb808df 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -27,7 +27,7 @@ import ( func TestNewCluster(t *testing.T) { numNodes := 3 - cluster := test.MustRunMainWithCluster(t, numNodes) + cluster := test.MustRunCluster(t, numNodes) coordinator := getCoordinator(cluster[0]) for i := 1; i < numNodes; i++ { @@ -78,7 +78,7 @@ func TestNewCluster(t *testing.T) { } } -func getCoordinator(m *test.Main) string { +func getCoordinator(m *test.Command) string { hosts := m.API.Hosts(context.Background()) for _, host := range hosts { if host.IsCoordinator { From 4f3cf9af30a89e7b41a73886bb1c2d5911af61a0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Jun 2018 13:20:57 -0500 Subject: [PATCH 07/10] Use GossipAddress() helper --- test/pilosa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pilosa.go b/test/pilosa.go index f68082985..d2f2f6106 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -179,7 +179,7 @@ func (c Cluster) Start() error { if err := cc.Start(); err != nil { return errors.Wrapf(err, "starting server %d", i) } - gossipSeeds[i] = cc.GossipTransport().URI.String() + gossipSeeds[i] = cc.GossipAddress() } return nil } From 5dd7a9556a09e1d6c80172e996e75bf3660fa25e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 28 Jun 2018 14:06:37 -0500 Subject: [PATCH 08/10] rename slice to shard --- Gopkg.lock | 2 +- NOTES | 12 +- api.go | 82 ++++---- apimethod_string.go | 6 +- broadcast.go | 10 +- broadcast_test.go | 4 +- client.go | 32 +-- cluster.go | 64 +++--- cluster_internal_test.go | 24 +-- cmd/import.go | 2 +- ctl/export.go | 14 +- ctl/import.go | 22 +- diagnostics.go | 6 +- executor.go | 298 ++++++++++++++-------------- executor_test.go | 204 +++++++++---------- field.go | 20 +- fragment.go | 68 +++---- fragment_internal_test.go | 12 +- handler.go | 6 +- holder.go | 42 ++-- holder_test.go | 44 ++-- http/client.go | 110 +++++----- http/client_test.go | 56 +++--- http/handler.go | 66 +++--- index.go | 26 +-- index_test.go | 4 +- internal/private.pb.go | 408 ++++++++++++++++++++------------------ internal/private.proto | 12 +- internal/public.pb.go | 187 +++++++++-------- internal/public.proto | 6 +- iterator.go | 4 +- pilosa.go | 2 +- roaring/roaring_test.go | 24 +-- row.go | 44 ++-- row_test.go | 20 +- server.go | 22 +- server/cluster_test.go | 24 +-- server/handler_test.go | 50 ++--- stats_test.go | 20 +- test/fragment.go | 4 +- test/handler.go | 6 +- test/holder.go | 4 +- utils_internal_test.go | 14 +- view.go | 100 +++++----- view_internal_test.go | 12 +- 45 files changed, 1120 insertions(+), 1079 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d3a12ef8a..33187bfe2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -310,6 +310,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" + inputs-digest = "da6d02118ca77527c4ff00e9522880032fc052fb39bc8efe6c76602857c8c84e" solver-name = "gps-cdcl" solver-version = 1 diff --git a/NOTES b/NOTES index 6b8e088ea..8da55fe11 100644 --- a/NOTES +++ b/NOTES @@ -14,13 +14,13 @@ │0000000000000000000000000000000000000000│ │────────────────────────────────────────┤ F ▶│0000000000000000000000000000000000000000│ - r ││0000000000000000000000000000000000000000│ - a ││0000000000000000000000000000000000000000│ - m ││0000000000000000000000000000000000000000│ - e ▶│0000000000000000000000000000000000000000│ + i ││0000000000000000000000000000000000000000│ + e ││0000000000000000000000000000000000000000│ + l ││0000000000000000000000000000000000000000│ + d ▶│0000000000000000000000000000000000000000│ └────────────────────────────────────────┘ ▲───────────▲ - Slice + Shard -Fragment=intersection of frame & slice +Fragment=intersection of field & shard diff --git a/api.go b/api.go index 5b730b017..296304127 100644 --- a/api.go +++ b/api.go @@ -115,7 +115,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeRowAttrs: req.ExcludeRowAttrs, ExcludeColumns: req.ExcludeColumns, } - results, err := api.server.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { return resp, errors.Wrap(err, "executing") } @@ -316,21 +316,21 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str return nil } -// ExportCSV encodes the fragment designated by the index,field,slice as +// ExportCSV encodes the fragment designated by the index,field,shard as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } - // Validate that this handler owns the slice. - if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { - api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return ErrClusterDoesNotOwnSlice + // Validate that this handler owns the shard. + if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + return ErrClusterDoesNotOwnShard } // Find the fragment. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return ErrFragmentNotFound } @@ -354,25 +354,25 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin return nil } -// SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { - if err := api.validate(apiSliceNodes); err != nil { +// ShardNodes returns the node and all replicas which should contain a shard's data. +func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) { + if err := api.validate(apiShardNodes); err != nil { return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.sliceNodes(indexName, slice), nil + return api.Cluster.shardNodes(indexName, shard), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return nil, ErrFragmentNotFound } @@ -382,7 +382,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName // UnmarshalFragment creates a new fragment (if necessary) and reads data from a // Reader which was previously written by MarshalFragment to populate the // fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error { +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, shard uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { return errors.Wrap(err, "validating api method") } @@ -400,7 +400,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldNa } // Retrieve fragment from field. - frag, err := view.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -430,7 +430,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice) + f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Shard) if f == nil { return nil, ErrFragmentNotFound } @@ -448,13 +448,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, shard uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, shard) if f == nil { return nil, ErrFragmentNotFound } @@ -552,7 +552,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ignore this error because views do not exist on all nodes due to slice distribution. + // Ignore this error because views do not exist on all nodes due to shard distribution. if err != ErrInvalidView { return errors.Wrap(err, "deleting view") } @@ -641,13 +641,13 @@ func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName s return attrs, nil } -// Import bulk imports data into a particular index,field,slice. +// Import bulk imports data into a particular index,field,shard. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } - _, field, err := api.indexField(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting field") } @@ -665,7 +665,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -676,21 +676,21 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return errors.Wrap(err, "validating api method") } - _, field, err := api.indexField(req.Index, req.Field, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Shard) if err != nil { return errors.Wrap(err, "getting field") } // Import into fragment. err = field.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.server.logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) + api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } -// MaxSlices returns the maximum slice number for each index in a map. -func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { - return api.Holder.MaxSlices() +// MaxShards returns the maximum shard number for each index in a map. +func (api *API) MaxShards(ctx context.Context) map[string]uint64 { + return api.Holder.MaxShards() } // StatsWithTags returns an instance of whatever implementation of StatsClient @@ -711,25 +711,25 @@ func (api *API) LongQueryTime() time.Duration { return api.Cluster.longQueryTime } -func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { - // Validate that this handler owns the slice. - if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { - api.server.logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) - return nil, nil, ErrClusterDoesNotOwnSlice +func (api *API) indexField(indexName string, fieldName string, shard uint64) (*Index, *Field, error) { + // Validate that this handler owns the shard. + if !api.Cluster.ownsShard(api.LocalID(), indexName, shard) { + api.server.logger.Printf("node %s does not own shard %d of index %s", api.LocalID(), shard, indexName) + return nil, nil, ErrClusterDoesNotOwnShard } // Find the Index. - api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, slice) + api.server.logger.Printf("importing: %v %v %v", indexName, fieldName, shard) index := api.Holder.Index(indexName) if index == nil { - api.server.logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error()) + api.server.logger.Printf("fragment error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } // Retrieve field. field := index.Field(fieldName) if field == nil { - api.server.logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error()) + api.server.logger.Printf("field error: index=%s, field=%s, shard=%d, err=%s", indexName, fieldName, shard, ErrFieldNotFound.Error()) return nil, nil, ErrFieldNotFound } return index, field, nil @@ -851,12 +851,12 @@ func (api *API) Version() string { // Info returns information about this server instance func (api *API) Info() ServerInfo { return ServerInfo{ - SliceWidth: SliceWidth, + ShardWidth: ShardWidth, } } type ServerInfo struct { - SliceWidth uint64 `json:"sliceWidth"` + ShardWidth uint64 `json:"shardWidth"` } type apiMethod int @@ -881,14 +881,14 @@ const ( //apiLocalID // not implemented //apiLongQueryTime // not implemented apiMarshalFragment - //apiMaxSlices // not implemented + //apiMaxShards // not implemented apiQuery apiRecalculateCaches apiRemoveNode apiResizeAbort //apiSchema // not implemented apiSetCoordinator - apiSliceNodes + apiShardNodes //apiState // not implemented //apiStatsWithTags // not implemented apiUnmarshalFragment @@ -923,7 +923,7 @@ var methodsNormal = map[apiMethod]struct{}{ apiQuery: struct{}{}, apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, - apiSliceNodes: struct{}{}, + apiShardNodes: struct{}{}, apiUnmarshalFragment: struct{}{}, apiViews: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index a2b934e0f..7004e7258 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -2,15 +2,15 @@ package pilosa -import "fmt" +import "strconv" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiUnmarshalFragmentapiViews" var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { - return fmt.Sprintf("apiMethod(%d)", i) + return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")" } return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] } diff --git a/broadcast.go b/broadcast.go index b7b13fe04..12f2b0bfe 100644 --- a/broadcast.go +++ b/broadcast.go @@ -62,7 +62,7 @@ type BroadcastHandler interface { // Broadcast message types. const ( - messageTypeCreateSlice = iota + messageTypeCreateShard = iota messageTypeCreateIndex messageTypeDeleteIndex messageTypeCreateField @@ -83,8 +83,8 @@ const ( func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { - case *internal.CreateSliceMessage: - typ = messageTypeCreateSlice + case *internal.CreateShardMessage: + typ = messageTypeCreateShard case *internal.CreateIndexMessage: typ = messageTypeCreateIndex case *internal.DeleteIndexMessage: @@ -129,8 +129,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { var m proto.Message switch typ { - case messageTypeCreateSlice: - m = &internal.CreateSliceMessage{} + case messageTypeCreateShard: + m = &internal.CreateShardMessage{} case messageTypeCreateIndex: m = &internal.CreateIndexMessage{} case messageTypeDeleteIndex: diff --git a/broadcast_test.go b/broadcast_test.go index 23bd962a7..5c97df0d0 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -30,9 +30,9 @@ import ( // Ensure a message can be marshaled and unmarshaled. func TestMessage_Marshal(t *testing.T) { - testMessageMarshal(t, &internal.CreateSliceMessage{ + testMessageMarshal(t, &internal.CreateShardMessage{ Index: "i", - Slice: 8, + Shard: 8, }) testMessageMarshal(t, &internal.DeleteIndexMessage{ diff --git a/client.go b/client.go index 292a71ef6..6a0b48d71 100644 --- a/client.go +++ b/client.go @@ -32,25 +32,25 @@ type FieldValue struct { // While I understand that putting the entire Client behind an interface might require this many methods, // I don't want to let it go unquestioned. type InternalClient interface { - MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) + MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error - FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) + FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error + Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error - ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error + ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error - FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) + FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error - RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) + RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) } //=============== @@ -81,7 +81,7 @@ func NewNopInternalClient() *NopInternalClient { var _ InternalClient = NewNopInternalClient() -func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +func (n *NopInternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { return nil, nil } func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { @@ -90,7 +90,7 @@ func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { return nil } -func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { +func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) { return nil, nil } func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { @@ -99,7 +99,7 @@ func (n *NopInternalClient) Query(ctx context.Context, index string, queryReques func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { return nil, nil } -func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { +func (n *NopInternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []Bit) error { return nil } func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { @@ -111,19 +111,19 @@ func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, option func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } -func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { +func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } -func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { +func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { return nil } func (n *NopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } -func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) { +func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, shard uint64) ([]FragmentBlock, error) { return nil, nil } -func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { +func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { return nil, nil, nil } func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { @@ -135,6 +135,6 @@ func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fi func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { return nil } -func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { +func (n *NopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index de728472a..21c91e528 100644 --- a/cluster.go +++ b/cluster.go @@ -576,7 +576,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool { type frag struct { field string view string - slice uint64 + shard uint64 } func fragsDiff(a, b []frag) []frag { @@ -623,15 +623,15 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { } } - return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews) + return c.fragCombos(idx.Name(), idx.MaxShard(), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index -// by creating every combination of field/view specified in `fieldViews` up to maxSlice. -func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { +// by creating every combination of field/view specified in `fieldViews` up to maxShard. +func (c *Cluster) fragCombos(idx string, maxShard uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) - for i := uint64(0); i <= maxSlice; i++ { - nodes := c.sliceNodes(idx, i) + for i := uint64(0); i <= maxShard; i++ { + nodes := c.shardNodes(idx, i) for _, n := range nodes { // for each field/view combination: for field, views := range fieldViews { @@ -762,7 +762,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R Index: idx.Name(), Field: frag.field, View: frag.view, - Slice: frag.slice, + Shard: frag.shard, } m[nodeID] = append(m[nodeID], src) @@ -772,10 +772,10 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// partition returns the partition that a slice belongs to. -func (c *Cluster) partition(index string, slice uint64) int { +// partition returns the partition that a shard belongs to. +func (c *Cluster) partition(index string, shard uint64) int { var buf [8]byte - binary.BigEndian.PutUint64(buf[:], slice) + binary.BigEndian.PutUint64(buf[:], shard) // Hash the bytes and mod by partition count. h := fnv.New64a() @@ -784,14 +784,14 @@ func (c *Cluster) partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.partitionN)) } -// sliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) sliceNodes(index string, slice uint64) []*Node { - return c.partitionNodes(c.partition(index, slice)) +// shardNodes returns a list of nodes that own a fragment. +func (c *Cluster) shardNodes(index string, shard uint64) []*Node { + return c.partitionNodes(c.partition(index, shard)) } -// ownsSlice returns true if a host owns a fragment. -func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID) +// ownsShard returns true if a host owns a fragment. +func (c *Cluster) ownsShard(nodeID string, index string, shard uint64) bool { + return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) } // partitionNodes returns a list of nodes that own a partition. @@ -817,20 +817,20 @@ func (c *Cluster) partitionNodes(partitionID int) []*Node { return nodes } -// containsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { +// containsShards is like OwnsShards, but it includes replicas. +func (c *Cluster) containsShards(index string, maxShard uint64, node *Node) []uint64 { + var shards []uint64 + for i := uint64(0); i <= maxShard; i++ { p := c.partition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { - slices = append(slices, i) + shards = append(shards, i) } } } - return slices + return shards } // Hasher represents an interface to hash integers into buckets. @@ -1211,7 +1211,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err // Request each source file in ResizeSources. for _, src := range instr.Sources { - c.logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) + c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) srcURI := decodeURI(src.Node.URI) @@ -1228,27 +1228,27 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err } // Create the local fragment. - frag, err := v.CreateFragmentIfNotExists(src.Slice) + frag, err := v.CreateFragmentIfNotExists(src.Shard) if err != nil { return errors.Wrap(err, "creating fragment") } - // Stream slice from remote node. - c.logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) + // Stream shard from remote node. + c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) + rd, err := c.InternalClient.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found - // on the remote node. This occurs when a slice has been skipped and + // on the remote node. This occurs when a shard has been skipped and // therefore doesn't contain data. The coordinator correctly determined - // the resize instruction to retrieve the slice, but it doesn't have data. + // the resize instruction to retrieve the shard, but it doesn't have data. // TODO: figure out a way to distinguish from "fragment not found" errors // which are true errors and which simply mean the fragment doesn't have data. if err == ErrFragmentNotFound { return nil } - return errors.Wrap(err, "retrieving slice") + return errors.Wrap(err, "retrieving shard") } else if rd == nil { - return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) + return fmt.Errorf("shard %v doesn't exist on host: %s", src.Shard, src.Node.URI) } // Write to local field and always close reader. @@ -1257,7 +1257,7 @@ func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) err _, err := frag.ReadFrom(rd) return err }(); err != nil { - return errors.Wrap(err, "copying remote slice") + return errors.Wrap(err, "copying remote shard") } } return nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index b53180b39..8e4621653 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -49,13 +49,13 @@ func TestFragCombos(t *testing.T) { tests := []struct { idx string - maxSlice uint64 + maxShard uint64 fieldViews viewsByField expected fragsByHost }{ { idx: "i", - maxSlice: uint64(2), + maxShard: uint64(2), fieldViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, @@ -64,7 +64,7 @@ func TestFragCombos(t *testing.T) { }, { idx: "foo", - maxSlice: uint64(3), + maxShard: uint64(3), fieldViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, @@ -74,7 +74,7 @@ func TestFragCombos(t *testing.T) { } for _, test := range tests { - actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews) + actual := c.fragCombos(test.idx, test.maxShard, test.fieldViews) if !reflect.DeepEqual(actual, test.expected) { t.Errorf("expected: %v, but got: %v", test.expected, actual) } @@ -339,13 +339,13 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { + if err := quick.Check(func(index string, shard uint64, partitionN int) bool { c := NewCluster() c.partitionN = partitionN - partitionID := c.partition(index, slice) + partitionID := c.partition(index, shard) if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) + t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) } return true @@ -380,14 +380,14 @@ func TestHasher(t *testing.T) { } } -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { +// Ensure ContainsShards can find the actual shard list for node and index. +func TestCluster_ContainsShards(t *testing.T) { c := NewTestCluster(5) c.ReplicaN = 3 - slices := c.containsSlices("test", 10, c.Nodes[2]) + shards := c.containsShards("test", 10, c.Nodes[2]) - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) + if !reflect.DeepEqual(shards, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected shars for node's index: %v", shards) } } diff --git a/cmd/import.go b/cmd/import.go index 5a27b4055..4d565adb7 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Use: "import", Short: "Bulk load data into pilosa.", Long: `Bulk imports one or more CSV files to a host's index and field. The data -of the CSV file are grouped by slice for the most efficient import. +of the CSV file are grouped by shard for the most efficient import. The format of the CSV file is: diff --git a/ctl/export.go b/ctl/export.go index 090904807..5bfbd529f 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -80,16 +80,16 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return errors.Wrap(err, "creating client") } - // Determine slice count. - maxSlices, err := client.MaxSliceByIndex(ctx) + // Determine shard count. + maxShards, err := client.MaxShardByIndex(ctx) if err != nil { - return errors.Wrap(err, "getting slice count") + return errors.Wrap(err, "getting shard count") } - // Export each slice. - for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { - logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil { + // Export each shard. + for shard := uint64(0); shard <= maxShards[cmd.Index]; shard++ { + logger.Printf("exporting shard: %d", shard) + if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, shard, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/import.go b/ctl/import.go index 05659fa27..370425115 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -118,7 +118,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } - // Import each path and import by slice. + // Import each path and import by shard. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) if err := cmd.importPath(ctx, fieldType, path); err != nil { @@ -243,18 +243,18 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group bits by slice. + // Group bits by shard. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := http.Bits(bits).GroupBySlice() + bitsByShard := http.Bits(bits).GroupByShard() // Parse path into bits. - for slice, chunk := range bitsBySlice { + for shard, chunk := range bitsByShard { if cmd.Sort { sort.Sort(http.BitsByPos(chunk)) } - logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(chunk)) + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, shard, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -437,18 +437,18 @@ func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group vals by slice. + // Group vals by shard. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := http.FieldValues(vals).GroupBySlice() + valsByShard := http.FieldValues(vals).GroupByShard() // Parse path into FieldValues. - for slice, vals := range valsBySlice { + for shard, vals := range valsByShard { if cmd.Sort { sort.Sort(http.FieldValues(vals)) } - logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil { + logger.Printf("importing shard: %d, n=%d", shard, len(vals)) + if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, shard, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/diagnostics.go b/diagnostics.go index b36335b38..74e7eebcf 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -216,14 +216,14 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { // EnrichWithSchemaProperties adds schema info to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { - var numSlices uint64 + var numShards uint64 numFields := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false for _, index := range d.server.holder.Indexes() { - numSlices += index.MaxSlice() + 1 + numShards += index.MaxShard() + 1 numIndexes += 1 for _, field := range index.Fields() { numFields += 1 @@ -238,7 +238,7 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { d.Set("NumIndexes", numIndexes) d.Set("NumFields", numFields) - d.Set("NumSlices", numSlices) + d.Set("NumShards", numShards) d.Set("BSIFieldCount", bsiFieldCount) d.Set("TimeQuantumEnabled", timeQuantumEnabled) } diff --git a/executor.go b/executor.go index 610696022..53d0de4a9 100644 --- a/executor.go +++ b/executor.go @@ -37,7 +37,7 @@ const ( rowLabel = "row" ) -// Executor recursively executes calls in a PQL query across all slices. +// Executor recursively executes calls in a PQL query across all shards. type Executor struct { Holder *Holder @@ -80,7 +80,7 @@ func NewExecutor(opts ...ExecutorOption) *Executor { } // Execute executes a PQL query. -func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { +func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { // Verify that an index is set. if index == "" { return nil, ErrIndexRequired @@ -108,7 +108,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic } } - results, err := e.execute(ctx, index, q, slices, opt) + results, err := e.execute(ctx, index, q, shards, opt) if err != nil { return nil, err } @@ -123,24 +123,24 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return results, nil } -func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { - // Don't bother calculating slices for query types that don't require it. - needsSlices := needsSlices(q.Calls) +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) ([]interface{}, error) { + // Don't bother calculating shards for query types that don't require it. + needsShards := needsShards(q.Calls) - // If slices are specified, then use that value for slices. If slices aren't + // If shards are specified, then use that value for shards. If shards aren't // specified, then include all of them. - if len(slices) == 0 && needsSlices { - // Round up the number of slices. + if len(shards) == 0 && needsShards { + // Round up the number of shards. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - maxSlice := idx.MaxSlice() + maxShard := idx.MaxShard() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) + // Generate a slice of all shards. + shards = make([]uint64, maxShard+1) + for i := range shards { + shards[i] = uint64(i) } } @@ -152,7 +152,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - v, err := e.executeCall(ctx, index, call, slices, opt) + v, err := e.executeCall(ctx, index, call, shards, opt) if err != nil { return nil, err } @@ -162,7 +162,7 @@ func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slic } // executeCall executes a call. -func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { +func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (interface{}, error) { if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } @@ -171,18 +171,18 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s switch c.Name { case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeSum(ctx, index, c, slices, opt) + return e.executeSum(ctx, index, c, shards, opt) case "Min": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeMin(ctx, index, c, slices, opt) + return e.executeMin(ctx, index, c, shards, opt) case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeMax(ctx, index, c, slices, opt) + return e.executeMax(ctx, index, c, shards, opt) case "Clear": return e.executeClearBit(ctx, index, c, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeCount(ctx, index, c, slices, opt) + return e.executeCount(ctx, index, c, shards, opt) case "Set": return e.executeSetBit(ctx, index, c, opt) case "SetValue": @@ -193,10 +193,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return nil, e.executeSetColumnAttrs(ctx, index, c, opt) case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeTopN(ctx, index, c, slices, opt) + return e.executeTopN(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeBitmapCall(ctx, index, c, slices, opt) + return e.executeBitmapCall(ctx, index, c, shards, opt) } } @@ -220,7 +220,7 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Sum(): field required") } @@ -230,8 +230,8 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeSumCountSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSumCountShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -240,7 +240,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other.Add(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -253,7 +253,7 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl } // executeMin executes a Min() call. -func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Min(): field required") } @@ -263,8 +263,8 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeMinSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMinShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -273,7 +273,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl return other.Smaller(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -286,7 +286,7 @@ func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, sl } // executeMax executes a Max() call. -func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { +func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Max(): field required") } @@ -296,8 +296,8 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeMaxSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeMaxShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -306,7 +306,7 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl return other.Larger(v.(ValCount)) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return ValCount{}, err } @@ -319,10 +319,10 @@ func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, sl } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Row, error) { +func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeBitmapCallSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeBitmapCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -335,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return other } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -384,31 +384,31 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return row, nil } -// executeBitmapCallSlice executes a bitmap call for a single slice. -func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBitmapCallShard executes a bitmap call for a single shard. +func (e *Executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { switch c.Name { case "Row": - return e.executeBitmapSlice(ctx, index, c, slice) + return e.executeBitmapShard(ctx, index, c, shard) case "Difference": - return e.executeDifferenceSlice(ctx, index, c, slice) + return e.executeDifferenceShard(ctx, index, c, shard) case "Intersect": - return e.executeIntersectSlice(ctx, index, c, slice) + return e.executeIntersectShard(ctx, index, c, shard) case "Range": - return e.executeRangeSlice(ctx, index, c, slice) + return e.executeRangeShard(ctx, index, c, shard) case "Union": - return e.executeUnionSlice(ctx, index, c, slice) + return e.executeUnionShard(ctx, index, c, shard) case "Xor": - return e.executeXorSlice(ctx, index, c, slice) + return e.executeXorShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } } -// executeSumCountSlice calculates the sum and count for bsiGroups on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeSumCountShard calculates the sum and count for bsiGroups on a shard. +func (e *Executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, errors.Wrap(err, "executing bitmap call") } @@ -427,7 +427,7 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -442,11 +442,11 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq }, nil } -// executeMinSlice calculates the min for bsiGroups on a slice. -func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMinShard calculates the min for bsiGroups on a shard. +func (e *Executor) executeMinShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -465,7 +465,7 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -480,11 +480,11 @@ func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Cal }, nil } -// executeMaxSlice calculates the max for bsiGroups on a slice. -func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMaxShard calculates the max for bsiGroups on a shard. +func (e *Executor) executeMaxShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return ValCount{}, err } @@ -503,7 +503,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal return ValCount{}, nil } - fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if fragment == nil { return ValCount{}, nil } @@ -521,7 +521,7 @@ func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Cal // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) { idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -532,7 +532,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s } // Execute original query. - pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) + pairs, err := e.executeTopNShards(ctx, index, c, shards, opt) if err != nil { return nil, errors.Wrap(err, "finding top results") } @@ -549,7 +549,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) + trimmedList, err := e.executeTopNShards(ctx, index, other, shards, opt) if err != nil { return nil, errors.Wrap(err, "retrieving full counts") } @@ -560,10 +560,10 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s return trimmedList, nil } -func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) ([]Pair, error) { // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - return e.executeTopNSlice(ctx, index, c, slice) + mapFn := func(shard uint64) (interface{}, error) { + return e.executeTopNShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -572,7 +572,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return Pairs(other).Add(v.([]Pair)) } - other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -584,32 +584,32 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C return results, nil } -// executeTopNSlice executes a TopN call for a single slice. -func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { +// executeTopNShard executes a TopN call for a single shard. +func (e *Executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { field, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } minThreshold, _, err := c.UintArg("threshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } attrValues, _ := c.Args["attrValues"].([]interface{}) tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") if err != nil { - return nil, fmt.Errorf("executeTopNSlice: %v", err) + return nil, fmt.Errorf("executeTopNShard: %v", err) } // Retrieve bitmap used to intersect. var src *Row if len(c.Children) == 1 { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return nil, err } @@ -623,7 +623,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca field = defaultField } - f := e.Holder.Fragment(index, field, ViewStandard, slice) + f := e.Holder.Fragment(index, field, ViewStandard, shard) if f == nil { return nil, nil } @@ -646,14 +646,14 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca }) } -// executeDifferenceSlice executes a difference() call for a local slice. -func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeDifferenceShard executes a difference() call for a local shard. +func (e *Executor) executeDifferenceShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Difference query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -668,7 +668,7 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c * return other, nil } -func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +func (e *Executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { @@ -693,21 +693,21 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice) + frag := e.Holder.Fragment(index, fieldName, ViewStandard, shard) if frag == nil { return NewRow(), nil } return frag.row(rowID), nil } -// executeIntersectSlice executes a intersect() call for a local slice. -func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeIntersectShard executes a intersect() call for a local shard. +func (e *Executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { var other *Row if len(c.Children) == 0 { return nil, fmt.Errorf("empty Intersect query is currently not supported") } for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -722,11 +722,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p return other, nil } -// executeRangeSlice executes a range() call for a local slice. -func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeRangeShard executes a range() call for a local shard. +func (e *Executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Handle bsiGroup ranges differently. if c.HasConditionArg() { - return e.executeBSIGroupRangeSlice(ctx, index, c, slice) + return e.executeBSIGroupRangeShard(ctx, index, c, shard) } // Parse field. @@ -750,7 +750,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Read row & column id. rowID, rowOK, err := c.UintArg(fieldName) if err != nil { - return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) + return nil, fmt.Errorf("executeRangeShard - reading row: %v", err) } if !rowOK { return nil, fmt.Errorf("Range() must specify %q", rowLabel) @@ -785,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based views. row := &Row{} for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, fieldName, view, slice) + f := e.Holder.Fragment(index, fieldName, view, shard) if f == nil { continue } @@ -795,8 +795,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return row, nil } -// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice. -func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. +func (e *Executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Only one conditional should be present. if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") @@ -836,7 +836,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -857,7 +857,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, // The reason we don't just call: // return f.RangeBetween(fieldName, predicates[0], predicates[1]) - // here is because we need the call to be slice-specific. + // here is because we need the call to be shard-specific. // Find bsiGroup. bsig := f.bsiGroup(fieldName) @@ -871,7 +871,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -904,7 +904,7 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } // Retrieve fragment. - frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) if frag == nil { return NewRow(), nil } @@ -925,11 +925,11 @@ func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, } } -// executeUnionSlice executes a union() call for a local slice. -func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeUnionShard executes a union() call for a local shard. +func (e *Executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -944,11 +944,11 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C return other, nil } -// executeXorSlice executes a xor() call for a local slice. -func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { +// executeXorShard executes a xor() call for a local shard. +func (e *Executor) executeXorShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { other := NewRow() for i, input := range c.Children { - row, err := e.executeBitmapCallSlice(ctx, index, input, slice) + row, err := e.executeBitmapCallShard(ctx, index, input, shard) if err != nil { return nil, err } @@ -964,7 +964,7 @@ func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Cal } // executeCount executes a count() call. -func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { +func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (uint64, error) { if len(c.Children) == 0 { return 0, errors.New("Count() requires an input bitmap") } else if len(c.Children) > 1 { @@ -972,8 +972,8 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, } // Execute calls in bulk on each remote node and merge. - mapFn := func(slice uint64) (interface{}, error) { - row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + mapFn := func(shard uint64) (interface{}, error) { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { return 0, err } @@ -986,7 +986,7 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, return other + v.(uint64) } - result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return 0, err } @@ -1032,9 +1032,9 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal // executeClearBitField executes a Clear() call for a single view. func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(rowID, colID, nil) @@ -1107,10 +1107,10 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, // executeSetBitField executes a Set() call for a specific view. func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { - slice := colID / SliceWidth + shard := colID / ShardWidth ret := false - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, node := range e.Cluster.shardNodes(index, shard) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(rowID, colID, timestamp) @@ -1389,12 +1389,12 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p return nil } -// exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +// exec executes a PQL query remotely for a set of shards on a node. +func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *pql.Query, shards []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ Query: q.String(), - Slices: slices, + Shards: shards, Remote: true, } @@ -1439,29 +1439,29 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * return results, nil } -// slicesByNode returns a mapping of nodes to slices. -// Returns errSliceUnavailable if a slice cannot be allocated to a node. -func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) { +// shardsByNode returns a mapping of nodes to shards. +// Returns errShardUnavailable if a shard cannot be allocated to a node. +func (e *Executor) shardsByNode(nodes []*Node, index string, shards []uint64) (map[*Node][]uint64, error) { m := make(map[*Node][]uint64) loop: - for _, slice := range slices { - for _, node := range e.Cluster.sliceNodes(index, slice) { + for _, shard := range shards { + for _, node := range e.Cluster.shardNodes(index, shard) { if Nodes(nodes).Contains(node) { - m[node] = append(m[node], slice) + m[node] = append(m[node], shard) continue loop } } - return nil, errSliceUnavailable + return nil, errShardUnavailable } return m, nil } // mapReduce maps and reduces data across the cluster. // -// If a mapping of slices to a node fails then the slices are resplit across +// If a mapping of shards to a node fails then the shards are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. -func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { +func (e *Executor) mapReduce(ctx context.Context, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { ch := make(chan mapResponse) // Wrap context with a cancel to kill goroutines on exit. @@ -1480,13 +1480,13 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, ch, nodes, index, shards, c, opt, mapFn, reduceFn); err != nil { return nil, errors.Wrap(err, "starting mapper") } // Iterate over all map responses and reduce. var result interface{} - var maxSlice int + var maxShard int for { select { case <-ctx.Done(): @@ -1500,7 +1500,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.shards, c, opt, mapFn, reduceFn); err == errShardUnavailable { return nil, resp.err } else if err != nil { return nil, err @@ -1511,32 +1511,32 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, // Reduce value. result = reduceFn(result, resp.result) - // If all slices have been processed then return. - maxSlice += len(resp.slices) - if maxSlice >= len(slices) { + // If all shards have been processed then return. + maxShard += len(resp.shards) + if maxShard >= len(shards) { return result, nil } } } } -func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { - // Group slices together by nodes. - m, err := e.slicesByNode(nodes, index, slices) +func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, shards []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { + // Group shards together by nodes. + m, err := e.shardsByNode(nodes, index, shards) if err != nil { return err } // Execute each node in a separate goroutine. - for n, nodeSlices := range m { - go func(n *Node, nodeSlices []uint64) { - resp := mapResponse{node: n, slices: nodeSlices} + for n, nodeShards := range m { + go func(n *Node, nodeShards []uint64) { + resp := mapResponse{node: n, shards: nodeShards} - // Send local slices to mapper, otherwise remote exec. + // Send local shards to mapper, otherwise remote exec. if n.ID == e.Node.ID { - resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) + resp.result, resp.err = e.mapperLocal(ctx, nodeShards, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.remoteExec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeShards, opt) if len(results) > 0 { resp.result = results[0] } @@ -1548,30 +1548,30 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod case <-ctx.Done(): case ch <- resp: } - }(n, nodeSlices) + }(n, nodeShards) } return nil } // mapperLocal performs map & reduce entirely on the local node. -func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { - ch := make(chan mapResponse, len(slices)) +func (e *Executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { + ch := make(chan mapResponse, len(shards)) - for _, slice := range slices { - go func(slice uint64) { - result, err := mapFn(slice) + for _, shard := range shards { + go func(shard uint64) { + result, err := mapFn(shard) // Return response to the channel. select { case <-ctx.Done(): case ch <- mapResponse{result: result, err: err}: } - }(slice) + }(shard) } // Reduce results - var maxSlice int + var maxShard int var result interface{} for { select { @@ -1582,11 +1582,11 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu return nil, resp.err } result = reduceFn(result, resp.result) - maxSlice++ + maxShard++ } - // Exit once all slices are processed. - if maxSlice == len(slices) { + // Exit once all shards are processed. + if maxShard == len(shards) { return result, nil } } @@ -1695,16 +1695,16 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res return result, nil } -// errSliceUnavailable is a marker error if no nodes are available. -var errSliceUnavailable = errors.New("slice unavailable") +// errShardUnavailable is a marker error if no nodes are available. +var errShardUnavailable = errors.New("shard unavailable") -type mapFunc func(slice uint64) (interface{}, error) +type mapFunc func(shard uint64) (interface{}, error) type reduceFunc func(prev, v interface{}) interface{} type mapResponse struct { node *Node - slices []uint64 + shards []uint64 result interface{} err error @@ -1740,7 +1740,7 @@ func hasOnlySetRowAttrs(calls []*pql.Call) bool { return true } -func needsSlices(calls []*pql.Call) bool { +func needsShards(calls []*pql.Call) bool { if len(calls) == 0 { return false } diff --git a/executor_test.go b/executor_test.go index 250d8b13d..dd1a714b8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -46,8 +46,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } @@ -57,7 +57,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) @@ -75,7 +75,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Inhibit row attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) @@ -95,12 +95,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20), + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), ), nil, nil); err != nil { t.Fatal(err) } - if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } }) @@ -170,17 +170,17 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 1) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -201,16 +201,16 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -234,16 +234,16 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, SliceWidth+1) - hldr.SetBit("i", "general", 10, SliceWidth+2) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, SliceWidth+2) + hldr.SetBit("i", "general", 11, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -253,8 +253,8 @@ func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() hldr.SetBit("i", "f", 10, 3) - hldr.SetBit("i", "f", 10, SliceWidth+1) - hldr.SetBit("i", "f", 10, SliceWidth+2) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+2) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { @@ -506,7 +506,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { @@ -516,12 +516,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, f=0) Set(1, f=0) - Set(`+strconv.Itoa(SliceWidth)+`, f=0) - Set(`+strconv.Itoa(SliceWidth+2)+`, f=0) - Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0) + Set(`+strconv.Itoa(ShardWidth)+`, f=0) + Set(`+strconv.Itoa(ShardWidth+2)+`, f=0) + Set(`+strconv.Itoa((5*ShardWidth)+100)+`, f=0) Set(0, f=10) - Set(`+strconv.Itoa(SliceWidth)+`, f=10) - Set(`+strconv.Itoa(SliceWidth)+`, f=20) + Set(`+strconv.Itoa(ShardWidth)+`, f=10) + Set(`+strconv.Itoa(ShardWidth)+`, f=20) Set(0, other=0) `), nil, nil); err != nil { t.Fatal(err) @@ -546,7 +546,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { @@ -586,13 +586,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 0, 2) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 1, SliceWidth+2) - hldr.SetBit("i", "f", 1, SliceWidth) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 1, ShardWidth+2) + hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -611,22 +611,22 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 0, 2*SliceWidth) - hldr.SetBit("i", "f", 0, 3*SliceWidth) - hldr.SetBit("i", "f", 0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 0, 2*ShardWidth) + hldr.SetBit("i", "f", 0, 3*ShardWidth) + hldr.SetBit("i", "f", 0, 4*ShardWidth) hldr.SetBit("i", "f", 1, 0) hldr.SetBit("i", "f", 1, 1) - hldr.SetBit("i", "f", 2, SliceWidth) - hldr.SetBit("i", "f", 2, SliceWidth+1) + hldr.SetBit("i", "f", 2, ShardWidth) + hldr.SetBit("i", "f", 2, ShardWidth+1) - hldr.SetBit("i", "f", 3, 2*SliceWidth) - hldr.SetBit("i", "f", 3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*ShardWidth) + hldr.SetBit("i", "f", 3, 2*ShardWidth+1) - hldr.SetBit("i", "f", 4, 3*SliceWidth) - hldr.SetBit("i", "f", 4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*ShardWidth) + hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) @@ -644,20 +644,20 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set columns for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 0, SliceWidth) - hldr.SetBit("i", "f", 10, SliceWidth) - hldr.SetBit("i", "f", 10, SliceWidth+1) - hldr.SetBit("i", "f", 20, SliceWidth) - hldr.SetBit("i", "f", 20, SliceWidth+1) - hldr.SetBit("i", "f", 20, SliceWidth+2) + hldr.SetBit("i", "f", 0, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth) + hldr.SetBit("i", "f", 20, ShardWidth+1) + hldr.SetBit("i", "f", 20, ShardWidth+2) // Create an intersecting row. - hldr.SetBit("i", "other", 100, SliceWidth) - hldr.SetBit("i", "other", 100, SliceWidth+1) - hldr.SetBit("i", "other", 100, SliceWidth+2) + hldr.SetBit("i", "other", 100, ShardWidth) + hldr.SetBit("i", "other", 100, ShardWidth+1) + hldr.SetBit("i", "other", 100, ShardWidth+2) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() @@ -683,7 +683,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, ShardWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) @@ -706,7 +706,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { defer hldr.Close() hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) - hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, ShardWidth) if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -747,18 +747,18 @@ func TestExecutor_Execute_MinMax(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, x=0) Set(3, x=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) Set(1, x=1) - Set(`+strconv.Itoa(SliceWidth+2)+`, x=2) + Set(`+strconv.Itoa(ShardWidth+2)+`, x=2) SetValue(col=0, f=20) SetValue(col=1, f=-5) SetValue(col=2, f=-5) SetValue(col=3, f=10) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, f=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, f=40) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, f=50) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, f=60) `), nil, nil); err != nil { t.Fatal(err) } @@ -857,14 +857,14 @@ func TestExecutor_Execute_Sum(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, x=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, x=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) SetValue(col=0, foo=20) SetValue(col=0, bar=2000) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=40) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=50) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) SetValue(col=0, other=1000) `), nil, nil); err != nil { t.Fatal(err) @@ -979,14 +979,14 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { if _, err := e.Execute(context.Background(), "i", test.MustParse(` Set(0, f=0) - Set(`+strconv.Itoa(SliceWidth+1)+`, f=0) + Set(`+strconv.Itoa(ShardWidth+1)+`, f=0) SetValue(col=50, foo=20) SetValue(col=50, bar=2000) - SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10) - SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20) - SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60) + SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) + SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=10) + SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=20) + SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) SetValue(col=0, other=1000) SetValue(col=0, edge=100) SetValue(col=1, edge=-100) @@ -997,7 +997,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1012,7 +1012,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // NEQ if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - @@ -1027,7 +1027,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("LT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1035,7 +1035,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1043,7 +1043,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("GT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1051,7 +1051,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Run("GTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -1129,35 +1129,35 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Row(f=10)` { t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{1}) { - t.Fatalf("unexpected slices: %+v", slices) + } else if !reflect.DeepEqual(shards, []uint64{1}) { + t.Fatalf("unexpected shards: %+v", shards) } - // Set columns in slice 0 & 2. + // Set columns in shard 0 & 2. r := pilosa.NewRow( - (0*SliceWidth)+1, - (0*SliceWidth)+2, - (2*SliceWidth)+4, + (0*ShardWidth)+1, + (0*ShardWidth)+2, + (2*ShardWidth)+4, ) return []interface{}{r}, nil } // Create local executor data. - // The local node owns slice 1. + // The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*ShardWidth + 4}) { t.Fatalf("unexpected columns: %+v", columns) } } @@ -1180,16 +1180,16 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(10)}, nil } - // Create local executor data. The local node owns slice 1. + // Create local executor data. The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, (2*SliceWidth)+1) - hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) + hldr.SetBit("i", "f", 10, (2*ShardWidth)+1) + hldr.SetBit("i", "f", 10, (2*ShardWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { @@ -1219,7 +1219,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Set(_col=2, f=10)` { @@ -1274,7 +1274,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) } else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` { @@ -1330,15 +1330,15 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if !reflect.DeepEqual(slices, []uint64{1, 3}) { - t.Fatalf("unexpected slices: %+v", slices) + } else if !reflect.DeepEqual(shards, []uint64{1, 3}) { + t.Fatalf("unexpected shards: %+v", shards) } // Query should be executed twice. Once to get the top bitmaps for the - // slices and a second time to get the counts for a set of bitmaps. + // shards and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: if query.String() != `TopN(_field="f", n=3)` { @@ -1361,12 +1361,12 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { }}, nil } - // Create local executor data on slice 2 & 4. + // Create local executor data on shard 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 30, (2*SliceWidth)+1) - hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) + hldr.SetBit("i", "f", 30, (2*ShardWidth)+1) + hldr.SetBit("i", "f", 30, (4*ShardWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil { @@ -1396,7 +1396,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { c.Nodes[1].URI = *uri // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) } else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` { @@ -1407,7 +1407,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { } // Create local executor data. - // The local node owns slice 1. + // The local node owns shard 1. hldr := test.MustOpenHolder() defer hldr.Close() @@ -1417,7 +1417,7 @@ func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { } f := hldr.Field("i", "f") s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+1) e := test.NewExecutor(hldr.Holder, c) if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { diff --git a/field.go b/field.go index 4fd684682..23de79f7c 100644 --- a/field.go +++ b/field.go @@ -152,15 +152,15 @@ func (f *Field) Path() string { return f.path } // RowAttrStore returns the attribute storage. func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } -// MaxSlice returns the max slice in the field. -func (f *Field) MaxSlice() uint64 { +// MaxShard returns the max shard in the field. +func (f *Field) MaxShard() uint64 { f.mu.RLock() defer f.mu.RUnlock() var max uint64 for _, view := range f.views { - if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max { - max = viewMaxSlice + if viewMaxShard := view.calculateMaxShard(); viewMaxShard > max { + max = viewMaxShard } } return max @@ -665,7 +665,7 @@ func (f *Field) Row(rowID uint64) (*Row, error) { return view.row(rowID), nil } -// ViewRow returns a row for a view and slice. +// ViewRow returns a row for a view and shard. // TODO: unexport this with views (it's only used in tests). func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { view := f.View(viewName) @@ -934,7 +934,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Attach bit to each standard view. for _, name := range standard { - key := importKey{View: name, Slice: columnID / SliceWidth} + key := importKey{View: name, Shard: columnID / ShardWidth} data := dataByFragment[key] data.RowIDs = append(data.RowIDs, rowID) data.ColumnIDs = append(data.ColumnIDs, columnID) @@ -949,7 +949,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.Wrap(err, "creating view") } - frag, err := view.CreateFragmentIfNotExists(key.Slice) + frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { return errors.Wrap(err, "creating view") } @@ -983,7 +983,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { // Attach value to each bsiGroup view. for _, name := range []string{viewName} { - key := importKey{View: name, Slice: columnID / SliceWidth} + key := importKey{View: name, Shard: columnID / ShardWidth} data := dataByFragment[key] data.ColumnIDs = append(data.ColumnIDs, columnID) data.Values = append(data.Values, value) @@ -1001,7 +1001,7 @@ func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { return errors.Wrap(err, "creating view") } - frag, err := view.CreateFragmentIfNotExists(key.Slice) + frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -1192,7 +1192,7 @@ func (b *bsiGroup) BitDepth() uint { // Note that in this case (because the range uses the full BitDepth 0 to 1023), // we can't simply return 1024. // In order to make this work, we effectively need to change the operator to LTE. -// Executor.executeBSIGroupRangeSlice() takes this into account and returns +// Executor.executeBSIGroupRangeShard() takes this into account and returns // `frag.FieldNotNull(bsig.BitDepth())` in such instances. func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { if op == pql.GT || op == pql.GTE { diff --git a/fragment.go b/fragment.go index 9b4780795..891c1570e 100644 --- a/fragment.go +++ b/fragment.go @@ -44,8 +44,8 @@ import ( ) const ( - // SliceWidth is the number of column IDs in a slice. - SliceWidth = 1048576 + // ShardWidth is the number of column IDs in a shard. + ShardWidth = 1048576 // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" @@ -63,7 +63,7 @@ const ( defaultFragmentMaxOpN = 2000 ) -// Fragment represents the intersection of a field and slice in an index. +// Fragment represents the intersection of a field and shard in an index. type Fragment struct { mu sync.RWMutex @@ -71,7 +71,7 @@ type Fragment struct { index string field string view string - slice uint64 + shard uint64 // File-backed storage path string @@ -110,13 +110,13 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, index, field, view string, slice uint64) *Fragment { +func NewFragment(path, index, field, view string, shard uint64) *Fragment { return &Fragment{ path: path, index: index, field: field, view: view, - slice: slice, + shard: shard, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -151,7 +151,7 @@ func (f *Fragment) Open() error { // Read last bit to determine max row. pos := f.storage.Max() - f.maxRowID = pos / SliceWidth + f.maxRowID = pos / ShardWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) return nil @@ -165,7 +165,7 @@ func (f *Fragment) Open() error { // openStorage opens the storage bitmap. func (f *Fragment) openStorage() error { - // Create a roaring bitmap to serve as storage for the slice. + // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() } @@ -257,7 +257,7 @@ func (f *Fragment) openCache() error { // Read in all rows by ID. // This will cause them to be added to the cache. for _, id := range pb.IDs { - n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) + n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth) f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -337,7 +337,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by - data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth) + data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. // We Clone() data because otherwise row will contains pointers to containers in storage. @@ -345,7 +345,7 @@ func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac row := &Row{ segments: []RowSegment{{ data: *data.Clone(), - slice: f.slice, + shard: f.shard, writable: false, }}, } @@ -837,9 +837,9 @@ func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64 // pos translates the row ID and column ID into a position in the storage bitmap. func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { - // Return an error if the column ID is out of the range of the fragment's slice. - minColumnID := f.slice * SliceWidth - if columnID < minColumnID || columnID >= minColumnID+SliceWidth { + // Return an error if the column ID is out of the range of the fragment's shard. + minColumnID := f.shard * ShardWidth + if columnID < minColumnID || columnID >= minColumnID+ShardWidth { return 0, errors.New("column out of bounds") } return pos(rowID, columnID), nil @@ -859,7 +859,7 @@ func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error { } // Invoke caller's function. - err = fn(i/SliceWidth, (f.slice*SliceWidth)+(i%SliceWidth)) + err = fn(i/ShardWidth, (f.shard*ShardWidth)+(i%ShardWidth)) }) return err } @@ -1093,16 +1093,16 @@ func (f *Fragment) Blocks() []FragmentBlock { if eof { return nil } - blockID := int(v / (HashBlockSize * SliceWidth)) + blockID := int(v / (HashBlockSize * ShardWidth)) for { // Check for multiple block checksums in a row. if n := f.readContiguousChecksums(&a, blockID); n > 0 { - itr.Seek(uint64(blockID+n) * HashBlockSize * SliceWidth) + itr.Seek(uint64(blockID+n) * HashBlockSize * ShardWidth) v, eof = itr.Next() if eof { break } - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) continue } @@ -1113,7 +1113,7 @@ func (f *Fragment) Blocks() []FragmentBlock { // Read all values for the block. for ; ; v, eof = itr.Next() { // Once we hit the next block, save the value for the next iteration. - blockID = int(v / (HashBlockSize * SliceWidth)) + blockID = int(v / (HashBlockSize * ShardWidth)) if blockID != h.blockID || eof { break } @@ -1160,9 +1160,9 @@ func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() - f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) { - rowIDs = append(rowIDs, i/SliceWidth) - columnIDs = append(columnIDs, i%SliceWidth) + f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) { + rowIDs = append(rowIDs, i/ShardWidth) + columnIDs = append(columnIDs, i%ShardWidth) }) return } @@ -1190,7 +1190,7 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize - maxColumnID := uint64(SliceWidth) + maxColumnID := uint64(ShardWidth) // Create buffered iterator for local block. itrs := make([]*BufIterator, 1, len(data)+1) @@ -1278,14 +1278,14 @@ func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // Set local bits. for i := range sets[0].columnIDs { - if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil { + if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.shard*ShardWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. for i := range clears[0].columnIDs { - if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.shard*ShardWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1423,8 +1423,8 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *Fragment) snapshot() error { - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice) - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) + completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) @@ -1736,7 +1736,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice) + nodes := s.Cluster.shardNodes(s.Fragment.index, s.Fragment.shard) if len(nodes) == 1 { return nil } @@ -1752,7 +1752,7 @@ func (s *FragmentSyncer) syncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1817,7 +1817,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var uris []*URI var pairSets []pairSet - for _, node := range s.Cluster.sliceNodes(f.index, f.slice) { + for _, node := range s.Cluster.shardNodes(f.index, f.shard) { if s.Node.ID == node.ID { continue } @@ -1831,7 +1831,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.shard, id) if err != nil { return errors.Wrap(err, "getting block") } @@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Only sync the standard block. for j := 0; j < len(set.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) count++ } for j := 0; j < len(clear.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) count++ } @@ -1933,5 +1933,5 @@ func byteSlicesEqual(a [][]byte) bool { // pos returns the row position of a row/column pair. func pos(rowID, columnID uint64) uint64 { - return (rowID * SliceWidth) + (columnID % SliceWidth) + return (rowID * ShardWidth) + (columnID % ShardWidth) } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e7a77dafb..c779dd596 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -742,7 +742,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { - slice := uint64(0) + shard := uint64(0) cacheSize := uint32(3) // Create Index. @@ -762,7 +762,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } // Create fragment. - frag, err := view.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } @@ -1181,7 +1181,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { for row := 0; row < 100; row++ { val := 1 i := 0 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 @@ -1215,7 +1215,7 @@ func BenchmarkFragment_Import(b *testing.B) { i := 0 for row := 0; row < 100; row++ { val := 1 - for col := 0; col < SliceWidth/2; col++ { + for col := 0; col < ShardWidth/2; col++ { rows[i] = uint64(row) cols[i] = uint64(val) val += 2 @@ -1237,7 +1237,7 @@ func BenchmarkFragment_Import(b *testing.B) { ///////////////////////////////////////////////////////////////////// // mustOpenFragment returns a new instance of Fragment with a temporary path. -func mustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { +func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -1248,7 +1248,7 @@ func mustOpenFragment(index, field, view string, slice uint64, cacheType string) cacheType = DefaultCacheType } - f := NewFragment(file.Name(), index, field, view, slice) + f := NewFragment(file.Name(), index, field, view, shard) f.CacheType = cacheType f.RowAttrStore = newMemAttrStore() diff --git a/handler.go b/handler.go index c9a476e13..1f3d04300 100644 --- a/handler.go +++ b/handler.go @@ -12,9 +12,9 @@ type QueryRequest struct { // The query string to parse and execute. Query string - // The slices to include in the query execution. - // If empty, all slices are included. - Slices []uint64 + // The shards to include in the query execution. + // If empty, all shards are included. + Shards []uint64 // Return column attributes, if true. ColumnAttrs bool diff --git a/holder.go b/holder.go index 316ca1244..bfb2ae758 100644 --- a/holder.go +++ b/holder.go @@ -200,11 +200,11 @@ func (h *Holder) HasData() (bool, error) { return false, nil } -// MaxSlices returns MaxSlice map for all indexes. -func (h *Holder) MaxSlices() map[string]uint64 { +// MaxShards returns MaxShard map for all indexes. +func (h *Holder) MaxShards() map[string]uint64 { a := make(map[string]uint64) for _, index := range h.Indexes() { - a[index.Name()] = index.MaxSlice() + a[index.Name()] = index.MaxShard() } return a } @@ -257,10 +257,10 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { return nil } -// EncodeMaxSlices creates and internal representation of max slices. -func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { - return &internal.MaxSlices{ - Standard: h.MaxSlices(), +// EncodeMaxShards creates and internal representation of max shards. +func (h *Holder) EncodeMaxShards() *internal.MaxShards { + return &internal.MaxShards{ + Standard: h.MaxShards(), } } @@ -411,13 +411,13 @@ func (h *Holder) View(index, field, name string) *View { return f.View(name) } -// Fragment returns the fragment for an index, field & slice. -func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment { +// Fragment returns the fragment for an index, field & shard. +func (h *Holder) Fragment(index, field, view string, shard uint64) *Fragment { v := h.View(index, field, view) if v == nil { return nil } - return v.Fragment(slice) + return v.Fragment(shard) } // monitorCacheFlush periodically flushes all fragment caches sequentially. @@ -619,9 +619,9 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { - // Ignore slices that this host doesn't own. - if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) { + for shard := uint64(0); shard <= s.Holder.Index(di.Name).MaxShard(); shard++ { + // Ignore shards that this host doesn't own. + if !s.Cluster.ownsShard(s.Node.ID, di.Name, shard) { continue } @@ -631,8 +631,8 @@ func (s *HolderSyncer) SyncHolder() error { } // Sync fragment if own it. - if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + if err := s.syncFragment(di.Name, fi.Name, vi.Name, shard); err != nil { + return fmt.Errorf("fragment sync error: index=%s, field=%s, shard=%d, err=%s", di.Name, fi.Name, shard, err) } } } @@ -736,7 +736,7 @@ func (s *HolderSyncer) syncField(index, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error { +func (s *HolderSyncer) syncFragment(index, field, view string, shard uint64) error { // Retrieve local field. f := s.Holder.Field(index, field) if f == nil { @@ -750,7 +750,7 @@ func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) err } // Ensure fragment exists locally. - frag, err := v.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return errors.Wrap(err, "creating fragment") } @@ -800,19 +800,19 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node) + containedShards := c.Cluster.containsShards(index.Name(), index.MaxShard(), c.Node) // Get the fragments registered in memory. for _, field := range index.Fields() { for _, view := range field.Views() { for _, fragment := range view.allFragments() { - fragSlice := fragment.slice + fragShard := fragment.shard // Ignore fragments that should be present. - if uint64InSlice(fragSlice, containedSlices) { + if uint64InSlice(fragShard, containedShards) { continue } // Delete fragment. - if err := view.deleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragShard); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_test.go b/holder_test.go index 9b3e21c12..f758798a4 100644 --- a/holder_test.go +++ b/holder_test.go @@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: slice=0, err=opening storage: unmarshal storage") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open fragment: shard=0, err=opening storage: unmarshal storage") { t.Fatalf("unexpected error: %s", err) } }) @@ -373,12 +373,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1 := test.MustOpenHolder() defer hldr1.Close() s.Handler.API.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } // Mock 2-node, fully replicated cluster. @@ -400,7 +400,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.SetBit("i", "f", 120, 10) hldr0.SetBit("i", "f", 200, 4) - hldr0.SetBit("i", "f0", 9, SliceWidth+5) + hldr0.SetBit("i", "f0", 9, ShardWidth+5) // Set a bit to create the fragment. hldr0.SetBit("y", "z", 0, 0) @@ -410,13 +410,13 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1.SetBit("i", "f", 3, 10) hldr1.SetBit("i", "f", 120, 10) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5) - hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+4) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5) + hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7) - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(3) + // Set highest shard. + hldr0.Index("i").SetRemoteMaxShard(1) + hldr0.Index("y").SetRemoteMaxShard(3) // Set up syncer. syncer := pilosa.HolderSyncer{ @@ -444,11 +444,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -482,15 +482,15 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { hldr0.SetBit("i", "f", 120, 10) hldr0.SetBit("i", "f", 200, 4) - hldr0.SetBit("i", "f0", 9, SliceWidth+5) + hldr0.SetBit("i", "f0", 9, ShardWidth+5) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5) - hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) + hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) - // Set highest slice. - hldr0.Index("i").SetRemoteMaxSlice(1) - hldr0.Index("y").SetRemoteMaxSlice(2) + // Set highest shard. + hldr0.Index("i").SetRemoteMaxShard(1) + hldr0.Index("y").SetRemoteMaxShard(2) // Keep replication the same and ensure we get the expected results. cluster.ReplicaN = 2 @@ -520,11 +520,11 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -562,7 +562,7 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/http/client.go b/http/client.go index 378413c2f..59efb4f72 100644 --- a/http/client.go +++ b/http/client.go @@ -73,15 +73,15 @@ func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) // Host returns the host the client was initialized with. func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) +// MaxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxShardByIndex(ctx) } -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { +// maxShardByIndex returns the number of shards on a server by index. +func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") + u := uriPathToURL(c.defaultURI, "/shards/max") // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -99,7 +99,7 @@ func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64 } defer resp.Body.Close() - var rsp getSlicesMaxResponse + var rsp getShardsMaxResponse if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("http: status=%d", resp.StatusCode) } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { @@ -184,11 +184,11 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo } } -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { +// FragmentNodes returns a list of nodes that own a shard. +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() + u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode() // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -272,23 +272,23 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s return qresp, nil } -// Import bulk imports bits for a single slice to a host. -func (c *InternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { +// Import bulk imports bits for a single shard to a host. +func (c *InternalClient) Import(ctx context.Context, index, field string, shard uint64, bits []pilosa.Bit) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - buf, err := marshalImportPayload(index, field, slice, bits) + buf, err := marshalImportPayload(index, field, shard, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Import to each node. @@ -343,7 +343,7 @@ func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fiel } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) { +func marshalImportPayload(index, field string, shard uint64, bits []pilosa.Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() @@ -353,7 +353,7 @@ func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, RowIDs: rowIDs, ColumnIDs: columnIDs, Timestamps: timestamps, @@ -423,23 +423,23 @@ func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, buf return nil } -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { +// ImportValue bulk imports field values for a single shard to a host. +func (c *InternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []pilosa.FieldValue) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - buf, err := marshalImportValuePayload(index, field, slice, vals) + buf, err := marshalImportValuePayload(index, field, shard, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Import to each node. @@ -453,7 +453,7 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s } // marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) { +func marshalImportValuePayload(index, field string, shard uint64, vals []pilosa.FieldValue) ([]byte, error) { // Separate row and column IDs to reduce allocations. columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() @@ -462,7 +462,7 @@ func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, ColumnIDs: columnIDs, Values: values, }) @@ -510,18 +510,18 @@ func (c *InternalClient) importValueNode(ctx context.Context, node *pilosa.Node, return nil } -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { +// ExportCSV bulk exports data for a single shard from a host to CSV format. +func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) + // Retrieve a list of nodes that own the shard. + nodes, err := c.FragmentNodes(ctx, index, shard) if err != nil { - return fmt.Errorf("slice nodes: %s", err) + return fmt.Errorf("shard nodes: %s", err) } // Attempt nodes in random order. @@ -529,7 +529,7 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, field, shard, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) continue } else { @@ -541,13 +541,13 @@ func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, sli } // exportNode copies a CSV export from a node to w. -func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, w io.Writer) error { // Create URL. u := nodePathToURL(node, "/export") u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Generate HTTP request. @@ -578,19 +578,19 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i return nil } -func (c *InternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { node := &pilosa.Node{ URI: uri, } - return c.backupSliceNode(ctx, index, field, slice, node) + return c.backupShardNode(ctx, index, field, shard, node) } -func (c *InternalClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { +func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { u := nodePathToURL(node, "/fragment/data") u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Build request. @@ -671,7 +671,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64) ([]pilosa.FragmentBlock, error) { if uri == nil { uri = c.defaultURI } @@ -679,7 +679,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in u.RawQuery = url.Values{ "index": {index}, "field": {field}, - "slice": {strconv.FormatUint(slice, 10)}, + "shard": {strconv.FormatUint(shard, 10)}, }.Encode() // Build request. @@ -716,11 +716,11 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in } // BlockData returns row/column id pairs for a block. -func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Field: field, - Slice: slice, + Shard: shard, Block: uint64(block), }) if err != nil { @@ -952,17 +952,17 @@ func (p Bits) Timestamps() []int64 { return other } -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit { +// GroupByShard returns a map of bits by shard. +func (p Bits) GroupByShard() map[uint64][]pilosa.Bit { m := make(map[uint64][]pilosa.Bit) for _, bit := range p { - slice := bit.ColumnID / pilosa.SliceWidth - m[slice] = append(m[slice], bit) + shard := bit.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], bit) } - for slice, bits := range m { + for shard, bits := range m { sort.Sort(Bits(bits)) - m[slice] = bits + m[shard] = bits } return m @@ -996,17 +996,17 @@ func (p FieldValues) Values() []int64 { return other } -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue { +// GroupByShard returns a map of field values by shard. +func (p FieldValues) GroupByShard() map[uint64][]pilosa.FieldValue { m := make(map[uint64][]pilosa.FieldValue) for _, val := range p { - slice := val.ColumnID / pilosa.SliceWidth - m[slice] = append(m[slice], val) + shard := val.ColumnID / pilosa.ShardWidth + m[shard] = append(m[shard], val) } - for slice, vals := range m { + for shard, vals := range m { sort.Sort(FieldValues(vals)) - m[slice] = vals + m[shard] = vals } return m @@ -1027,7 +1027,7 @@ func (p BitsByPos) Less(i, j int) bool { // pos returns the row position of a row/column pair. func pos(rowID, columnID uint64) uint64 { - return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth) + return (rowID * pilosa.ShardWidth) + (columnID % pilosa.ShardWidth) } func uriPathToURL(uri *pilosa.URI, path string) url.URL { diff --git a/http/client_test.go b/http/client_test.go index 7363879ce..e5a595605 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -62,42 +62,42 @@ func TestClient_MultiNode(t *testing.T) { defer s[i].Close() } - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster - return e.Execute(ctx, index, query, slices, opt) + return e.Execute(ctx, index, query, shards, opt) } - // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - sliceNums := []uint64{1, 2, 6} + // Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN. + shardNums := []uint64{1, 2, 6} - // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())` + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsShards("i", 20, s[i].HostURI())` owns := [][]uint64{ {1, 3, 4, 8, 10, 13, 17, 19}, {2, 5, 7, 11, 12, 14, 18}, {0, 6, 9, 15, 16, 20}, } - for i, num := range sliceNums { + for i, num := range shardNums { ownsNum := false for _, ownNum := range owns[i] { if ownNum == num { @@ -106,18 +106,18 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %v", num, s[i].Host(), owns) + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, s[i].Host(), owns) } } - baseBit0 := pilosa.SliceWidth * sliceNums[0] - baseBit1 := pilosa.SliceWidth * sliceNums[1] - baseBit2 := pilosa.SliceWidth * sliceNums[2] + baseBit0 := pilosa.ShardWidth * shardNums[0] + baseBit1 := pilosa.ShardWidth * shardNums[1] + baseBit2 := pilosa.ShardWidth * shardNums[2] - maxSlice := uint64(0) - for _, x := range sliceNums { - if x > maxSlice { - maxSlice = x + maxShard := uint64(0) + for _, x := range shardNums { + if x > maxShard { + maxShard = x } } @@ -145,9 +145,9 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache() - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache() - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[0]).RecalculateCache() + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[1]).RecalculateCache() + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[2]).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -165,18 +165,18 @@ func TestClient_MultiNode(t *testing.T) { t.Fatal(err) } - // Check the results before every node has the correct max slice value. + // Check the results before every node has the correct max shard value. pairs := result.Results[0].Pairs for _, pair := range pairs { if pair.ID == 22 && pair.Count != 3 { - t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) + t.Fatalf("Invalid Cluster wide MaxShard prevents accurate calculation of %s", pair) } } - // Set max slice to correct value. - hldr[0].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[1].Index("i").SetRemoteMaxSlice(maxSlice) - hldr[2].Index("i").SetRemoteMaxSlice(maxSlice) + // Set max shard to correct value. + hldr[0].Index("i").SetRemoteMaxShard(maxShard) + hldr[1].Index("i").SetRemoteMaxShard(maxShard) + hldr[2].Index("i").SetRemoteMaxShard(maxShard) result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { @@ -328,7 +328,7 @@ func TestClient_FragmentBlocks(t *testing.T) { hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) - // Set a bit on a different slice. + // Set a bit on a different shard. hldr.SetBit("i", "f", 0, 1) c := MustNewClient(cmd.URL(), defaultClient) blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) diff --git a/http/handler.go b/http/handler.go index 58a3f751d..4cc7099ba 100644 --- a/http/handler.go +++ b/http/handler.go @@ -156,13 +156,13 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") + h.validators["GetShardMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "shard") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -195,7 +195,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") @@ -385,20 +385,20 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { +// handleGetShardsMax handles GET /shards/max requests. +func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), + if err := json.NewEncoder(w).Encode(getShardsMaxResponse{ + Standard: h.API.MaxShards(r.Context()), }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) + h.Logger.Printf("write shards-max response error: %s", err) } } -type getSlicesMaxResponse struct { +type getShardsMaxResponse struct { Standard map[string]uint64 `json:"standard"` } @@ -839,15 +839,15 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } query := string(buf) - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) + // Parse list of shards. + shards, err := parseUint64Slice(q.Get("shards")) if err != nil { - return nil, errors.New("invalid slice argument") + return nil, errors.New("invalid shard argument") } return &pilosa.QueryRequest{ Query: query, - Slices: slices, + Shards: shards, ColumnAttrs: q.Get("columnAttrs") == "true", ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", ExcludeColumns: q.Get("excludeColumns") == "true", @@ -908,7 +908,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { fallthrough case pilosa.ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -961,7 +961,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) fallthrough case pilosa.ErrFieldNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -998,17 +998,17 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() index, field := q.Get("index"), q.Get("field") - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) + http.Error(w, "invalid shard", http.StatusBadRequest) return } - if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { + if err = h.API.ExportCSV(r.Context(), index, field, shard, w); err != nil { switch errors.Cause(err) { case pilosa.ErrFragmentNotFound: break - case pilosa.ErrClusterDoesNotOwnSlice: + case pilosa.ErrClusterDoesNotOwnShard: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1026,15 +1026,15 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) q := r.URL.Query() index := q.Get("index") - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + // Read shard parameter. + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) + http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) return } // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) + nodes, err := h.API.ShardNodes(r.Context(), index, shard) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -1072,15 +1072,15 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - // Read slice parameter. + // Read shard parameter. q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) + http.Error(w, "shard required", http.StatusBadRequest) return } - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), shard) if err != nil { if errors.Cause(err) == pilosa.ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -1131,7 +1131,7 @@ const ( func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { req := &pilosa.QueryRequest{ Query: pb.Query, - Slices: pb.Slices, + Shards: pb.Shards, ColumnAttrs: pb.ColumnAttrs, Remote: pb.Remote, ExcludeRowAttrs: pb.ExcludeRowAttrs, diff --git a/index.go b/index.go index a118b1808..ae6141b90 100644 --- a/index.go +++ b/index.go @@ -38,8 +38,8 @@ type Index struct { // Fields by name. fields map[string]*Field - // Max Slice on any node in the cluster, according to this node. - remoteMaxSlice uint64 + // Max shard on any node in the cluster, according to this node. + remoteMaxShard uint64 NewAttrStore func(string) AttrStore @@ -64,7 +64,7 @@ func NewIndex(path, name string) (*Index, error) { name: name, fields: make(map[string]*Field), - remoteMaxSlice: 0, + remoteMaxShard: 0, NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, @@ -210,30 +210,30 @@ func (i *Index) Close() error { return nil } -// MaxSlice returns the max slice in the index according to this node. -func (i *Index) MaxSlice() uint64 { +// MaxShard returns the max shard in the index according to this node. +func (i *Index) MaxShard() uint64 { if i == nil { return 0 } i.mu.RLock() defer i.mu.RUnlock() - max := i.remoteMaxSlice + max := i.remoteMaxShard for _, f := range i.fields { - if slice := f.MaxSlice(); slice > max { - max = slice + if shard := f.MaxShard(); shard > max { + max = shard } } - i.Stats.Gauge("maxSlice", float64(max), 1.0) + i.Stats.Gauge("maxShard", float64(max), 1.0) return max } -// SetRemoteMaxSlice sets the remote max slice value received from another node. -func (i *Index) SetRemoteMaxSlice(newmax uint64) { +// SetRemoteMaxShard sets the remote max shard value received from another node. +func (i *Index) SetRemoteMaxShard(newmax uint64) { i.mu.Lock() defer i.mu.Unlock() - i.remoteMaxSlice = newmax + i.remoteMaxShard = newmax } // FieldPath returns the path to a field in the index. @@ -427,7 +427,7 @@ func hasTime(a []*time.Time) bool { type importKey struct { View string - Slice uint64 + Shard uint64 } type importData struct { diff --git a/index_test.go b/index_test.go index 5beb67f00..d1a740a4f 100644 --- a/index_test.go +++ b/index_test.go @@ -23,8 +23,8 @@ import ( "github.com/pilosa/pilosa/test" ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth +// ShardWidth is a helper reference to use when testing. +const ShardWidth = pilosa.ShardWidth // Ensure index can open and retrieve a field. func TestIndex_CreateFieldIfNotExists(t *testing.T) { diff --git a/internal/private.pb.go b/internal/private.pb.go index c3dadb455..8fdcfa7db 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: private.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -14,8 +15,8 @@ BlockDataRequest BlockDataResponse Cache - MaxSlices - CreateSliceMessage + MaxShards + CreateShardMessage DeleteIndexMessage CreateIndexMessage CreateFieldMessage @@ -159,7 +160,7 @@ type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,4,opt,name=Shard,proto3" json:"Shard,omitempty"` Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } @@ -189,9 +190,9 @@ func (m *BlockDataRequest) GetView() string { return "" } -func (m *BlockDataRequest) GetSlice() uint64 { +func (m *BlockDataRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -243,42 +244,42 @@ func (m *Cache) GetIDs() []uint64 { return nil } -type MaxSlices struct { +type MaxShards struct { Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } -func (m *MaxSlices) Reset() { *m = MaxSlices{} } -func (m *MaxSlices) String() string { return proto.CompactTextString(m) } -func (*MaxSlices) ProtoMessage() {} -func (*MaxSlices) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } +func (m *MaxShards) Reset() { *m = MaxShards{} } +func (m *MaxShards) String() string { return proto.CompactTextString(m) } +func (*MaxShards) ProtoMessage() {} +func (*MaxShards) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{6} } -func (m *MaxSlices) GetStandard() map[string]uint64 { +func (m *MaxShards) GetStandard() map[string]uint64 { if m != nil { return m.Standard } return nil } -type CreateSliceMessage struct { +type CreateShardMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,2,opt,name=Shard,proto3" json:"Shard,omitempty"` } -func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } -func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) } -func (*CreateSliceMessage) ProtoMessage() {} -func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } +func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } +func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } +func (*CreateShardMessage) ProtoMessage() {} +func (*CreateShardMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } -func (m *CreateSliceMessage) GetIndex() string { +func (m *CreateShardMessage) GetIndex() string { if m != nil { return m.Index } return "" } -func (m *CreateSliceMessage) GetSlice() uint64 { +func (m *CreateShardMessage) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -565,7 +566,7 @@ func (m *NodeEventMessage) GetNode() *Node { type NodeStatus struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` - MaxSlices *MaxSlices `protobuf:"bytes,2,opt,name=MaxSlices" json:"MaxSlices,omitempty"` + MaxShards *MaxShards `protobuf:"bytes,2,opt,name=MaxShards" json:"MaxShards,omitempty"` Schema *Schema `protobuf:"bytes,3,opt,name=Schema" json:"Schema,omitempty"` } @@ -581,9 +582,9 @@ func (m *NodeStatus) GetNode() *Node { return nil } -func (m *NodeStatus) GetMaxSlices() *MaxSlices { +func (m *NodeStatus) GetMaxShards() *MaxShards { if m != nil { - return m.MaxSlices + return m.MaxShards } return nil } @@ -792,7 +793,7 @@ type ResizeSource struct { Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` - Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` } func (m *ResizeSource) Reset() { *m = ResizeSource{} } @@ -828,9 +829,9 @@ func (m *ResizeSource) GetView() string { return "" } -func (m *ResizeSource) GetSlice() uint64 { +func (m *ResizeSource) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -940,8 +941,8 @@ func init() { proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*MaxSlices)(nil), "internal.MaxSlices") - proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") + proto.RegisterType((*MaxShards)(nil), "internal.MaxShards") + proto.RegisterType((*CreateShardMessage)(nil), "internal.CreateShardMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") @@ -1111,10 +1112,10 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Block)) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x20 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } if len(m.View) > 0 { dAtA[i] = 0x2a @@ -1212,7 +1213,7 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *MaxSlices) Marshal() (dAtA []byte, err error) { +func (m *MaxShards) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1222,7 +1223,7 @@ func (m *MaxSlices) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { +func (m *MaxShards) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1246,7 +1247,7 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateShardMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1256,7 +1257,7 @@ func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateShardMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1267,10 +1268,10 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -1685,11 +1686,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { } i += n12 } - if m.MaxSlices != nil { + if m.MaxShards != nil { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n13, err := m.MaxSlices.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxShards.Size())) + n13, err := m.MaxShards.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1980,10 +1981,10 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) i += copy(dAtA[i:], m.View) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x28 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + i = encodeVarintPrivate(dAtA, i, uint64(m.Shard)) } return i, nil } @@ -2140,6 +2141,24 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2212,8 +2231,8 @@ func (m *BlockDataRequest) Size() (n int) { if m.Block != 0 { n += 1 + sovPrivate(uint64(m.Block)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } l = len(m.View) if l > 0 { @@ -2255,7 +2274,7 @@ func (m *Cache) Size() (n int) { return n } -func (m *MaxSlices) Size() (n int) { +func (m *MaxShards) Size() (n int) { var l int _ = l if len(m.Standard) > 0 { @@ -2269,15 +2288,15 @@ func (m *MaxSlices) Size() (n int) { return n } -func (m *CreateSliceMessage) Size() (n int) { +func (m *CreateShardMessage) Size() (n int) { var l int _ = l l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -2454,8 +2473,8 @@ func (m *NodeStatus) Size() (n int) { l = m.Node.Size() n += 1 + l + sovPrivate(uint64(l)) } - if m.MaxSlices != nil { - l = m.MaxSlices.Size() + if m.MaxShards != nil { + l = m.MaxShards.Size() n += 1 + l + sovPrivate(uint64(l)) } if m.Schema != nil { @@ -2591,8 +2610,8 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPrivate(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPrivate(uint64(m.Shard)) } return n } @@ -3140,9 +3159,9 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3152,7 +3171,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3493,7 +3512,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { } return nil } -func (m *MaxSlices) Unmarshal(dAtA []byte) error { +func (m *MaxShards) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3516,10 +3535,10 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: MaxSlices: wiretype end group for non-group") + return fmt.Errorf("proto: MaxShards: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: MaxSlices: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: MaxShards: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3548,14 +3567,51 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 + if iNdEx < postIndex { + var valuekey uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3565,69 +3621,31 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + valuekey |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy } + m.Standard[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.Standard[mapkey] = mapvalue } - m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3650,7 +3668,7 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { +func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3673,10 +3691,10 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateShardMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateShardMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3710,9 +3728,9 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3722,7 +3740,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -5053,7 +5071,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxShards", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -5077,10 +5095,10 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.MaxSlices == nil { - m.MaxSlices = &MaxSlices{} + if m.MaxShards == nil { + m.MaxShards = &MaxShards{} } - if err := m.MaxSlices.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.MaxShards.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -6080,9 +6098,9 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -6092,7 +6110,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -6681,70 +6699,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1028 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, - 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, - 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, - 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, - 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, - 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, - 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, - 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, - 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, - 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, - 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, - 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, - 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, - 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, - 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, - 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, - 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, - 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, - 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, - 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, - 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, - 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, - 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, - 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, - 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, - 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, - 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, - 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, - 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, - 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, - 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, - 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, - 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, - 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, - 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, - 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, - 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, - 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, - 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, - 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, - 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, - 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, - 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, - 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, - 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, - 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, - 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, - 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, - 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, - 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, - 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, - 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, - 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, - 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, - 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, - 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, - 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, - 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, - 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, - 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, - 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, - 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, - 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, - 0x0a, 0x0b, 0x00, 0x00, + // 1027 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0x1b, 0x45, + 0x18, 0x66, 0xbd, 0x6b, 0xc7, 0x7e, 0x53, 0x87, 0x64, 0x0a, 0x61, 0x8b, 0x50, 0x6a, 0x46, 0x95, + 0x1a, 0x7a, 0x88, 0x4a, 0x7b, 0xe1, 0xab, 0x52, 0x14, 0x3b, 0xc0, 0x02, 0x09, 0x30, 0x9b, 0xf4, + 0xd6, 0xc3, 0xd4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0x96, 0xdd, 0xd9, 0x24, 0xee, 0x81, 0x2b, 0x5c, + 0xb8, 0x23, 0x7e, 0x09, 0x3f, 0x81, 0x23, 0x3f, 0x01, 0x85, 0x3f, 0x82, 0xe6, 0x9d, 0xd9, 0x8f, + 0xc4, 0x4e, 0x53, 0x85, 0xde, 0xe6, 0xfd, 0x7e, 0xe6, 0xfd, 0x9a, 0x81, 0x7e, 0x9a, 0x45, 0x27, + 0x5c, 0x89, 0xad, 0x34, 0x93, 0x4a, 0x92, 0x6e, 0x94, 0x28, 0x91, 0x25, 0x3c, 0xa6, 0x77, 0xa1, + 0x17, 0x24, 0x13, 0x71, 0xb6, 0x27, 0x14, 0x27, 0x04, 0xbc, 0x6f, 0xc5, 0x2c, 0xf7, 0xdd, 0x81, + 0xb3, 0xd9, 0x65, 0x78, 0xa6, 0x7f, 0x3a, 0x70, 0xeb, 0xcb, 0x48, 0xc4, 0x93, 0xef, 0x53, 0x15, + 0xc9, 0x24, 0x27, 0x1f, 0x40, 0x6f, 0xc8, 0xc7, 0x47, 0xe2, 0x60, 0x96, 0x0a, 0xd4, 0xec, 0xb1, + 0x9a, 0x51, 0x49, 0xc3, 0xe8, 0xa5, 0xf0, 0xbd, 0x81, 0xb3, 0xd9, 0x67, 0x35, 0x83, 0x0c, 0x60, + 0xf9, 0x20, 0x9a, 0x8a, 0x1f, 0x0b, 0x9e, 0xa8, 0x62, 0xea, 0xb7, 0xd1, 0xba, 0xc9, 0xd2, 0x10, + 0xd0, 0x71, 0x17, 0x45, 0x78, 0x26, 0xab, 0xe0, 0xee, 0x45, 0x89, 0xdf, 0x1b, 0x38, 0x9b, 0x2e, + 0xd3, 0x47, 0xe4, 0xf0, 0x33, 0x1f, 0x2c, 0x87, 0x9f, 0x55, 0xd0, 0x97, 0x1b, 0xd0, 0x29, 0xac, + 0x04, 0xd3, 0x54, 0x66, 0x8a, 0x89, 0x3c, 0x95, 0x49, 0x8e, 0x9e, 0x76, 0xb3, 0xcc, 0x77, 0xd0, + 0xb9, 0x3e, 0xd2, 0x9f, 0x61, 0x75, 0x27, 0x96, 0xe3, 0xe3, 0x11, 0x57, 0x9c, 0x89, 0x9f, 0x0a, + 0x91, 0x2b, 0xf2, 0x0e, 0xb4, 0x31, 0x27, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x79, 0xf0, 0x5b, 0x86, + 0x8b, 0x84, 0xe6, 0xa2, 0x3d, 0x66, 0xc2, 0x63, 0x86, 0xd0, 0xdc, 0xf0, 0x88, 0x67, 0x13, 0xcc, + 0x80, 0xc7, 0x0c, 0xa1, 0x31, 0x3e, 0x8d, 0xc4, 0xa9, 0xbd, 0x36, 0x9e, 0x69, 0x00, 0x6b, 0x8d, + 0xf8, 0x16, 0xe6, 0x3a, 0x74, 0x98, 0x3c, 0x0d, 0x46, 0xb9, 0xef, 0x0c, 0xdc, 0x4d, 0x8f, 0x59, + 0x0a, 0x93, 0x2b, 0xe3, 0x62, 0x9a, 0x68, 0x51, 0x0b, 0x45, 0x35, 0x83, 0xde, 0x81, 0x36, 0x66, + 0x5a, 0xdf, 0xb2, 0xb6, 0xd5, 0x47, 0xfa, 0x8b, 0x03, 0xbd, 0x3d, 0x7e, 0x86, 0x30, 0x72, 0xf2, + 0x04, 0xba, 0xa1, 0xe2, 0xc9, 0x44, 0x03, 0xd4, 0x4a, 0xcb, 0x8f, 0x3e, 0xdc, 0x2a, 0x1b, 0x62, + 0xab, 0x52, 0xdb, 0x2a, 0x75, 0x76, 0x13, 0x95, 0xcd, 0x58, 0x65, 0xf2, 0xfe, 0xe7, 0xd0, 0xbf, + 0x20, 0xd2, 0xf1, 0x8e, 0xc5, 0xac, 0xcc, 0xea, 0xb1, 0x98, 0xe9, 0xfb, 0x9f, 0xf0, 0xb8, 0x10, + 0x98, 0x2b, 0x8f, 0x19, 0xe2, 0xb3, 0xd6, 0x27, 0x0e, 0xdd, 0x06, 0x32, 0xcc, 0x04, 0x57, 0x02, + 0x83, 0xec, 0x89, 0x3c, 0xe7, 0x2f, 0xc4, 0xd5, 0x19, 0x37, 0x59, 0x6c, 0x35, 0xb2, 0x48, 0x1f, + 0x00, 0x19, 0x89, 0x58, 0x28, 0x61, 0xfb, 0xf6, 0x15, 0x1e, 0x68, 0x58, 0x46, 0xbb, 0x5e, 0x97, + 0xdc, 0x07, 0x4f, 0x0f, 0x01, 0x06, 0x5b, 0x7e, 0x74, 0xbb, 0xce, 0x48, 0x35, 0x1f, 0x0c, 0x15, + 0x68, 0x5c, 0x3a, 0xc5, 0x0e, 0xb8, 0xf6, 0x0a, 0x0b, 0x9a, 0xe6, 0x81, 0x0d, 0xe5, 0x62, 0xa8, + 0xf5, 0x3a, 0x54, 0x73, 0xd0, 0x6c, 0xb4, 0xed, 0xf2, 0xba, 0x37, 0x8d, 0x46, 0x9f, 0x59, 0xae, + 0xee, 0xbf, 0x7d, 0x3e, 0x15, 0xd6, 0x06, 0xcf, 0x15, 0x94, 0xd6, 0xf5, 0x50, 0xb4, 0x7b, 0xdd, + 0xb3, 0x7a, 0x3f, 0xb8, 0xda, 0x3d, 0x12, 0xf4, 0x31, 0x74, 0xc2, 0xf1, 0x91, 0x98, 0x72, 0xf2, + 0x11, 0x2c, 0x21, 0x0e, 0x91, 0xdb, 0xb6, 0x7a, 0xfb, 0x52, 0x12, 0x59, 0x29, 0xa7, 0x23, 0x8b, + 0x7f, 0x21, 0xa6, 0xfb, 0xd0, 0xc1, 0xe8, 0xb9, 0xef, 0x5d, 0x76, 0x83, 0x7c, 0x66, 0xc5, 0x74, + 0x17, 0xdc, 0x43, 0x16, 0xe8, 0x71, 0x41, 0x04, 0xa5, 0x17, 0x4b, 0x69, 0xdf, 0x5f, 0xcb, 0x5c, + 0xd9, 0x6c, 0xe0, 0x59, 0xf3, 0x7e, 0x90, 0x99, 0xc2, 0xd4, 0xf7, 0x19, 0x9e, 0xe9, 0x33, 0xf0, + 0xf6, 0xe5, 0x44, 0x90, 0x15, 0x68, 0x05, 0x23, 0xeb, 0xa3, 0x15, 0x8c, 0xc8, 0x5d, 0x74, 0x6f, + 0x53, 0xd3, 0xaf, 0x41, 0x1c, 0xb2, 0x80, 0x61, 0xe0, 0x7b, 0xd0, 0x0f, 0xf2, 0xa1, 0x94, 0xd9, + 0x24, 0x4a, 0xb8, 0x92, 0x99, 0x5d, 0x9c, 0x17, 0x99, 0x74, 0x1b, 0x56, 0xb5, 0xfb, 0x50, 0x71, + 0x25, 0xca, 0xfa, 0xad, 0x43, 0x47, 0xf3, 0xaa, 0x70, 0x96, 0xc2, 0x96, 0xd7, 0x7a, 0x65, 0x05, + 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0xbb, 0x27, 0x22, 0x51, 0x8d, 0x0e, 0x40, 0x1a, 0x1d, 0xf4, 0x99, + 0x21, 0x08, 0x35, 0x57, 0xb1, 0x98, 0x57, 0x6a, 0xcc, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, + 0x4a, 0x40, 0x45, 0x5e, 0x99, 0x38, 0x57, 0x9b, 0x90, 0x8f, 0x1b, 0xeb, 0x63, 0x7e, 0x40, 0x2a, + 0x11, 0x6b, 0x2c, 0x99, 0xcd, 0xb2, 0x2d, 0x6c, 0x97, 0xaf, 0xd6, 0xfa, 0x86, 0x6f, 0xcb, 0xc4, + 0x69, 0x04, 0xfd, 0x61, 0x5c, 0xe4, 0x4a, 0x64, 0x16, 0x91, 0x5e, 0x73, 0x86, 0x51, 0xe5, 0xa7, + 0x66, 0x2c, 0x4e, 0x11, 0xb9, 0x07, 0x6d, 0x8d, 0xd4, 0xf4, 0xe6, 0xfc, 0x35, 0x8c, 0x90, 0x3e, + 0x85, 0xee, 0x4e, 0x18, 0x7c, 0x95, 0xc9, 0x22, 0x5d, 0xd8, 0x79, 0xe5, 0xeb, 0xd3, 0x9a, 0x7f, + 0x7d, 0xdc, 0xb9, 0xd7, 0xc7, 0xab, 0x5e, 0x1f, 0x1a, 0xc2, 0x9a, 0x59, 0x09, 0x7a, 0x24, 0x6e, + 0xb2, 0x11, 0xca, 0xa7, 0xc1, 0x6d, 0x3c, 0x0d, 0x21, 0xac, 0x99, 0xc9, 0x7f, 0x93, 0x4e, 0xff, + 0x68, 0xc1, 0x1a, 0x13, 0x79, 0xf4, 0x52, 0x04, 0x49, 0xae, 0xb2, 0x62, 0xac, 0x07, 0x5c, 0xdb, + 0x7f, 0x23, 0x9f, 0xdb, 0x6c, 0xbb, 0xcc, 0x10, 0xaf, 0xd3, 0x4c, 0xe4, 0x21, 0x2c, 0x5f, 0x1e, + 0x80, 0x79, 0xd5, 0xa6, 0x0a, 0x79, 0x08, 0x4b, 0xa1, 0x2c, 0xb2, 0xb1, 0x28, 0xc7, 0xbb, 0xb1, + 0x74, 0x0c, 0x32, 0x23, 0x66, 0xa5, 0x5a, 0xa3, 0x95, 0xda, 0xaf, 0x6e, 0x25, 0xf2, 0xe4, 0x52, + 0x2b, 0xf9, 0x1d, 0x34, 0x78, 0xaf, 0x36, 0xb8, 0x20, 0x66, 0x17, 0xb5, 0xe9, 0xaf, 0x0e, 0xdc, + 0x6a, 0x42, 0x78, 0xad, 0xd9, 0xa8, 0x2a, 0xd2, 0x5a, 0x58, 0x11, 0x77, 0x51, 0x45, 0xbc, 0xba, + 0x22, 0xf5, 0x2b, 0xd7, 0x6e, 0xbe, 0x72, 0xc7, 0x70, 0x67, 0xae, 0x4c, 0x43, 0x39, 0x4d, 0x75, + 0x3f, 0xfc, 0x8f, 0x72, 0xe9, 0xad, 0x91, 0x65, 0xb6, 0x50, 0x3d, 0x66, 0x08, 0xfa, 0x29, 0xbc, + 0x1b, 0x0a, 0xd5, 0x28, 0x52, 0xd9, 0x6d, 0x03, 0x70, 0xf7, 0xc5, 0xe9, 0x15, 0xd7, 0xd7, 0x22, + 0xfa, 0x05, 0xf8, 0x87, 0xe9, 0x84, 0x2b, 0x71, 0x23, 0xeb, 0x1d, 0xe8, 0x1e, 0xc8, 0x54, 0xc6, + 0xf2, 0xc5, 0xec, 0x9a, 0xa9, 0xf7, 0x61, 0xc9, 0xac, 0x48, 0xf3, 0xf1, 0xe9, 0xb1, 0x92, 0xa4, + 0xb7, 0x75, 0x43, 0x8f, 0x79, 0x3c, 0x2e, 0x62, 0x0d, 0x43, 0xff, 0x80, 0xf2, 0x9d, 0xd5, 0xbf, + 0xce, 0x37, 0x9c, 0xbf, 0xcf, 0x37, 0x9c, 0x7f, 0xce, 0x37, 0x9c, 0xdf, 0xff, 0xdd, 0x78, 0xeb, + 0x79, 0x07, 0x7f, 0xbe, 0x8f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x39, 0x2f, 0x93, 0x68, 0x0a, + 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 23bb4886c..408c065e5 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -24,7 +24,7 @@ message BlockDataRequest { string Index = 1; string Field = 2; string View = 5; - uint64 Slice = 4; + uint64 Shard = 4; uint64 Block = 3; } @@ -37,13 +37,13 @@ message Cache { repeated uint64 IDs = 1; } -message MaxSlices { +message MaxShards { map Standard = 1; } -message CreateSliceMessage { +message CreateShardMessage { string Index = 1; - uint64 Slice = 2; + uint64 Shard = 2; } message DeleteIndexMessage { @@ -105,7 +105,7 @@ message NodeEventMessage { message NodeStatus { Node Node = 1; - MaxSlices MaxSlices = 2; + MaxShards MaxShards = 2; Schema Schema = 3; } @@ -148,7 +148,7 @@ message ResizeSource { string Index = 2; string Field = 3; string View = 4; - uint64 Slice = 5; + uint64 Shard = 5; } message ResizeInstructionComplete { diff --git a/internal/public.pb.go b/internal/public.pb.go index 3cb6fa270..cd1927f06 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,5 +1,6 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-gogo. // source: public.proto +// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import encoding_binary "encoding/binary" - import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -268,7 +267,7 @@ func (m *AttrMap) GetAttrs() []*Attr { type QueryRequest struct { Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + Shards []uint64 `protobuf:"varint,2,rep,packed,name=Shards" json:"Shards,omitempty"` ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` @@ -287,9 +286,9 @@ func (m *QueryRequest) GetQuery() string { return "" } -func (m *QueryRequest) GetSlices() []uint64 { +func (m *QueryRequest) GetShards() []uint64 { if m != nil { - return m.Slices + return m.Shards } return nil } @@ -413,7 +412,7 @@ func (m *QueryResult) GetChanged() bool { type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` RowKeys []string `protobuf:"bytes,7,rep,name=RowKeys" json:"RowKeys,omitempty"` @@ -440,9 +439,9 @@ func (m *ImportRequest) GetField() string { return "" } -func (m *ImportRequest) GetSlice() uint64 { +func (m *ImportRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -485,7 +484,7 @@ func (m *ImportRequest) GetTimestamps() []int64 { type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Shard uint64 `protobuf:"varint,3,opt,name=Shard,proto3" json:"Shard,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` @@ -510,9 +509,9 @@ func (m *ImportValueRequest) GetField() string { return "" } -func (m *ImportValueRequest) GetSlice() uint64 { +func (m *ImportValueRequest) GetShard() uint64 { if m != nil { - return m.Slice + return m.Shard } return 0 } @@ -800,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) - i += 8 + i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) } return i, nil } @@ -857,10 +855,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Query))) i += copy(dAtA[i:], m.Query) } - if len(m.Slices) > 0 { - dAtA4 := make([]byte, len(m.Slices)*10) + if len(m.Shards) > 0 { + dAtA4 := make([]byte, len(m.Shards)*10) var j3 int - for _, num := range m.Slices { + for _, num := range m.Shards { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1062,10 +1060,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { dAtA8 := make([]byte, len(m.RowIDs)*10) @@ -1179,10 +1177,10 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) } - if m.Slice != 0 { + if m.Shard != 0 { dAtA[i] = 0x18 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) + i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { dAtA14 := make([]byte, len(m.ColumnIDs)*10) @@ -1237,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1378,9 +1394,9 @@ func (m *QueryRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if len(m.Slices) > 0 { + if len(m.Shards) > 0 { l = 0 - for _, e := range m.Slices { + for _, e := range m.Shards { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1462,8 +1478,8 @@ func (m *ImportRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) } if len(m.RowIDs) > 0 { l = 0 @@ -1512,8 +1528,8 @@ func (m *ImportValueRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.Slice != 0 { - n += 1 + sovPublic(uint64(m.Slice)) + if m.Shard != 0 { + n += 1 + sovPublic(uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { l = 0 @@ -2317,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + v = uint64(dAtA[iNdEx-8]) + v |= uint64(dAtA[iNdEx-7]) << 8 + v |= uint64(dAtA[iNdEx-6]) << 16 + v |= uint64(dAtA[iNdEx-5]) << 24 + v |= uint64(dAtA[iNdEx-4]) << 32 + v |= uint64(dAtA[iNdEx-3]) << 40 + v |= uint64(dAtA[iNdEx-2]) << 48 + v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex @@ -2497,7 +2520,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -2537,10 +2560,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Slices = append(m.Slices, v) + m.Shards = append(m.Shards, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shards", wireType) } case 3: if wireType != 0 { @@ -3078,9 +3101,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3090,7 +3113,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3449,9 +3472,9 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) } - m.Slice = 0 + m.Shard = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3461,7 +3484,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift + m.Shard |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3748,49 +3771,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 699 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, - 0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e, - 0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54, - 0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d, - 0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73, - 0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26, - 0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0, - 0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe, - 0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46, - 0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee, - 0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3, - 0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5, - 0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12, - 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b, - 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98, - 0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c, - 0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf, - 0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6, - 0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda, - 0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa, - 0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3, - 0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57, - 0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2, - 0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd, - 0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10, - 0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3, - 0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95, - 0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1, - 0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59, - 0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5, - 0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab, - 0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb, - 0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94, - 0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82, - 0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9, - 0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9, - 0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7, - 0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1, - 0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f, - 0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d, - 0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66, - 0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90, - 0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00, - 0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00, + // 701 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84, + 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b, + 0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0, + 0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6, + 0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5, + 0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8, + 0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41, + 0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69, + 0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b, + 0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93, + 0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1, + 0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3, + 0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c, + 0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7, + 0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22, + 0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb, + 0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49, + 0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75, + 0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88, + 0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c, + 0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d, + 0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97, + 0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd, + 0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44, + 0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58, + 0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd, + 0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1, + 0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85, + 0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2, + 0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6, + 0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b, + 0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85, + 0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84, + 0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64, + 0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66, + 0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, + 0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9, + 0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17, + 0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09, + 0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e, + 0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8, + 0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc, + 0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index cd34b88ff..04c98d070 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -46,7 +46,7 @@ message AttrMap { message QueryRequest { string Query = 1; - repeated uint64 Slices = 2; + repeated uint64 Shards = 2; bool ColumnAttrs = 3; bool Remote = 5; bool ExcludeRowAttrs = 6; @@ -71,7 +71,7 @@ message QueryResult { message ImportRequest { string Index = 1; string Field = 2; - uint64 Slice = 3; + uint64 Shard = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; repeated string RowKeys = 7; @@ -82,7 +82,7 @@ message ImportRequest { message ImportValueRequest { string Index = 1; string Field = 2; - uint64 Slice = 3; + uint64 Shard = 3; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; diff --git a/iterator.go b/iterator.go index 473e862dd..d7526b779 100644 --- a/iterator.go +++ b/iterator.go @@ -184,11 +184,11 @@ func NewRoaringIterator(itr *roaring.Iterator) *RoaringIterator { // Seek moves the cursor to a pair matching bseek/pseek. // If the pair is not found then it moves to the next pair. func (itr *RoaringIterator) Seek(bseek, pseek uint64) { - itr.itr.Seek((bseek * SliceWidth) + pseek) + itr.itr.Seek((bseek * ShardWidth) + pseek) } // Next returns the next column/row ID pair. func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) { v, eof := itr.itr.Next() - return v / SliceWidth, v % SliceWidth, eof + return v / ShardWidth, v % ShardWidth, eof } diff --git a/pilosa.go b/pilosa.go index bff167a7d..9505bf513 100644 --- a/pilosa.go +++ b/pilosa.go @@ -56,7 +56,7 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c1291d1f7..d29f1226f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -471,10 +471,10 @@ func TestBitmap_Difference(t *testing.T) { } func TestBitmap_Difference2(t *testing.T) { - bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.SliceWidth+5, pilosa.SliceWidth+7) - bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.SliceWidth+1) + bm0 := roaring.NewFileBitmap(0, 1, 2, 131072, 262144, pilosa.ShardWidth+5, pilosa.ShardWidth+7) + bm1 := roaring.NewFileBitmap(2, 3, 100000, 262144, 2*pilosa.ShardWidth+1) result := bm0.Difference(bm1) - if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.SliceWidth + 5, pilosa.SliceWidth + 7}) { + if !reflect.DeepEqual(result.Slice(), []uint64{0, 1, 131072, pilosa.ShardWidth + 5, pilosa.ShardWidth + 7}) { t.Fatalf("unexpected : %v", result.Slice()) } } @@ -1161,7 +1161,7 @@ func BenchmarkContainerLinear(b *testing.B) { bm := roaring.NewFileBitmap() for row := uint64(1); row < NumRows; row++ { for col := uint64(1); col < NumColums; col++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1172,7 +1172,7 @@ func BenchmarkContainerReverse(b *testing.B) { bm := roaring.NewFileBitmap() for row := NumRows - 1; row >= 1; row-- { for col := NumColums - 1; col >= 1; col-- { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1183,7 +1183,7 @@ func BenchmarkContainerColumn(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < NumRows; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1196,8 +1196,8 @@ func BenchmarkContainerOutsideIn(b *testing.B) { for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row < middle; row++ { - bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add(row*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((NumRows-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1209,8 +1209,8 @@ func BenchmarkContainerInsideOut(b *testing.B) { bm := roaring.NewFileBitmap() for col := uint64(1); col < NumColums; col++ { for row := uint64(1); row <= middle; row++ { - bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) - bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add((middle+row)*pilosa.ShardWidth + (col * MaxContainerVal)) + bm.Add((middle-row)*pilosa.ShardWidth + (col * MaxContainerVal)) } } } @@ -1219,7 +1219,7 @@ func BenchmarkContainerInsideOut(b *testing.B) { func BenchmarkSliceAscending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(0); col < pilosa.SliceWidth; col++ { + for col := uint64(0); col < pilosa.ShardWidth; col++ { bm.Add(col) } } @@ -1228,7 +1228,7 @@ func BenchmarkSliceAscending(b *testing.B) { func BenchmarkSliceDescending(b *testing.B) { for n := 0; n < b.N; n++ { bm := roaring.NewFileBitmap() - for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { + for col := uint64(pilosa.ShardWidth); col > uint64(0); col-- { bm.Add(col) } } diff --git a/row.go b/row.go index a59bd73e0..cbfa6b270 100644 --- a/row.go +++ b/row.go @@ -157,12 +157,12 @@ func (r *Row) Difference(other *Row) *Row { // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { - return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i) + return r.createSegmentIfNotExists(i / ShardWidth).SetBit(i) } // ClearBit clears the i-th column of the row. func (r *Row) ClearBit(i uint64) (changed bool) { - s := r.segment(i / SliceWidth) + s := r.segment(i / ShardWidth) if s == nil { return false } @@ -174,24 +174,24 @@ func (r *Row) Segments() []RowSegment { return r.segments } -// segment returns a segment for a given slice. +// segment returns a segment for a given shard. // Returns nil if segment does not exist. -func (r *Row) segment(slice uint64) *RowSegment { +func (r *Row) segment(shard uint64) *RowSegment { if i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice - }); i < len(r.segments) && r.segments[i].slice == slice { + return r.segments[i].shard >= shard + }); i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } return nil } -func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment { +func (r *Row) createSegmentIfNotExists(shard uint64) *RowSegment { i := sort.Search(len(r.segments), func(i int) bool { - return r.segments[i].slice >= slice + return r.segments[i].shard >= shard }) // Return exact match. - if i < len(r.segments) && r.segments[i].slice == slice { + if i < len(r.segments) && r.segments[i].shard == shard { return &r.segments[i] } @@ -202,7 +202,7 @@ func (r *Row) createSegmentIfNotExists(slice uint64) *RowSegment { } r.segments[i] = RowSegment{ data: *roaring.NewBitmap(), - slice: slice, + shard: shard, writable: true, } @@ -218,7 +218,7 @@ func (r *Row) InvalidateCount() { // IncrementCount increments the row cached counter, note this is an optimization that assumes that the caller is aware the size increased. func (r *Row) IncrementCount(i uint64) { - seg := r.segment(i / SliceWidth) + seg := r.segment(i / ShardWidth) if seg != nil { seg.n++ } @@ -227,7 +227,7 @@ func (r *Row) IncrementCount(i uint64) { // DecrementCount decrements the row cached counter. func (r *Row) DecrementCount(i uint64) { - seg := r.segment(i / SliceWidth) + seg := r.segment(i / ShardWidth) if seg != nil { if seg.n > 0 { seg.n-- @@ -308,10 +308,10 @@ func Union(rows []*Row) *Row { // RowSegment holds a subset of a row. // This could point to a mmapped roaring bitmap or an in-memory bitmap. The -// width of the segment will always match the slice width. +// width of the segment will always match the shard width. type RowSegment struct { - // Slice this segment belongs to - slice uint64 + // Shard this segment belongs to + shard uint64 // Underlying raw bitmap implementation. // This is an mmapped bitmap if writable is false. Otherwise @@ -345,7 +345,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -356,7 +356,7 @@ func (s *RowSegment) Union(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -367,7 +367,7 @@ func (s *RowSegment) Difference(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -378,7 +378,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment { return &RowSegment{ data: *data, - slice: s.slice, + shard: s.shard, n: data.Count(), } } @@ -464,15 +464,15 @@ func (itr *mergeSegmentIterator) next() (s0, s1 *RowSegment) { } // Otherwise determine which is first. - if s0.slice < s1.slice { + if s0.shard < s1.shard { itr.a0 = itr.a0[1:] return s0, nil - } else if s0.slice > s1.slice { + } else if s0.shard > s1.shard { itr.a1 = itr.a1[1:] return s1, nil } - // Return both if slices are equal. + // Return both if shards are equal. itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:] return s0, s1 } diff --git a/row_test.go b/row_test.go index bc1cc0c68..7f1279ceb 100644 --- a/row_test.go +++ b/row_test.go @@ -30,7 +30,7 @@ func TestRow_Merge(t *testing.T) { exp uint64 }{ { - r1: pilosa.NewRow(1, 2, 3, SliceWidth+1, 2*SliceWidth), + r1: pilosa.NewRow(1, 2, 3, ShardWidth+1, 2*ShardWidth), r2: pilosa.NewRow(3, 4, 5), exp: 7, }, @@ -56,9 +56,9 @@ func TestRow_Merge(t *testing.T) { // Ensure a row can Xor'ed func TestRow_Xor(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth, 2 * ShardWidth} res := r1.Xor(r2) if res.Count() != 3 { @@ -78,9 +78,9 @@ func TestRow_Xor(t *testing.T) { } func TestRow_Union_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{0, 1, ShardWidth, 2 * ShardWidth} res := r1.Union(r2) if res.Count() != 4 { @@ -99,9 +99,9 @@ func TestRow_Union_Segment(t *testing.T) { } func TestRow_Difference_Segment(t *testing.T) { - r1 := pilosa.NewRow(0, 1, SliceWidth) - r2 := pilosa.NewRow(0, 2*SliceWidth) - exp := []uint64{1, SliceWidth} + r1 := pilosa.NewRow(0, 1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + exp := []uint64{1, ShardWidth} res := r1.Difference(r2) if res.Count() != 2 { diff --git a/server.go b/server.go index c07720a54..b04661f53 100644 --- a/server.go +++ b/server.go @@ -412,12 +412,12 @@ func (s *Server) monitorAntiEntropy() { // ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { - case *internal.CreateSliceMessage: + case *internal.CreateShardMessage: idx := s.holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - idx.SetRemoteMaxSlice(obj.Slice) + idx.SetRemoteMaxShard(obj.Shard) case *internal.CreateIndexMessage: opt := IndexOptions{} _, err := s.holder.CreateIndex(obj.Index, opt) @@ -537,7 +537,7 @@ func (s *Server) Node() *Node { // where a node fails to receive a Broadcast message, or // when a new (empty) node needs to get in sync with the // rest of the cluster, two things are shared via gossip: -// - MaxSlice by Index +// - MaxShard by Index // - Schema // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalStatus() (proto.Message, error) { @@ -550,7 +550,7 @@ func (s *Server) LocalStatus() (proto.Message, error) { ns := internal.NodeStatus{ Node: EncodeNode(s.cluster.Node), - MaxSlices: s.holder.EncodeMaxSlices(), + MaxShards: s.holder.EncodeMaxShards(), Schema: s.holder.EncodeSchema(), } @@ -593,19 +593,19 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return errors.Wrap(err, "applying schema") } - // Sync maxSlices. - oldmaxslices := s.holder.MaxSlices() - for index, newMax := range ns.MaxSlices.Standard { + // Sync maxShards. + oldmaxshards := s.holder.MaxShards() + for index, newMax := range ns.MaxShards.Standard { localIndex := s.holder.Index(index) // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation + // indexes should be created and synced prior to shard creation if localIndex == nil { s.logger.Printf("Local Index not found: %s", index) continue } - if newMax > oldmaxslices[index] { - oldmaxslices[index] = newMax - localIndex.SetRemoteMaxSlice(newMax) + if newMax > oldmaxshards[index] { + oldmaxshards[index] = newMax + localIndex.SetRemoteMaxShard(newMax) } } diff --git a/server/cluster_test.go b/server/cluster_test.go index d8c2d7802..aff676ae0 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -95,22 +95,22 @@ func TestMain_SendReceiveMessage(t *testing.T) { // We have to wait for the broadcast message to be sent before checking state. time.Sleep(1 * time.Second) - // Make sure node0 knows about the latest MaxSlice. - maxSlices0, err := client0.MaxSliceByIndex(context.Background()) + // Make sure node0 knows about the latest MaxShard. + maxShards0, err := client0.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices0["i"] != 2 { - t.Fatalf("unexpected maxSlice on node0: %d", maxSlices0["i"]) + if maxShards0["i"] != 2 { + t.Fatalf("unexpected maxShard on node0: %d", maxShards0["i"]) } - // Make sure node1 knows about the latest MaxSlice. - maxSlices1, err := client1.MaxSliceByIndex(context.Background()) + // Make sure node1 knows about the latest MaxShard. + maxShards1, err := client1.MaxShardByIndex(context.Background()) if err != nil { t.Fatal(err) } - if maxSlices1["i"] != 2 { - t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) + if maxShards1["i"] != 2 { + t.Fatalf("unexpected maxShard on node1: %d", maxShards1["i"]) } } @@ -181,7 +181,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("ContinuousSlices", func(t *testing.T) { + t.Run("ContinuousShards", func(t *testing.T) { // Configure node0 m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() @@ -222,7 +222,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) - t.Run("SkippedSlice", func(t *testing.T) { + t.Run("SkippedShard", func(t *testing.T) { // Configure node0 m0 := test.MustRunCluster(t, 1)[0] defer m0.Close() @@ -239,7 +239,7 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } - // Write data on first node. Note that no data is placed on slice 1. + // Write data on first node. Note that no data is placed on shard 1. if _, err := m0.Query("i", "", ` Set(1, f=1) Set(2400000, f=1) @@ -390,7 +390,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.ShardWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { diff --git a/server/handler_test.go b/server/handler_test.go index 2981f375c..169065971 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -55,7 +55,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { + } else if body := w.Body.String(); body != fmt.Sprintf("{\"shardWidth\":%d}\n", pilosa.ShardWidth) { t.Fatalf("unexpected body: %s", body) } }) @@ -112,19 +112,19 @@ func TestHandler_Endpoints(t *testing.T) { // TODO need to test aborting a cluster resize job. this may not be the right place }) - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) - hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) - hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.ShardWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.ShardWidth)+4) hldr.SetBit("i0", "f0", 31, 1) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) - hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.ShardWidth)+8) - t.Run("Max Slice", func(t *testing.T) { + t.Run("Max Shard", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/shards/max", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { @@ -132,9 +132,9 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Slices args", func(t *testing.T) { + t.Run("Shards args", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { @@ -142,11 +142,11 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Slices args protobuf", func(t *testing.T) { + t.Run("Shards args protobuf", func(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ Query: "Count(Row(f0=30))", - Slices: []uint64{0, 1}, + Shards: []uint64{0, 1}, }) if err != nil { t.Fatal(err) @@ -169,17 +169,17 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Query args error", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=a,b", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { + } else if body := w.Body.String(); body != `{"error":"invalid shard argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) } }) t.Run("Query params err", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?shards=0,1&db=sample", strings.NewReader("Count(Row(f0=30))"))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { @@ -217,9 +217,9 @@ func TestHandler_Endpoints(t *testing.T) { }) f0 := i0.Field("f0") - if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil { + if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) } else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil { t.Fatal(err) @@ -249,7 +249,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -285,7 +285,7 @@ func TestHandler_Endpoints(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) { + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) @@ -301,7 +301,7 @@ func TestHandler_Endpoints(t *testing.T) { if a := resp.ColumnAttrSets; len(a) != 2 { t.Fatalf("unexpected column attributes length: %d", len(a)) - } else if a[0].ID != pilosa.SliceWidth+1 { + } else if a[0].ID != pilosa.ShardWidth+1 { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected column attr length: %d", len(a)) @@ -376,7 +376,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Err Parse", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?shards=0,1", strings.NewReader("bad_fn("))) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { @@ -507,7 +507,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Fragment Nodes", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil) + r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -520,7 +520,7 @@ func TestHandler_Endpoints(t *testing.T) { // invalid argument should return BadRequest w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) @@ -528,7 +528,7 @@ func TestHandler_Endpoints(t *testing.T) { // index is required w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) diff --git a/stats_test.go b/stats_test.go index 932672e3a..3452f2b8c 100644 --- a/stats_test.go +++ b/stats_test.go @@ -39,43 +39,43 @@ func TestMultiStatClient_Expvar(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, SliceWidth) - hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) hldr.ClearBit("d", "f", 0, 1) - if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Gauge creates a unique key, subsequent Gauge calls will overwrite hldr.Stats.Gauge("g", 5, 1.0) hldr.Stats.Gauge("g", 8, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Set creates a unique key, subsequent sets will overwrite hldr.Stats.Set("s", "4", 1.0) hldr.Stats.Set("s", "7", 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Record timing duration and a uniquely Set key/value dur, _ := time.ParseDuration("123us") hldr.Stats.Timing("tt", dur, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Expvar histogram is implemented as a gauge hldr.Stats.Histogram("hh", 3, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"shard:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "shard:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } @@ -91,8 +91,8 @@ func TestStatsCount_TopN(t *testing.T) { hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, SliceWidth) - hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.SetBit("d", "f", 0, ShardWidth) + hldr.SetBit("d", "f", 0, ShardWidth+2) // Execute query. called := false diff --git a/test/fragment.go b/test/fragment.go index 6faf5d72d..90d2fa291 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -18,8 +18,8 @@ import ( "github.com/pilosa/pilosa" ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth +// ShardWidth is a helper reference to use when testing. +const ShardWidth = pilosa.ShardWidth // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { diff --git a/test/handler.go b/test/handler.go index 048d9b883..944cfb8ee 100644 --- a/test/handler.go +++ b/test/handler.go @@ -64,13 +64,13 @@ func MustNewHandler(opts ...http.HandlerOption) *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) + ExecuteFn func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, index, query, slices, opt) +func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return c.ExecuteFn(ctx, index, query, shards, opt) } // Server represents a test wrapper for httptest.Server. diff --git a/test/holder.go b/test/holder.go index 648d910ed..3d09c9e34 100644 --- a/test/holder.go +++ b/test/holder.go @@ -90,7 +90,7 @@ func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { } // MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { +func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, shard uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { @@ -100,7 +100,7 @@ func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, if err != nil { panic(err) } - frag, err := v.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { panic(err) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 5e19a83b9..7843ed443 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -119,9 +119,9 @@ func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) erro func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / SliceWidth - nodes := c0.sliceNodes(index, slice) + c0 := t.Clusters[0] // use the first node's cluster to determine shard location. + shard := colID / ShardWidth + nodes := c0.shardNodes(index, shard) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -355,7 +355,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) + // basically need to mimic this: client.RetrieveShardFromURI(context.Background(), src.Index, src.Field, src.View, src.Shard, srcURI) instrNode := DecodeNode(instr.Node) destCluster := t.clusterByID(instrNode.ID) @@ -368,14 +368,14 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi srcNode := DecodeNode(src.Node) srcCluster := t.clusterByID(srcNode.ID) - srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice) - destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Slice) + srcFragment := srcCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard) + destFragment := destCluster.holder.Fragment(src.Index, src.Field, src.View, src.Shard) if destFragment == nil { // Create fragment on destination if it doesn't exist. f := destCluster.holder.Field(src.Index, src.Field) v := f.View(src.View) var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + destFragment, err = v.CreateFragmentIfNotExists(src.Shard) if err != nil { return err } diff --git a/view.go b/view.go index 732ecd485..25b90cfb4 100644 --- a/view.go +++ b/view.go @@ -49,13 +49,13 @@ type View struct { cacheSize uint32 - // Fragments by slice. + // Fragments by shard. cacheType string // passed in by field fragments map[uint64]*Fragment - // maxSlice maintains this view's max slice in order to - // prevent sending multiple `CreateSliceMessage` messages - maxSlice uint64 + // maxShard maintains this view's max shard in order to + // prevent sending multiple `CreateShardMessage` messages + maxShard uint64 broadcaster Broadcaster stats StatsClient @@ -132,17 +132,17 @@ func (v *View) openFragments() error { } // Parse filename into integer. - slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { continue } - frag := v.newFragment(v.fragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) + return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.slice] = frag + v.fragments[frag.shard] = frag } return nil @@ -164,15 +164,15 @@ func (v *View) close() error { return nil } -// calculateMaxSlice returns the max slice in the view. -func (v *View) calculateMaxSlice() uint64 { +// calculateMaxShard returns the max shard in the view. +func (v *View) calculateMaxShard() uint64 { v.mu.RLock() defer v.mu.RUnlock() var max uint64 - for slice := range v.fragments { - if slice > max { - max = slice + for shard := range v.fragments { + if shard > max { + max = shard } } @@ -180,18 +180,18 @@ func (v *View) calculateMaxSlice() uint64 { } // fragmentPath returns the path to a fragment in the view. -func (v *View) fragmentPath(slice uint64) string { - return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) +func (v *View) fragmentPath(shard uint64) string { + return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10)) } -// Fragment returns a fragment in the view by slice. -func (v *View) Fragment(slice uint64) *Fragment { +// Fragment returns a fragment in the view by shard. +func (v *View) Fragment(shard uint64) *Fragment { v.mu.RLock() defer v.mu.RUnlock() - return v.fragment(slice) + return v.fragment(shard) } -func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } +func (v *View) fragment(shard uint64) *Fragment { return v.fragments[shard] } // allFragments returns a list of all fragments in the view. func (v *View) allFragments() []*Fragment { @@ -212,64 +212,64 @@ func (v *View) recalculateCaches() { } } -// CreateFragmentIfNotExists returns a fragment in the view by slice. -func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { +// CreateFragmentIfNotExists returns a fragment in the view by shard. +func (v *View) CreateFragmentIfNotExists(shard uint64) (*Fragment, error) { v.mu.Lock() defer v.mu.Unlock() - return v.createFragmentIfNotExists(slice) + return v.createFragmentIfNotExists(shard) } -func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { +func (v *View) createFragmentIfNotExists(shard uint64) (*Fragment, error) { // Find fragment in cache first. - if frag := v.fragments[slice]; frag != nil { + if frag := v.fragments[shard]; frag != nil { return frag, nil } // Initialize and open fragment. - frag := v.newFragment(v.fragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } frag.RowAttrStore = v.RowAttrStore - // Broadcast a message that a new max slice was just created. - if slice > v.maxSlice { - v.maxSlice = slice + // Broadcast a message that a new max shard was just created. + if shard > v.maxShard { + v.maxShard = shard - // Send the create slice message to all nodes. + // Send the create shard message to all nodes. err := v.broadcaster.SendSync( - &internal.CreateSliceMessage{ + &internal.CreateShardMessage{ Index: v.index, - Slice: slice, + Shard: shard, }) if err != nil { - return nil, errors.Wrap(err, "sending createslice message") + return nil, errors.Wrap(err, "sending createshard message") } } // Save to lookup. - v.fragments[slice] = frag + v.fragments[shard] = frag return frag, nil } -func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.index, v.field, v.name, slice) +func (v *View) newFragment(path string, shard uint64) *Fragment { + frag := NewFragment(path, v.index, v.field, v.name, shard) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.Logger - frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) + frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) return frag } // deleteFragment removes the fragment from the view. -func (v *View) deleteFragment(slice uint64) error { +func (v *View) deleteFragment(shard uint64) error { - fragment := v.fragments[slice] + fragment := v.fragments[shard] if fragment == nil { return ErrFragmentNotFound } - v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -283,15 +283,15 @@ func (v *View) deleteFragment(slice uint64) error { // Delete fragment cache file. if err := os.Remove(fragment.cachePath()); err != nil { - v.Logger.Printf("no cache file to delete for slice %d", slice) + v.Logger.Printf("no cache file to delete for shard %d", shard) } - delete(v.fragments, slice) + delete(v.fragments, shard) return nil } -// row returns a row for a slice of the view. +// row returns a row for a shard of the view. func (v *View) row(rowID uint64) *Row { row := NewRow() for _, frag := range v.allFragments() { @@ -307,8 +307,8 @@ func (v *View) row(rowID uint64) *Row { // setBit sets a bit within the view. func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } @@ -317,8 +317,8 @@ func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { // clearBit clears a bit within the view. func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } @@ -327,8 +327,8 @@ func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { // value uses a column of bits to read a multi-bit value. func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return value, exists, err } @@ -337,8 +337,8 @@ func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, // setValue uses a column of bits to set a multi-bit value. func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { - slice := columnID / SliceWidth - frag, err := v.CreateFragmentIfNotExists(slice) + shard := columnID / ShardWidth + frag, err := v.CreateFragmentIfNotExists(shard) if err != nil { return changed, err } diff --git a/view_internal_test.go b/view_internal_test.go index d0e8bfdd1..a592330cb 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -39,27 +39,27 @@ func TestView_DeleteFragment(t *testing.T) { v := mustOpenView("i", "f", "v") defer v.close() - slice := uint64(9) + shard := uint64(9) // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) + fragment, err := v.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } else if fragment == nil { t.Fatal("expected fragment") } - err = v.deleteFragment(slice) + err = v.deleteFragment(shard) if err != nil { t.Fatal(err) } - if v.Fragment(slice) != nil { + if v.Fragment(shard) != nil { t.Fatal("fragment still exists in view") } - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) + // Recreate fragment with same shard, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(shard) if err != nil { t.Fatal(err) } else if fragment == fragment2 { From 4054f33ad5fd0d870efaacfe0740a1188d99a189 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 28 Jun 2018 14:40:23 -0500 Subject: [PATCH 09/10] exported DefaultCacheSize, not sure why i had unexproted --- field.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/field.go b/field.go index 32d9625be..f4100fa7e 100644 --- a/field.go +++ b/field.go @@ -38,7 +38,7 @@ const ( DefaultCacheType = CacheTypeRanked // Default ranked field cache - defaultCacheSize = 50000 + DefaultCacheSize = 50000 bitsPerWord = 32 << (^uint(0) >> 63) // either 32 or 64 maxInt = 1<<(bitsPerWord-1) - 1 // either 1<<31 - 1 or 1<<63 - 1 From 86b24e38245b68bf1e248acbe1b9ec90aa22e508 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 28 Jun 2018 15:51:51 -0500 Subject: [PATCH 10/10] Remove API.Broadcaster --- api.go | 28 +++++++++++----------------- test/handler.go | 25 ------------------------- 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/api.go b/api.go index 296304127..0f19ef19d 100644 --- a/api.go +++ b/api.go @@ -35,10 +35,9 @@ import ( // API provides the top level programmatic interface to Pilosa. It is usually // wrapped by a handler which provides an external interface (e.g. HTTP). type API struct { - Holder *Holder - Broadcaster Broadcaster - Cluster *Cluster - server *Server + Holder *Holder + Cluster *Cluster + server *Server } // APIOption is a functional option type for pilosa.API @@ -48,7 +47,6 @@ func OptAPIServer(s *Server) APIOption { return func(a *API) error { a.server = s a.Holder = s.holder - a.Broadcaster = s a.Cluster = s.cluster return nil } @@ -56,11 +54,7 @@ func OptAPIServer(s *Server) APIOption { // NewAPI returns a new API instance. func NewAPI(opts ...APIOption) (*API, error) { - api := &API{ - Broadcaster: NopBroadcaster, - //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop - //StatusHandler: NopStatusHandler, // TODO: implement the nop - } + api := &API{} for _, opt := range opts { err := opt(api) @@ -190,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "creating index") } // Send the create index message to all nodes. - err = api.Broadcaster.SendSync( + err = api.server.SendSync( &internal.CreateIndexMessage{ Index: indexName, Meta: options.Encode(), @@ -229,7 +223,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return errors.Wrap(err, "deleting index") } // Send the delete index message to all nodes. - err = api.Broadcaster.SendSync( + err = api.server.SendSync( &internal.DeleteIndexMessage{ Index: indexName, }) @@ -269,7 +263,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str } // Send the create field message to all nodes. - err = api.Broadcaster.SendSync( + err = api.server.SendSync( &internal.CreateFieldMessage{ Index: indexName, Field: fieldName, @@ -303,7 +297,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str } // Send the delete field message to all nodes. - err := api.Broadcaster.SendSync( + err := api.server.SendSync( &internal.DeleteFieldMessage{ Index: indexName, Field: fieldName, @@ -476,7 +470,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return errors.Wrap(err, "validating api method") } - err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) + err := api.server.SendSync(&internal.RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") } @@ -559,7 +553,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri } // Send the delete view message to all nodes. - err := api.Broadcaster.SendSync( + err := api.server.SendSync( &internal.DeleteViewMessage{ Index: indexName, Field: fieldName, @@ -753,7 +747,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode } // Send the set-coordinator message to new node. - err = api.Broadcaster.SendTo( + err = api.server.SendTo( newNode, &internal.SetCoordinatorMessage{ New: EncodeNode(newNode), diff --git a/test/handler.go b/test/handler.go index 944cfb8ee..cc3044a85 100644 --- a/test/handler.go +++ b/test/handler.go @@ -36,31 +36,6 @@ type Handler struct { Executor HandlerExecutor } -// NewHandler returns a new instance of Handler. -func NewHandler(opts ...http.HandlerOption) (*Handler, error) { - handler, err := http.NewHandler(opts...) - if err != nil { - return nil, err - } - h := &Handler{ - Handler: handler, - } - - // Handler test messages can no-op. - h.API.Broadcaster = pilosa.NopBroadcaster - - return h, nil -} - -// MustNewHandler returns a new instance of Handler. -func MustNewHandler(opts ...http.HandlerOption) *Handler { - h, err := NewHandler(opts...) - if err != nil { - panic(err) - } - return h -} - // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster