Broadcast CreateSliceMessage when a new max slice is created

This commit is contained in:
Travis 2017-04-25 10:50:11 -05:00
parent bb2f9c4c31
commit 7246e1e00d
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
7 changed files with 118 additions and 51 deletions

View file

@ -67,7 +67,8 @@ func NewFrame(path, index, name string) (*Frame, error) {
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
stats: NopStatsClient,
broadcaster: NopBroadcaster,
stats: NopStatsClient,
rowLabel: DefaultRowLabel,
inverseEnabled: DefaultInverseEnabled,
@ -423,6 +424,7 @@ func (f *Frame) newView(path, name string) *View {
view.LogOutput = f.LogOutput
view.RowAttrStore = f.rowAttrStore
view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name))
view.broadcaster = f.broadcaster
return view
}

View file

@ -46,7 +46,8 @@ func NewHolder() *Holder {
indexes: make(map[string]*Index),
closing: make(chan struct{}, 0),
Stats: NopStatsClient,
Broadcaster: NopBroadcaster,
Stats: NopStatsClient,
CacheFlushInterval: DefaultCacheFlushInterval,

View file

@ -68,8 +68,9 @@ func NewIndex(path, name string) (*Index, error) {
columnLabel: DefaultColumnLabel,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
broadcaster: NopBroadcaster,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}, nil
}

View file

@ -126,8 +126,9 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 {
}
type CreateSliceMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"`
}
func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} }
@ -583,6 +584,16 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
}
if m.IsInverse {
dAtA[i] = 0x18
i++
if m.IsInverse {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -1044,6 +1055,9 @@ func (m *CreateSliceMessage) Size() (n int) {
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
}
if m.IsInverse {
n += 2
}
return n
}
@ -2249,6 +2263,26 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IsInverse = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -3374,45 +3408,45 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 626 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40,
0x10, 0xc5, 0x89, 0x53, 0x9a, 0xa9, 0x5a, 0xda, 0xa5, 0x42, 0xa6, 0x42, 0x51, 0xb4, 0x07, 0x5a,
0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x14, 0x35, 0x12, 0x2d, 0x62, 0x53, 0x71, 0xdf,
0x36, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0x7c, 0x07, 0x12, 0x27, 0x3e, 0x80,
0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0x67, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc, 0x76,
0xde, 0xcc, 0xce, 0x7b, 0xfb, 0x3c, 0x63, 0x58, 0x1d, 0xa7, 0xe1, 0x44, 0x6a, 0xdc, 0x1b, 0xa7,
0x89, 0x4e, 0xd8, 0x72, 0x18, 0x6b, 0x4c, 0x63, 0x19, 0xf1, 0x57, 0xd0, 0xec, 0xc5, 0x03, 0xbc,
0x3c, 0x41, 0x2d, 0x59, 0x1b, 0x56, 0x3a, 0x49, 0x94, 0x8d, 0xe2, 0x97, 0xf2, 0x1c, 0xa3, 0xc0,
0x6b, 0x7b, 0x3b, 0x4d, 0xb1, 0x08, 0x99, 0x8a, 0xb3, 0x70, 0x84, 0xaf, 0x33, 0x19, 0xeb, 0x6c,
0x14, 0xd4, 0x6c, 0xc5, 0x02, 0xc4, 0xbf, 0x7a, 0xd0, 0x7c, 0x91, 0xca, 0x11, 0x52, 0xc7, 0x2d,
0x58, 0x16, 0xc9, 0x74, 0xb1, 0x5d, 0x11, 0xb3, 0x87, 0xb0, 0xd6, 0x8b, 0x27, 0x98, 0x2a, 0x3c,
0x8a, 0xe5, 0x79, 0x84, 0x03, 0x6a, 0xb7, 0x2c, 0x2a, 0x28, 0x7b, 0x00, 0xcd, 0x8e, 0xbc, 0x78,
0x87, 0x67, 0xb3, 0x31, 0x06, 0x75, 0x6a, 0x52, 0x02, 0x45, 0xb6, 0x1f, 0x7e, 0xc0, 0xc0, 0x6f,
0x7b, 0x3b, 0xab, 0xa2, 0x04, 0xaa, 0x7a, 0x1b, 0x57, 0xf5, 0x72, 0x58, 0xeb, 0x8d, 0xc6, 0x49,
0xaa, 0x05, 0xaa, 0x71, 0x12, 0x2b, 0x64, 0xeb, 0x50, 0x3f, 0x4a, 0x53, 0x27, 0xd7, 0x1c, 0xf9,
0x47, 0x58, 0x3f, 0x8c, 0x92, 0x8b, 0x61, 0x57, 0x6a, 0x29, 0xf0, 0x7d, 0x86, 0x4a, 0xb3, 0x4d,
0x68, 0x90, 0x71, 0xae, 0xce, 0x06, 0x06, 0xa5, 0xc7, 0x3b, 0x67, 0x6c, 0x60, 0x50, 0xba, 0x4f,
0xea, 0x7d, 0x61, 0x03, 0x83, 0xf6, 0xa3, 0xf0, 0xc2, 0xaa, 0xf6, 0x85, 0x0d, 0x18, 0x03, 0xff,
0x4d, 0x88, 0x53, 0x27, 0x95, 0xce, 0xbc, 0x07, 0x1b, 0x0b, 0xfc, 0x4e, 0xe6, 0x3d, 0x58, 0x12,
0xc9, 0xb4, 0xd7, 0x55, 0x81, 0xd7, 0xae, 0xef, 0xf8, 0xc2, 0x45, 0x64, 0x08, 0x7d, 0x31, 0x93,
0xaa, 0x51, 0xaa, 0x04, 0xf8, 0x7d, 0x68, 0x90, 0x3b, 0xe6, 0x95, 0xe5, 0x5d, 0x73, 0xe4, 0x9f,
0x3d, 0xd8, 0x38, 0x91, 0x97, 0x24, 0x43, 0x15, 0x34, 0xc7, 0xd0, 0x2c, 0x40, 0xaa, 0x5e, 0xd9,
0xdf, 0xdd, 0xcb, 0xc7, 0x67, 0xef, 0x4a, 0x7d, 0x89, 0x1c, 0xc5, 0x3a, 0x9d, 0x89, 0xf2, 0xf2,
0xd6, 0x73, 0x58, 0xfb, 0x3d, 0x69, 0x34, 0x0c, 0x71, 0x96, 0x3b, 0x3d, 0xc4, 0x99, 0xf1, 0x64,
0x22, 0xa3, 0xcc, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0xad, 0x3d, 0xf1, 0xf8, 0x01, 0xb0, 0x4e, 0x8a,
0x52, 0x23, 0x35, 0x38, 0x41, 0xa5, 0xe4, 0x5b, 0xbc, 0xfe, 0x2b, 0x58, 0x67, 0x6b, 0x0b, 0xce,
0xf2, 0x5d, 0x60, 0x5d, 0x8c, 0x50, 0xa3, 0x1b, 0xf8, 0xbf, 0x74, 0xe0, 0xfd, 0x9c, 0xed, 0xe6,
0x5a, 0xb6, 0x0d, 0xbe, 0x99, 0x75, 0x22, 0x5b, 0xd9, 0xbf, 0x5b, 0x9a, 0x53, 0x2c, 0x96, 0xa0,
0x02, 0x1e, 0xe6, 0x4d, 0xdd, 0x7e, 0xdc, 0xf0, 0x84, 0x3f, 0x0c, 0x52, 0x4e, 0x55, 0xaf, 0x52,
0x15, 0x1b, 0xe7, 0xa8, 0x0e, 0xf2, 0xb7, 0xfe, 0x2f, 0x15, 0xef, 0x3a, 0xd4, 0x0c, 0xe4, 0xa9,
0xc9, 0xda, 0x3b, 0x74, 0xbe, 0xfe, 0xc9, 0x55, 0x1d, 0x5f, 0x3c, 0x47, 0xf9, 0x6f, 0x6d, 0x2a,
0xce, 0x99, 0xdf, 0x48, 0x3e, 0x3a, 0x6e, 0x87, 0x8a, 0x98, 0x6d, 0xc3, 0x12, 0xb1, 0xaa, 0xc0,
0xa7, 0xe9, 0xbc, 0x53, 0x51, 0x23, 0x5c, 0xda, 0x2c, 0x8c, 0x1b, 0xe3, 0x86, 0x5d, 0x18, 0x1b,
0x71, 0x09, 0x70, 0x9a, 0x0c, 0xb0, 0xaf, 0xa5, 0xce, 0x94, 0xd1, 0x79, 0x9c, 0x28, 0x9d, 0xeb,
0x34, 0x67, 0x9a, 0x27, 0x2d, 0x75, 0xe1, 0x10, 0x05, 0xec, 0x11, 0xdc, 0x26, 0x9d, 0xa8, 0x82,
0x7a, 0x95, 0x99, 0x12, 0x22, 0xcf, 0xf3, 0x67, 0xb0, 0xda, 0x89, 0x32, 0xa5, 0x31, 0x75, 0x2c,
0xbb, 0xd0, 0x30, 0x9c, 0xf9, 0x46, 0x6d, 0x96, 0x37, 0x4b, 0x29, 0xc2, 0x96, 0x1c, 0xae, 0x7f,
0x9b, 0xb7, 0xbc, 0xef, 0xf3, 0x96, 0xf7, 0x63, 0xde, 0xf2, 0x3e, 0xfd, 0x6c, 0xdd, 0x3a, 0x5f,
0xa2, 0xbf, 0xf8, 0xe3, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3c, 0x08, 0xd4, 0xe9, 0xd6, 0x05,
0x00, 0x00,
// 640 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x4e, 0x14, 0x4d,
0x10, 0xfe, 0x67, 0x77, 0x96, 0x7f, 0xa7, 0x08, 0x08, 0x2d, 0x31, 0x23, 0x21, 0x9b, 0x4d, 0x1f,
0x04, 0x39, 0x70, 0xc0, 0x8b, 0x51, 0x0f, 0x86, 0x5d, 0x0c, 0x93, 0x08, 0xc6, 0x5e, 0xe2, 0xd1,
0xa4, 0x81, 0x8a, 0x4e, 0x98, 0x9d, 0x59, 0xa7, 0x7b, 0x80, 0xf5, 0xe0, 0x73, 0x98, 0x78, 0xf2,
0x01, 0x7c, 0x0f, 0x8f, 0x3e, 0x82, 0xc1, 0x17, 0x31, 0x5d, 0xdd, 0x33, 0xb3, 0x0e, 0x22, 0xd1,
0x5b, 0xd7, 0x57, 0xd5, 0xf5, 0x7d, 0xfd, 0x4d, 0xd5, 0xc0, 0xc2, 0x24, 0x8f, 0xcf, 0xa4, 0xc6,
0xad, 0x49, 0x9e, 0xe9, 0x8c, 0x75, 0xe3, 0x54, 0x63, 0x9e, 0xca, 0x84, 0xbf, 0x80, 0x20, 0x4a,
0x4f, 0xf0, 0x62, 0x1f, 0xb5, 0x64, 0x7d, 0x98, 0x1f, 0x64, 0x49, 0x31, 0x4e, 0x9f, 0xcb, 0x23,
0x4c, 0x42, 0xaf, 0xef, 0x6d, 0x04, 0x62, 0x16, 0x32, 0x15, 0x87, 0xf1, 0x18, 0x5f, 0x16, 0x32,
0xd5, 0xc5, 0x38, 0x6c, 0xd9, 0x8a, 0x19, 0x88, 0x7f, 0xf1, 0x20, 0x78, 0x96, 0xcb, 0x31, 0x52,
0xc7, 0x55, 0xe8, 0x8a, 0xec, 0x7c, 0xb6, 0x5d, 0x15, 0xb3, 0x7b, 0xb0, 0x18, 0xa5, 0x67, 0x98,
0x2b, 0xdc, 0x4d, 0xe5, 0x51, 0x82, 0x27, 0xd4, 0xae, 0x2b, 0x1a, 0x28, 0x5b, 0x83, 0x60, 0x20,
0x8f, 0xdf, 0xe2, 0xe1, 0x74, 0x82, 0x61, 0x9b, 0x9a, 0xd4, 0x40, 0x95, 0x1d, 0xc5, 0xef, 0x31,
0xf4, 0xfb, 0xde, 0xc6, 0x82, 0xa8, 0x81, 0xa6, 0xde, 0xce, 0x55, 0xbd, 0x1c, 0x16, 0xa3, 0xf1,
0x24, 0xcb, 0xb5, 0x40, 0x35, 0xc9, 0x52, 0x85, 0x6c, 0x09, 0xda, 0xbb, 0x79, 0xee, 0xe4, 0x9a,
0x23, 0xff, 0x00, 0x4b, 0x3b, 0x49, 0x76, 0x7c, 0x3a, 0x94, 0x5a, 0x0a, 0x7c, 0x57, 0xa0, 0xd2,
0x6c, 0x05, 0x3a, 0x64, 0x9c, 0xab, 0xb3, 0x81, 0x41, 0xe9, 0xf1, 0xce, 0x19, 0x1b, 0x18, 0x94,
0xee, 0x93, 0x7a, 0x5f, 0xd8, 0xc0, 0xa0, 0xa3, 0x24, 0x3e, 0xb6, 0xaa, 0x7d, 0x61, 0x03, 0xc6,
0xc0, 0x7f, 0x15, 0xe3, 0xb9, 0x93, 0x4a, 0x67, 0x1e, 0xc1, 0xf2, 0x0c, 0xbf, 0x93, 0x79, 0x07,
0xe6, 0x44, 0x76, 0x1e, 0x0d, 0x55, 0xe8, 0xf5, 0xdb, 0x1b, 0xbe, 0x70, 0x11, 0x19, 0x42, 0x5f,
0xcc, 0xa4, 0x5a, 0x94, 0xaa, 0x01, 0x7e, 0x17, 0x3a, 0xe4, 0x8e, 0x79, 0x65, 0x7d, 0xd7, 0x1c,
0xf9, 0x27, 0x0f, 0x96, 0xf7, 0xe5, 0x05, 0xc9, 0x50, 0x15, 0xcd, 0x1e, 0x04, 0x15, 0x48, 0xd5,
0xf3, 0xdb, 0x9b, 0x5b, 0xe5, 0xf8, 0x6c, 0x5d, 0xa9, 0xaf, 0x91, 0xdd, 0x54, 0xe7, 0x53, 0x51,
0x5f, 0x5e, 0x7d, 0x02, 0x8b, 0xbf, 0x26, 0x8d, 0x86, 0x53, 0x9c, 0x96, 0x4e, 0x9f, 0xe2, 0xd4,
0x78, 0x72, 0x26, 0x93, 0xc2, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0x6a, 0x3d, 0xf4, 0xf8, 0x6b, 0x60,
0x83, 0x1c, 0xa5, 0x46, 0x6a, 0xb0, 0x8f, 0x4a, 0xc9, 0x37, 0x78, 0xfd, 0x57, 0xb0, 0xce, 0xb6,
0x66, 0x9d, 0x5d, 0x83, 0x20, 0x52, 0x6e, 0xb6, 0xe8, 0x4b, 0x74, 0x45, 0x0d, 0xf0, 0x4d, 0x60,
0x43, 0x4c, 0x50, 0xa3, 0x5b, 0x87, 0x3f, 0xf4, 0xe7, 0xa3, 0x52, 0xcb, 0xcd, 0xb5, 0x6c, 0x1d,
0x7c, 0xb3, 0x09, 0x24, 0x65, 0x7e, 0xfb, 0x76, 0x6d, 0x5d, 0xb5, 0x76, 0x82, 0x0a, 0x78, 0x5c,
0x36, 0x75, 0xdb, 0x73, 0xc3, 0x03, 0x7f, 0x33, 0x66, 0x25, 0x55, 0xbb, 0x49, 0x55, 0xed, 0xa3,
0xa3, 0x7a, 0x5a, 0xbe, 0xf5, 0x5f, 0xa9, 0xf8, 0xd0, 0xa1, 0x66, 0x5c, 0x0f, 0x4c, 0xd6, 0xde,
0xa1, 0xf3, 0xf5, 0x4f, 0x6e, 0xea, 0xf8, 0xec, 0x39, 0xca, 0xbf, 0x6b, 0xd3, 0x70, 0xce, 0xfc,
0x64, 0xca, 0xc1, 0x72, 0x1b, 0x56, 0xc5, 0x6c, 0x1d, 0xe6, 0x88, 0x55, 0x85, 0x3e, 0xcd, 0xee,
0xad, 0x86, 0x1a, 0xe1, 0xd2, 0x66, 0x9d, 0xdc, 0x90, 0x77, 0xec, 0x3a, 0xd9, 0x88, 0x4b, 0x80,
0x83, 0xec, 0x04, 0x47, 0x5a, 0xea, 0x42, 0x19, 0x9d, 0x7b, 0x99, 0xd2, 0xa5, 0x4e, 0x73, 0xa6,
0x69, 0xd3, 0x52, 0x57, 0x0e, 0x51, 0xc0, 0xee, 0xc3, 0xff, 0xa4, 0x13, 0x55, 0xd8, 0x6e, 0x32,
0x53, 0x42, 0x94, 0x79, 0xfe, 0x18, 0x16, 0x06, 0x49, 0xa1, 0x34, 0xe6, 0x8e, 0x65, 0x13, 0x3a,
0x86, 0xb3, 0xdc, 0xb7, 0x95, 0xfa, 0x66, 0x2d, 0x45, 0xd8, 0x92, 0x9d, 0xa5, 0xaf, 0x97, 0x3d,
0xef, 0xdb, 0x65, 0xcf, 0xfb, 0x7e, 0xd9, 0xf3, 0x3e, 0xfe, 0xe8, 0xfd, 0x77, 0x34, 0x47, 0xff,
0xf8, 0x07, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1c, 0x79, 0x08, 0xf4, 0x05, 0x00, 0x00,
}

View file

@ -43,6 +43,7 @@ message MaxSlicesResponse {
message CreateSliceMessage {
string Index = 1;
uint64 Slice = 2;
bool IsInverse = 3;
}
message DeleteIndexMessage {

View file

@ -245,7 +245,11 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
idx.SetRemoteMaxSlice(obj.Slice)
if obj.IsInverse {
idx.SetRemoteMaxInverseSlice(obj.Slice)
} else {
idx.SetRemoteMaxSlice(obj.Slice)
}
case *internal.CreateIndexMessage:
opt := IndexOptions{
ColumnLabel: obj.Meta.ColumnLabel,

30
view.go
View file

@ -9,6 +9,8 @@ import (
"strconv"
"strings"
"sync"
"github.com/pilosa/pilosa/internal"
)
// View layout modes.
@ -36,7 +38,12 @@ type View struct {
cacheType string // passed in by frame
fragments map[uint64]*Fragment
stats StatsClient
// maxSlice maintains this view's max slice in order to
// prevent sending multiple `CreateSliceMessage` messages
maxSlice uint64
broadcaster Broadcaster
stats StatsClient
RowAttrStore *AttrStore
LogOutput io.Writer
@ -54,8 +61,9 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View {
cacheType: DefaultCacheType,
fragments: make(map[uint64]*Fragment),
stats: NopStatsClient,
LogOutput: ioutil.Discard,
broadcaster: NopBroadcaster,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}
}
@ -207,6 +215,22 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
}
frag.RowAttrStore = v.RowAttrStore
// Broadcast a message that a new max slice was just created.
if slice > v.maxSlice {
v.maxSlice = slice
// Send the create slice message to all nodes.
err := v.broadcaster.SendAsync(
&internal.CreateSliceMessage{
Index: v.index,
Slice: slice,
IsInverse: IsInverseView(v.name),
})
if err != nil {
return nil, err
}
}
// Save to lookup.
v.fragments[slice] = frag