scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-09 06:31:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #496 from raskle/495-owns-slice

Browse source 
OwnsSlice
This commit is contained in:
Michael Baird 2017-04-26 17:10:53 -05:00 • committed by GitHub
commit bb2f9c4c31
7 changed files with 164 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
cluster.go
View file

@ -225,6 +225,20 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node {
return nodes
}
// OwnsSlices find the set of slices owned by the node per Index
func (c *Cluster) OwnsSlices(index string, maxSlice uint64, host string) []uint64 {
var slices []uint64
for i := uint64(0); i <= maxSlice; i++ {
p := c.Partition(index, i)
// Determine primary owner node.
nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes))
if c.Nodes[nodeIndex].Host == host {
slices = append(slices, i)
}
}
return slices
}
// Hasher represents an interface to hash integers into buckets.
type Hasher interface {
// Hashes the key into a number between [0,N).

10
cluster_test.go
View file

@ -118,6 +118,16 @@ func TestCluster_NodeStates(t *testing.T) {
}
}
// Ensure OwnsSlices can find the actual slice list for node and index
func TestCluster_OwnsSlices(t *testing.T) {
c := NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
if !reflect.DeepEqual(slices, []uint64{1, 3, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()

2
index.go
View file

@ -531,7 +531,7 @@ func encodeIndex(d *Index) *internal.Index {
ColumnLabel: d.columnLabel,
TimeQuantum: string(d.timeQuantum),
},
MaxSlice: d.remoteMaxSlice,
MaxSlice: d.MaxSlice(),
Frames: encodeFrames(d.Frames()),
}
}

166
internal/private.pb.go
View file

@ -211,6 +211,7 @@ type Index struct {
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"`
Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"`
Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"`
}
func (m *Index) Reset() { *m = Index{} }
@ -795,6 +796,23 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if len(m.Slices) > 0 {
dAtA12 := make([]byte, len(m.Slices)*10)
var j11 int
for _, num := range m.Slices {
for num >= 1<<7 {
dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j11++
}
dAtA12[j11] = uint8(num)
j11++
}
dAtA[i] = 0x2a
i++
i = encodeVarintPrivate(dAtA, i, uint64(j11))
i += copy(dAtA[i:], dAtA12[:j11])
}
return i, nil
}
@ -1119,6 +1137,13 @@ func (m *Index) Size() (n int) {
n += 1 + l + sovPrivate(uint64(l))
}
}
if len(m.Slices) > 0 {
l = 0
for _, e := range m.Slices {
l += sovPrivate(uint64(e))
}
n += 1 + sovPrivate(uint64(l)) + l
}
return n
}
@ -2938,6 +2963,68 @@ func (m *Index) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 5:
if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
packedLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if packedLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + packedLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
for iNdEx < postIndex {
var v uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Slices = append(m.Slices, v)
}
} else if wireType == 0 {
var v uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Slices = append(m.Slices, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType)
}
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -3287,44 +3374,45 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 617 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40,
// 626 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40,
0x10, 0xc5, 0x89, 0x53, 0x9a, 0xa9, 0x5a, 0xda, 0xa5, 0x42, 0xa6, 0x42, 0x51, 0xb4, 0x07, 0x5a,
0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, 0xdf,
0x24, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0xdc, 0xf9, 0x03, 0x24, 0xbe, 0x81,
0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0xc7, 0x6b, 0x3b, 0xb8, 0x94, 0x0a, 0x6e, 0x3b,
0x6f, 0x66, 0xe7, 0xbd, 0x7d, 0x9e, 0x31, 0x6c, 0x4e, 0x63, 0x7f, 0x26, 0x35, 0x1e, 0x4d, 0xe3,
0x48, 0x47, 0x6c, 0xdd, 0x0f, 0x35, 0xc6, 0xa1, 0x0c, 0xf8, 0x2b, 0xa8, 0x77, 0xc3, 0x11, 0x5e,
0xf6, 0x50, 0x4b, 0xd6, 0x84, 0x8d, 0x56, 0x14, 0x24, 0x93, 0xf0, 0xa5, 0x1c, 0x60, 0xe0, 0x39,
0x4d, 0xe7, 0xa0, 0x2e, 0x56, 0x21, 0x53, 0x71, 0xe1, 0x4f, 0xf0, 0x75, 0x22, 0x43, 0x9d, 0x4c,
0xbc, 0x4a, 0x5a, 0xb1, 0x02, 0xf1, 0xaf, 0x0e, 0xd4, 0x5f, 0xc4, 0x72, 0x82, 0xd4, 0x71, 0x0f,
0xd6, 0x45, 0x34, 0x5f, 0x6d, 0x97, 0xc7, 0xec, 0x21, 0x6c, 0x75, 0xc3, 0x19, 0xc6, 0x0a, 0x3b,
0xa1, 0x1c, 0x04, 0x38, 0xa2, 0x76, 0xeb, 0xa2, 0x84, 0xb2, 0x07, 0x50, 0x6f, 0xc9, 0xe1, 0x3b,
0xbc, 0x58, 0x4c, 0xd1, 0xab, 0x52, 0x93, 0x02, 0xc8, 0xb3, 0x7d, 0xff, 0x03, 0x7a, 0x6e, 0xd3,
0x39, 0xd8, 0x14, 0x05, 0x50, 0xd6, 0x5b, 0xbb, 0xaa, 0x97, 0xc3, 0x56, 0x77, 0x32, 0x8d, 0x62,
0x2d, 0x50, 0x4d, 0xa3, 0x50, 0x21, 0xdb, 0x86, 0x6a, 0x27, 0x8e, 0xad, 0x5c, 0x73, 0xe4, 0x1f,
0x61, 0xfb, 0x34, 0x88, 0x86, 0xe3, 0xb6, 0xd4, 0x52, 0xe0, 0xfb, 0x04, 0x95, 0x66, 0xbb, 0x50,
0x23, 0xe3, 0x6c, 0x5d, 0x1a, 0x18, 0x94, 0x1e, 0x6f, 0x9d, 0x49, 0x03, 0x83, 0xd2, 0x7d, 0x52,
0xef, 0x8a, 0x34, 0x30, 0x68, 0x3f, 0xf0, 0x87, 0xa9, 0x6a, 0x57, 0xa4, 0x01, 0x63, 0xe0, 0xbe,
0xf1, 0x71, 0x6e, 0xa5, 0xd2, 0x99, 0x77, 0x61, 0x67, 0x85, 0xdf, 0xca, 0xbc, 0x07, 0x6b, 0x22,
0x9a, 0x77, 0xdb, 0xca, 0x73, 0x9a, 0xd5, 0x03, 0x57, 0xd8, 0x88, 0x0c, 0xa1, 0x2f, 0x66, 0x52,
0x15, 0x4a, 0x15, 0x00, 0xbf, 0x0f, 0x35, 0x72, 0xc7, 0xbc, 0xb2, 0xb8, 0x6b, 0x8e, 0xfc, 0x8b,
0x03, 0x3b, 0x3d, 0x79, 0x49, 0x32, 0x54, 0x4e, 0x73, 0x06, 0xf5, 0x1c, 0xa4, 0xea, 0x8d, 0xe3,
0xc3, 0xa3, 0x6c, 0x7c, 0x8e, 0xae, 0xd4, 0x17, 0x48, 0x27, 0xd4, 0xf1, 0x42, 0x14, 0x97, 0xf7,
0x9e, 0xc3, 0xd6, 0xef, 0x49, 0xa3, 0x61, 0x8c, 0x8b, 0xcc, 0xe9, 0x31, 0x2e, 0x8c, 0x27, 0x33,
0x19, 0x24, 0xa9, 0x7f, 0xae, 0x48, 0x83, 0xa7, 0x95, 0x27, 0x0e, 0x3f, 0x01, 0xd6, 0x8a, 0x51,
0x6a, 0xa4, 0x06, 0x3d, 0x54, 0x4a, 0xbe, 0xc5, 0xeb, 0xbf, 0x42, 0xea, 0x6c, 0x65, 0xc5, 0x59,
0x7e, 0x08, 0xac, 0x8d, 0x01, 0x6a, 0xb4, 0x03, 0xff, 0x97, 0x0e, 0xbc, 0x9f, 0xb1, 0xdd, 0x5c,
0xcb, 0xf6, 0xc1, 0x35, 0xb3, 0x4e, 0x64, 0x1b, 0xc7, 0x77, 0x0b, 0x73, 0xf2, 0xc5, 0x12, 0x54,
0xc0, 0xfd, 0xac, 0xa9, 0xdd, 0x8f, 0x1b, 0x9e, 0xf0, 0x87, 0x41, 0xca, 0xa8, 0xaa, 0x65, 0xaa,
0x7c, 0xe3, 0x2c, 0xd5, 0x49, 0xf6, 0xd6, 0xff, 0xa5, 0xe2, 0x6d, 0x8b, 0x9a, 0x81, 0x3c, 0x37,
0xd9, 0xf4, 0x0e, 0x9d, 0xaf, 0x7f, 0x72, 0x59, 0xc7, 0x27, 0xc7, 0x52, 0xfe, 0x5b, 0x9b, 0x92,
0x73, 0xe6, 0x37, 0x92, 0x8d, 0x8e, 0xdd, 0xa1, 0x3c, 0x66, 0xfb, 0xb0, 0x46, 0xac, 0xca, 0x73,
0x69, 0x3a, 0xef, 0x94, 0xd4, 0x08, 0x9b, 0xe6, 0x12, 0xe0, 0x3c, 0x1a, 0x61, 0x5f, 0x4b, 0x9d,
0x28, 0xa3, 0xe7, 0x2c, 0x52, 0x3a, 0xd3, 0x63, 0xce, 0x34, 0x37, 0x5a, 0xea, 0xdc, 0x09, 0x0a,
0xd8, 0x23, 0xb8, 0x4d, 0x7a, 0x50, 0x79, 0xd5, 0x32, 0x03, 0x25, 0x44, 0x96, 0xe7, 0xcf, 0x60,
0xb3, 0x15, 0x24, 0x4a, 0x63, 0x6c, 0x59, 0x0e, 0xa1, 0x66, 0x38, 0xb3, 0xcd, 0xd9, 0x2d, 0x6e,
0x16, 0x52, 0x44, 0x5a, 0x72, 0xba, 0xfd, 0x6d, 0xd9, 0x70, 0xbe, 0x2f, 0x1b, 0xce, 0x8f, 0x65,
0xc3, 0xf9, 0xfc, 0xb3, 0x71, 0x6b, 0xb0, 0x46, 0x7f, 0xeb, 0xc7, 0xbf, 0x02, 0x00, 0x00, 0xff,
0xff, 0x77, 0x02, 0x29, 0xbb, 0xbe, 0x05, 0x00, 0x00,
0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x14, 0x35, 0x12, 0x2d, 0x62, 0x53, 0x71, 0xdf,
0x36, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0x7c, 0x07, 0x12, 0x27, 0x3e, 0x80,
0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0x67, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc, 0x76,
0xde, 0xcc, 0xce, 0x7b, 0xfb, 0x3c, 0x63, 0x58, 0x1d, 0xa7, 0xe1, 0x44, 0x6a, 0xdc, 0x1b, 0xa7,
0x89, 0x4e, 0xd8, 0x72, 0x18, 0x6b, 0x4c, 0x63, 0x19, 0xf1, 0x57, 0xd0, 0xec, 0xc5, 0x03, 0xbc,
0x3c, 0x41, 0x2d, 0x59, 0x1b, 0x56, 0x3a, 0x49, 0x94, 0x8d, 0xe2, 0x97, 0xf2, 0x1c, 0xa3, 0xc0,
0x6b, 0x7b, 0x3b, 0x4d, 0xb1, 0x08, 0x99, 0x8a, 0xb3, 0x70, 0x84, 0xaf, 0x33, 0x19, 0xeb, 0x6c,
0x14, 0xd4, 0x6c, 0xc5, 0x02, 0xc4, 0xbf, 0x7a, 0xd0, 0x7c, 0x91, 0xca, 0x11, 0x52, 0xc7, 0x2d,
0x58, 0x16, 0xc9, 0x74, 0xb1, 0x5d, 0x11, 0xb3, 0x87, 0xb0, 0xd6, 0x8b, 0x27, 0x98, 0x2a, 0x3c,
0x8a, 0xe5, 0x79, 0x84, 0x03, 0x6a, 0xb7, 0x2c, 0x2a, 0x28, 0x7b, 0x00, 0xcd, 0x8e, 0xbc, 0x78,
0x87, 0x67, 0xb3, 0x31, 0x06, 0x75, 0x6a, 0x52, 0x02, 0x45, 0xb6, 0x1f, 0x7e, 0xc0, 0xc0, 0x6f,
0x7b, 0x3b, 0xab, 0xa2, 0x04, 0xaa, 0x7a, 0x1b, 0x57, 0xf5, 0x72, 0x58, 0xeb, 0x8d, 0xc6, 0x49,
0xaa, 0x05, 0xaa, 0x71, 0x12, 0x2b, 0x64, 0xeb, 0x50, 0x3f, 0x4a, 0x53, 0x27, 0xd7, 0x1c, 0xf9,
0x47, 0x58, 0x3f, 0x8c, 0x92, 0x8b, 0x61, 0x57, 0x6a, 0x29, 0xf0, 0x7d, 0x86, 0x4a, 0xb3, 0x4d,
0x68, 0x90, 0x71, 0xae, 0xce, 0x06, 0x06, 0xa5, 0xc7, 0x3b, 0x67, 0x6c, 0x60, 0x50, 0xba, 0x4f,
0xea, 0x7d, 0x61, 0x03, 0x83, 0xf6, 0xa3, 0xf0, 0xc2, 0xaa, 0xf6, 0x85, 0x0d, 0x18, 0x03, 0xff,
0x4d, 0x88, 0x53, 0x27, 0x95, 0xce, 0xbc, 0x07, 0x1b, 0x0b, 0xfc, 0x4e, 0xe6, 0x3d, 0x58, 0x12,
0xc9, 0xb4, 0xd7, 0x55, 0x81, 0xd7, 0xae, 0xef, 0xf8, 0xc2, 0x45, 0x64, 0x08, 0x7d, 0x31, 0x93,
0xaa, 0x51, 0xaa, 0x04, 0xf8, 0x7d, 0x68, 0x90, 0x3b, 0xe6, 0x95, 0xe5, 0x5d, 0x73, 0xe4, 0x9f,
0x3d, 0xd8, 0x38, 0x91, 0x97, 0x24, 0x43, 0x15, 0x34, 0xc7, 0xd0, 0x2c, 0x40, 0xaa, 0x5e, 0xd9,
0xdf, 0xdd, 0xcb, 0xc7, 0x67, 0xef, 0x4a, 0x7d, 0x89, 0x1c, 0xc5, 0x3a, 0x9d, 0x89, 0xf2, 0xf2,
0xd6, 0x73, 0x58, 0xfb, 0x3d, 0x69, 0x34, 0x0c, 0x71, 0x96, 0x3b, 0x3d, 0xc4, 0x99, 0xf1, 0x64,
0x22, 0xa3, 0xcc, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0xad, 0x3d, 0xf1, 0xf8, 0x01, 0xb0, 0x4e, 0x8a,
0x52, 0x23, 0x35, 0x38, 0x41, 0xa5, 0xe4, 0x5b, 0xbc, 0xfe, 0x2b, 0x58, 0x67, 0x6b, 0x0b, 0xce,
0xf2, 0x5d, 0x60, 0x5d, 0x8c, 0x50, 0xa3, 0x1b, 0xf8, 0xbf, 0x74, 0xe0, 0xfd, 0x9c, 0xed, 0xe6,
0x5a, 0xb6, 0x0d, 0xbe, 0x99, 0x75, 0x22, 0x5b, 0xd9, 0xbf, 0x5b, 0x9a, 0x53, 0x2c, 0x96, 0xa0,
0x02, 0x1e, 0xe6, 0x4d, 0xdd, 0x7e, 0xdc, 0xf0, 0x84, 0x3f, 0x0c, 0x52, 0x4e, 0x55, 0xaf, 0x52,
0x15, 0x1b, 0xe7, 0xa8, 0x0e, 0xf2, 0xb7, 0xfe, 0x2f, 0x15, 0xef, 0x3a, 0xd4, 0x0c, 0xe4, 0xa9,
0xc9, 0xda, 0x3b, 0x74, 0xbe, 0xfe, 0xc9, 0x55, 0x1d, 0x5f, 0x3c, 0x47, 0xf9, 0x6f, 0x6d, 0x2a,
0xce, 0x99, 0xdf, 0x48, 0x3e, 0x3a, 0x6e, 0x87, 0x8a, 0x98, 0x6d, 0xc3, 0x12, 0xb1, 0xaa, 0xc0,
0xa7, 0xe9, 0xbc, 0x53, 0x51, 0x23, 0x5c, 0xda, 0x2c, 0x8c, 0x1b, 0xe3, 0x86, 0x5d, 0x18, 0x1b,
0x71, 0x09, 0x70, 0x9a, 0x0c, 0xb0, 0xaf, 0xa5, 0xce, 0x94, 0xd1, 0x79, 0x9c, 0x28, 0x9d, 0xeb,
0x34, 0x67, 0x9a, 0x27, 0x2d, 0x75, 0xe1, 0x10, 0x05, 0xec, 0x11, 0xdc, 0x26, 0x9d, 0xa8, 0x82,
0x7a, 0x95, 0x99, 0x12, 0x22, 0xcf, 0xf3, 0x67, 0xb0, 0xda, 0x89, 0x32, 0xa5, 0x31, 0x75, 0x2c,
0xbb, 0xd0, 0x30, 0x9c, 0xf9, 0x46, 0x6d, 0x96, 0x37, 0x4b, 0x29, 0xc2, 0x96, 0x1c, 0xae, 0x7f,
0x9b, 0xb7, 0xbc, 0xef, 0xf3, 0x96, 0xf7, 0x63, 0xde, 0xf2, 0x3e, 0xfd, 0x6c, 0xdd, 0x3a, 0x5f,
0xa2, 0xbf, 0xf8, 0xe3, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3c, 0x08, 0xd4, 0xe9, 0xd6, 0x05,
0x00, 0x00,
}

1
internal/private.proto
View file

@ -75,6 +75,7 @@ message Index {
IndexMeta Meta = 2;
uint64 MaxSlice = 3;
repeated Frame Frames = 4;
repeated uint64 Slices = 5;
}
message NodeStatus {

2
internal/public.pb.go
View file

@ -2576,7 +2576,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 576 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57,
0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4,
0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b,

12
server.go
View file

@ -289,11 +289,19 @@ func (s *Server) LocalStatus() (proto.Message, error) {
if s.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
return &internal.NodeStatus{
ns := internal.NodeStatus{
Host: s.Host,
State: NodeStateUp,
Indexes: encodeIndexes(s.Holder.Indexes()),
}, nil
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Host)
}
return &ns, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.