Merge pull request #449 from benbjohnson/remove-ranked-suffix

Add cacheType to frame options.
This commit is contained in:
Ben Johnson 2017-04-18 09:14:40 -06:00 • committed by GitHub
commit 11882b4577
10 changed files with 158 additions and 55 deletions

9
db.go
View file

@ -358,9 +358,10 @@ func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, err
}
func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
if name == "" {
return nil, errors.New("frame name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Initialize frame.
@ -380,6 +381,12 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, err
}
// Set cache type.
if opt.CacheType == "" {
opt.CacheType = DefaultCacheType
}
f.cacheType = opt.CacheType
// Set options.
if opt.RowLabel != "" {
f.rowLabel = opt.RowLabel

View file

@ -16,7 +16,6 @@ import (
"log"
"os"
"sort"
"strings"
"sync"
"syscall"
"time"
@ -69,7 +68,8 @@ type Fragment struct {
opN int // number of ops since snapshot
// Cache for bitmap counts.
cache Cache
cacheType string // passed in by frame
cache Cache
// Cache containing full bitmaps (not just counts).
bitmapCache BitmapCache
@ -95,11 +95,12 @@ type Fragment struct {
// NewFragment returns a new instance of Fragment.
func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
return &Fragment{
path: path,
db: db,
frame: frame,
view: view,
slice: slice,
path: path,
db: db,
frame: frame,
view: view,
slice: slice,
cacheType: DefaultCacheType,
LogOutput: ioutil.Discard,
MaxOpN: DefaultFragmentMaxOpN,
@ -219,13 +220,16 @@ func (f *Fragment) openStorage() error {
// openCache initializes the cache from bitmap ids persisted to disk.
func (f *Fragment) openCache() error {
// Determine cache type from frame name.
if strings.HasSuffix(f.frame, FrameSuffixRank) {
switch f.cacheType {
case CacheTypeRanked:
c := NewRankCache()
c.ThresholdLength = 50000
c.ThresholdIndex = 45000
f.cache = c
} else {
case CacheTypeLRU:
f.cache = NewLRUCache(50000)
default:
return ErrInvalidCacheType
}
// Read cache data from disk.

View file

@ -386,8 +386,26 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_RankCache_Persistence(t *testing.T) {
f := MustOpenFragment("d", "f.n", pilosa.ViewStandard, 0)
defer f.Close()
db := MustOpenDB()
defer db.Close()
// Create frame.
frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
t.Fatal(err)
}
// Create view.
view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard)
if err != nil {
t.Fatal(err)
}
// Create fragment.
f, err := view.CreateFragmentIfNotExists(0)
if err != nil {
t.Fatal(err)
}
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
@ -403,11 +421,14 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
// Reopen the fragment.
if err := f.Reopen(); err != nil {
// Reopen the database.
if err := db.Reopen(); err != nil {
t.Fatal(err)
}
// Re-fetch fragment.
f = db.Frame("f").View(pilosa.ViewStandard).Fragment(0)
// Re-verify correct cache type and size.
if cache, ok := f.Cache().(*pilosa.RankCache); !ok {
t.Fatalf("unexpected cache: %T", f.Cache())

View file

@ -15,14 +15,10 @@ import (
"github.com/pilosa/pilosa/internal"
)
const (
// FrameSuffixRank is the suffix used for rank-based frames.
FrameSuffixRank = ".n"
)
// Default frame settings.
const (
DefaultRowLabel = "id"
DefaultCacheType = CacheTypeLRU
DefaultInverseEnabled = false
)
@ -42,8 +38,9 @@ type Frame struct {
stats StatsClient
// Frame settings.
inverseEnabled bool
rowLabel string
cacheType string
inverseEnabled bool
LogOutput io.Writer
}
@ -65,8 +62,9 @@ func NewFrame(path, db, name string) (*Frame, error) {
stats: NopStatsClient,
inverseEnabled: DefaultInverseEnabled,
rowLabel: DefaultRowLabel,
cacheType: DefaultCacheType,
inverseEnabled: DefaultInverseEnabled,
LogOutput: ioutil.Discard,
}, nil
@ -118,7 +116,6 @@ func (f *Frame) SetRowLabel(v string) error {
return nil
}
// Make sure rowLabel is valid name
err := ValidateName(v)
if err != nil {
@ -142,6 +139,11 @@ func (f *Frame) RowLabel() string {
return v
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
}
// InverseEnabled returns true if an inverse view is available.
func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
@ -151,8 +153,9 @@ func (f *Frame) InverseEnabled() bool {
func (f *Frame) Options() FrameOptions {
f.mu.Lock()
opt := FrameOptions{
InverseEnabled: f.inverseEnabled,
RowLabel: f.rowLabel,
CacheType: f.cacheType,
InverseEnabled: f.inverseEnabled,
}
f.mu.Unlock()
return opt
@ -230,6 +233,7 @@ func (f *Frame) loadMeta() error {
if os.IsNotExist(err) {
f.timeQuantum = ""
f.rowLabel = DefaultRowLabel
f.cacheType = DefaultCacheType
f.inverseEnabled = DefaultInverseEnabled
return nil
} else if err != nil {
@ -245,6 +249,12 @@ func (f *Frame) loadMeta() error {
f.rowLabel = pb.RowLabel
f.inverseEnabled = pb.InverseEnabled
// Copy cache type.
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
}
return nil
}
@ -254,6 +264,7 @@ func (f *Frame) saveMeta() error {
buf, err := proto.Marshal(&internal.Frame{
TimeQuantum: string(f.timeQuantum),
RowLabel: f.rowLabel,
CacheType: f.cacheType,
InverseEnabled: f.inverseEnabled,
})
if err != nil {
@ -366,6 +377,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.db, f.name, name)
view.cacheType = f.cacheType
view.LogOutput = f.LogOutput
view.BitmapAttrStore = f.bitmapAttrStore
view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name))
@ -546,6 +558,7 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
RowLabel string `json:"rowLabel,omitempty"`
CacheType string `json:"cacheType,omitempty"`
InverseEnabled bool `json:"inverseEnabled,omitempty"`
}
@ -561,3 +574,19 @@ func (p importBitSet) Swap(i, j int) {
}
func (p importBitSet) Len() int { return len(p.bitmapIDs) }
func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] }
// Cache types.
const (
CacheTypeLRU = "lru"
CacheTypeRanked = "ranked"
)
// IsValidCacheType returns true if v is a valid cache type.
func IsValidCacheType(v string) bool {
switch v {
case CacheTypeLRU, CacheTypeRanked:
return true
default:
return false
}
}

View file

@ -1219,10 +1219,6 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
return
}
for _, view := range views {
println("dbg/views", view)
}
// Loop over each slice and import it if this node owns it.
//travis
for slice := uint64(0); slice <= maxSlices[db]; slice++ {

View file

@ -50,6 +50,7 @@ type Frame struct {
TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
RowLabel string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"`
InverseEnabled bool `protobuf:"varint,3,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,4,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
}
func (m *Frame) Reset() { *m = Frame{} }
@ -190,6 +191,12 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if len(m.CacheType) > 0 {
dAtA[i] = 0x22
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType)))
i += copy(dAtA[i:], m.CacheType)
}
return i, nil
}
@ -439,6 +446,10 @@ func (m *Frame) Size() (n int) {
if m.InverseEnabled {
n += 2
}
l = len(m.CacheType)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
return n
}
@ -751,6 +762,35 @@ func (m *Frame) Unmarshal(dAtA []byte) error {
}
}
m.InverseEnabled = bool(v != 0)
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.CacheType = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -1576,30 +1616,31 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 389 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x92, 0x4f, 0x8b, 0xd3, 0x40,
0x18, 0xc6, 0x9d, 0x34, 0x91, 0xf6, 0x5d, 0x0c, 0xdd, 0x61, 0x0f, 0x61, 0x91, 0x10, 0x06, 0x94,
0xe2, 0xa1, 0x07, 0xbd, 0x88, 0x78, 0xca, 0xb6, 0xb2, 0x05, 0x05, 0x77, 0x14, 0xef, 0xd3, 0xfa,
0x8a, 0xa1, 0x93, 0x99, 0x38, 0x99, 0x74, 0xdb, 0x6f, 0x22, 0xf8, 0x85, 0x3c, 0xfa, 0x11, 0xa4,
0x7e, 0x11, 0x99, 0x49, 0xda, 0x46, 0x3d, 0xb8, 0xb7, 0x79, 0x7f, 0xef, 0xbf, 0xe7, 0x7d, 0x18,
0x78, 0x50, 0x99, 0x62, 0x23, 0x2c, 0x4e, 0x2b, 0xa3, 0xad, 0xa6, 0xc3, 0x42, 0x59, 0x34, 0x4a,
0x48, 0x76, 0x0d, 0xc1, 0x2c, 0xa7, 0x19, 0x9c, 0xbd, 0x2f, 0x4a, 0xbc, 0x69, 0x84, 0xb2, 0x4d,
0x99, 0x90, 0x8c, 0x4c, 0x46, 0xbc, 0x8f, 0x5c, 0xc5, 0x95, 0x96, 0x4d, 0xa9, 0x5e, 0x8b, 0x25,
0xca, 0x24, 0x68, 0x2b, 0x7a, 0x88, 0x95, 0x10, 0xbd, 0x32, 0xa2, 0xc4, 0x3b, 0x0c, 0xbb, 0x84,
0x21, 0xd7, 0xb7, 0xfd, 0x49, 0xc7, 0x98, 0x3e, 0x86, 0x78, 0xa1, 0x36, 0x68, 0x6a, 0x9c, 0x2b,
0xb1, 0x94, 0xf8, 0x31, 0x19, 0x64, 0x64, 0x32, 0xe4, 0x7f, 0x51, 0xc6, 0x20, 0x5e, 0x94, 0x95,
0x36, 0x96, 0x63, 0x5d, 0x69, 0x55, 0x23, 0x1d, 0xc3, 0x60, 0x6e, 0x4c, 0xb7, 0xcf, 0x3d, 0xd9,
0x16, 0xc6, 0xb9, 0xd4, 0xab, 0xf5, 0x4c, 0x58, 0xc1, 0xf1, 0x4b, 0x83, 0xb5, 0xa5, 0xb1, 0x3b,
0xb8, 0x2b, 0x72, 0xa7, 0x5f, 0x74, 0xb2, 0x3b, 0x21, 0xdd, 0x0d, 0x17, 0x10, 0xf9, 0x4e, 0xbf,
0x3c, 0xe4, 0x6d, 0xe0, 0xe8, 0x3b, 0x59, 0xac, 0x30, 0x09, 0x5b, 0xea, 0x03, 0x4a, 0x21, 0xfc,
0x50, 0xe0, 0x6d, 0x12, 0xf9, 0x01, 0xfe, 0xcd, 0x6e, 0xe0, 0xbc, 0xb7, 0xb9, 0x13, 0xf8, 0x10,
0x46, 0x79, 0x61, 0x4b, 0x51, 0x2d, 0x66, 0x75, 0x42, 0xb2, 0xc1, 0x24, 0xe4, 0x27, 0x40, 0x53,
0x80, 0xb7, 0x46, 0x7f, 0x2a, 0x24, 0xba, 0x74, 0xe0, 0xd3, 0x3d, 0xc2, 0x1e, 0x41, 0x74, 0x25,
0x56, 0x9f, 0xff, 0x33, 0x86, 0x7d, 0x23, 0x70, 0xfe, 0x46, 0x6c, 0xbd, 0xb4, 0xfa, 0xb8, 0xfa,
0x1a, 0x46, 0x47, 0xe8, 0x7b, 0xce, 0x9e, 0x3e, 0x99, 0x1e, 0x3e, 0xc1, 0xf4, 0x9f, 0xfa, 0x13,
0x99, 0x2b, 0x6b, 0x76, 0xfc, 0xd4, 0x7c, 0xf9, 0x12, 0xe2, 0x3f, 0x93, 0xce, 0xf7, 0x35, 0xee,
0x0e, 0xbe, 0xaf, 0x71, 0xe7, 0x7c, 0xda, 0x08, 0xd9, 0xb4, 0x9e, 0x86, 0xbc, 0x0d, 0x5e, 0x04,
0xcf, 0x49, 0x3e, 0xfe, 0xbe, 0x4f, 0xc9, 0x8f, 0x7d, 0x4a, 0x7e, 0xee, 0x53, 0xf2, 0xf5, 0x57,
0x7a, 0x6f, 0x79, 0xdf, 0xff, 0xc8, 0x67, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1e, 0x5e,
0x10, 0xa2, 0x02, 0x00, 0x00,
// 406 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xd1, 0x8a, 0xd3, 0x40,
0x14, 0x86, 0x9d, 0x36, 0x95, 0xf6, 0x2c, 0x86, 0xee, 0xb0, 0x17, 0x61, 0x59, 0x42, 0x18, 0x50,
0x8a, 0x17, 0xbd, 0xd0, 0x1b, 0x11, 0xaf, 0xb2, 0xad, 0x6c, 0x41, 0xc1, 0x1d, 0x17, 0xef, 0xa7,
0xf5, 0x88, 0xa1, 0x93, 0x99, 0x38, 0x99, 0x74, 0x37, 0xcf, 0xe0, 0x0b, 0x08, 0xbe, 0x90, 0x97,
0x3e, 0x82, 0xd4, 0x17, 0x91, 0x99, 0xa4, 0x4d, 0x54, 0x10, 0xef, 0xe6, 0x7c, 0x73, 0xce, 0x3f,
0xff, 0xf9, 0x19, 0x78, 0x50, 0x98, 0x6c, 0x27, 0x2c, 0xce, 0x0b, 0xa3, 0xad, 0xa6, 0xe3, 0x4c,
0x59, 0x34, 0x4a, 0x48, 0x76, 0x05, 0x83, 0x45, 0x4a, 0x13, 0x38, 0xb9, 0xc9, 0x72, 0xbc, 0xae,
0x84, 0xb2, 0x55, 0x1e, 0x91, 0x84, 0xcc, 0x26, 0xbc, 0x8f, 0x5c, 0xc7, 0xa5, 0x96, 0x55, 0xae,
0x5e, 0x89, 0x35, 0xca, 0x68, 0xd0, 0x74, 0xf4, 0x10, 0xfb, 0x4c, 0x60, 0xf4, 0xd2, 0x88, 0x1c,
0xff, 0x43, 0xed, 0x1c, 0xc6, 0x5c, 0xdf, 0xf6, 0xa5, 0x8e, 0x35, 0x7d, 0x04, 0xe1, 0x4a, 0xed,
0xd0, 0x94, 0xb8, 0x54, 0x62, 0x2d, 0xf1, 0x7d, 0x34, 0x4c, 0xc8, 0x6c, 0xcc, 0xff, 0xa0, 0xf4,
0x02, 0x26, 0x97, 0x62, 0xf3, 0x11, 0x6f, 0xea, 0x02, 0xa3, 0xc0, 0x8b, 0x74, 0x80, 0x31, 0x08,
0x57, 0x79, 0xa1, 0x8d, 0xe5, 0x58, 0x16, 0x5a, 0x95, 0x48, 0xa7, 0x30, 0x5c, 0x1a, 0xd3, 0xba,
0x71, 0x47, 0x76, 0x07, 0xd3, 0x54, 0xea, 0xcd, 0x76, 0x21, 0xac, 0xe0, 0xf8, 0xa9, 0xc2, 0xd2,
0xd2, 0xd0, 0xe5, 0xd1, 0x36, 0xb9, 0x64, 0xce, 0xda, 0xa5, 0x5a, 0x9b, 0xed, 0x86, 0x67, 0x30,
0xf2, 0x93, 0xde, 0x5a, 0xc0, 0x9b, 0xc2, 0xd1, 0xb7, 0x32, 0xdb, 0x34, 0x6e, 0x02, 0xde, 0x14,
0x94, 0x42, 0xf0, 0x2e, 0xc3, 0xdb, 0x68, 0xe4, 0x05, 0xfc, 0x99, 0x5d, 0xc3, 0x69, 0xef, 0xe5,
0xd6, 0xe0, 0x05, 0x4c, 0xd2, 0xcc, 0xe6, 0xa2, 0x58, 0x2d, 0xca, 0x88, 0x24, 0xc3, 0x59, 0xc0,
0x3b, 0x40, 0x63, 0x80, 0x37, 0x46, 0x7f, 0xc8, 0x24, 0xba, 0xeb, 0x81, 0xbf, 0xee, 0x11, 0xf6,
0x10, 0x46, 0x7e, 0xfb, 0x7f, 0xcb, 0xb0, 0xaf, 0x04, 0x4e, 0x5f, 0x8b, 0x3b, 0x6f, 0xad, 0x3c,
0x3e, 0x7d, 0x05, 0x93, 0x23, 0xf4, 0x33, 0x27, 0x4f, 0x1e, 0xcf, 0x0f, 0x7f, 0x64, 0xfe, 0x57,
0x7f, 0x47, 0x96, 0xca, 0x9a, 0x9a, 0x77, 0xc3, 0xe7, 0x2f, 0x20, 0xfc, 0xfd, 0xd2, 0xe5, 0xbe,
0xc5, 0xfa, 0x90, 0xfb, 0x16, 0x6b, 0x97, 0xd3, 0x4e, 0xc8, 0xaa, 0xc9, 0x34, 0xe0, 0x4d, 0xf1,
0x7c, 0xf0, 0x8c, 0xa4, 0xd3, 0x6f, 0xfb, 0x98, 0x7c, 0xdf, 0xc7, 0xe4, 0xc7, 0x3e, 0x26, 0x5f,
0x7e, 0xc6, 0xf7, 0xd6, 0xf7, 0xfd, 0x87, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x99, 0x45,
0xbe, 0x6d, 0xc1, 0x02, 0x00, 0x00,
}

View file

@ -11,6 +11,7 @@ message Frame {
string TimeQuantum = 1;
string RowLabel = 2;
bool InverseEnabled = 3;
string CacheType = 4;
}
message ImportResponse {

View file

@ -2616,7 +2616,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 572 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd4, 0x3c,
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd4, 0x3c,
0x14, 0xfd, 0x3c, 0xc9, 0xfc, 0xdd, 0x69, 0xab, 0x7e, 0x16, 0xa0, 0x08, 0xa1, 0x51, 0x14, 0xb1,
0xc8, 0x86, 0xa9, 0x54, 0x1e, 0x00, 0x91, 0x4e, 0x2b, 0x8d, 0x10, 0x55, 0xeb, 0x16, 0x76, 0x2c,
0xd2, 0x62, 0x8a, 0xa5, 0xfc, 0x61, 0x3b, 0x8b, 0x59, 0xb2, 0xe0, 0x1d, 0x10, 0x6f, 0x00, 0x4f,

View file

@ -21,7 +21,8 @@ var (
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrInvalidView = errors.New("invalid view")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]")

View file

@ -31,6 +31,7 @@ type View struct {
name string
// Fragments by slice.
cacheType string // passed in by frame
fragments map[uint64]*Fragment
stats StatsClient
@ -47,6 +48,7 @@ func NewView(path, db, frame, name string) *View {
frame: frame,
name: name,
cacheType: DefaultCacheType,
fragments: make(map[uint64]*Fragment),
stats: NopStatsClient,
@ -212,6 +214,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.db, v.frame, v.name, slice)
frag.cacheType = v.cacheType
frag.LogOutput = v.LogOutput
frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
return frag