// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pilosa import ( "fmt" "io" "io/ioutil" "os" "path/filepath" "strconv" "strings" "sync" "github.com/pilosa/pilosa/internal" ) // View layout modes. const ( ViewStandard = "standard" ViewInverse = "inverse" ) // IsValidView returns true if name is valid. func IsValidView(name string) bool { return name == ViewStandard || name == ViewInverse } // View represents a container for frame data. type View struct { mu sync.Mutex path string index string frame string name string cacheSize uint32 // Fragments by slice. cacheType string // passed in by frame fragments map[uint64]*Fragment // maxSlice maintains this view's max slice in order to // prevent sending multiple `CreateSliceMessage` messages maxSlice uint64 broadcaster Broadcaster stats StatsClient RowAttrStore *AttrStore LogOutput io.Writer } // NewView returns a new instance of View. func NewView(path, index, frame, name string, cacheSize uint32) *View { return &View{ path: path, index: index, frame: frame, name: name, cacheSize: cacheSize, cacheType: DefaultCacheType, fragments: make(map[uint64]*Fragment), broadcaster: NopBroadcaster, stats: NopStatsClient, LogOutput: ioutil.Discard, } } // Name returns the name the view was initialized with. func (v *View) Name() string { return v.name } // Index returns the index name the view was initialized with. func (v *View) Index() string { return v.index } // Frame returns the frame name the view was initialized with. func (v *View) Frame() string { return v.frame } // Path returns the path the view was initialized with. func (v *View) Path() string { return v.path } // Open opens and initializes the view. func (v *View) Open() error { if err := func() error { // Ensure the view's path exists. if err := os.MkdirAll(v.path, 0777); err != nil { return err } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { return err } if err := v.openFragments(); err != nil { return err } return nil }(); err != nil { v.Close() return err } return nil } // openFragments opens and initializes the fragments inside the view. func (v *View) openFragments() error { file, err := os.Open(filepath.Join(v.path, "fragments")) if os.IsNotExist(err) { return nil } else if err != nil { return err } defer file.Close() fis, err := file.Readdir(0) if err != nil { return err } for _, fi := range fis { if fi.IsDir() { continue } // Parse filename into integer. slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) if err != nil { continue } frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) } frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag } return nil } // Close closes the view and its fragments. func (v *View) Close() error { v.mu.Lock() defer v.mu.Unlock() // Close all fragments. for _, frag := range v.fragments { _ = frag.Close() } v.fragments = make(map[uint64]*Fragment) return nil } // MaxSlice returns the max slice in the view. func (v *View) MaxSlice() uint64 { v.mu.Lock() defer v.mu.Unlock() var max uint64 for slice := range v.fragments { if slice > max { max = slice } } return max } // FragmentPath returns the path to a fragment in the view. func (v *View) FragmentPath(slice uint64) string { return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) } // Fragment returns a fragment in the view by slice. func (v *View) Fragment(slice uint64) *Fragment { v.mu.Lock() defer v.mu.Unlock() return v.fragment(slice) } func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } // Fragments returns a list of all fragments in the view. func (v *View) Fragments() []*Fragment { v.mu.Lock() defer v.mu.Unlock() other := make([]*Fragment, 0, len(v.fragments)) for _, fragment := range v.fragments { other = append(other, fragment) } return other } // CreateFragmentIfNotExists returns a fragment in the view by slice. func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { v.mu.Lock() defer v.mu.Unlock() return v.createFragmentIfNotExists(slice) } func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { // Find fragment in cache first. if frag := v.fragments[slice]; frag != nil { return frag, nil } // Initialize and open fragment. frag := v.newFragment(v.FragmentPath(slice), slice) if err := frag.Open(); err != nil { return nil, err } frag.RowAttrStore = v.RowAttrStore // Broadcast a message that a new max slice was just created. if slice > v.maxSlice { v.maxSlice = slice // Send the create slice message to all nodes. err := v.broadcaster.SendAsync( &internal.CreateSliceMessage{ Index: v.index, Slice: slice, IsInverse: IsInverseView(v.name), }) if err != nil { return nil, err } } // Save to lookup. v.fragments[slice] = frag return frag, nil } func (v *View) newFragment(path string, slice uint64) *Fragment { frag := NewFragment(path, v.index, v.frame, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag } // SetBit sets a bit within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } return frag.SetBit(rowID, columnID) } // ClearBit clears a bit within the view. func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } return frag.ClearBit(rowID, columnID) } // IsInverseView returns true if the view is used for storing an inverted representation. func IsInverseView(name string) bool { return strings.HasPrefix(name, ViewInverse) } type viewSlice []*View func (p viewSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p viewSlice) Len() int { return len(p) } func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // ViewInfo represents schema information for a view. type ViewInfo struct { Name string `json:"name"` } type viewInfoSlice []*ViewInfo func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p viewInfoSlice) Len() int { return len(p) } func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }