Separate max slices for standard and inverse views.

Previously only the standard max slice was available. This commit
changes it so that the inverse max can be retrieved separately
through the `DB` and `Frame` types as well as through the
`HTTP` API and `Client`.
This commit is contained in:
Ben Johnson 2017-03-28 13:01:04 -06:00
parent 07ce417f55
commit 912fcb3464
6 changed files with 118 additions and 54 deletions

View file

@ -47,11 +47,24 @@ func (c *Client) Host() string { return c.host }
// MaxSliceByDatabase returns the number of slices on a server by database.
func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByDatabase(ctx, false)
}
// MaxInverseSliceByDatabase returns the number of inverse slices on a server by database.
func (c *Client) MaxInverseSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByDatabase(ctx, true)
}
// maxSliceByDatabase returns the number of slices on a server by database.
func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[string]uint64, error) {
// Execute request against the host.
u := url.URL{
Scheme: "http",
Host: c.host,
Path: "/slices/max",
RawQuery: (&url.Values{
"inverse": {strconv.FormatBool(inverse)},
}).Encode(),
}
// Build request.

90
db.go
View file

@ -37,7 +37,8 @@ type DB struct {
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node
remoteMaxSlice uint64
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
// Profile attribute storage and cache
profileAttrStore *AttrStore
@ -55,10 +56,12 @@ func NewDB(path, name string) (*DB, error) {
}
return &DB{
path: path,
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
path: path,
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
profileAttrStore: NewAttrStore(filepath.Join(path, ".data")),
@ -240,6 +243,35 @@ func (db *DB) MaxSlice() uint64 {
return max
}
func (db *DB) SetRemoteMaxSlice(v uint64) {
db.mu.Lock()
defer db.mu.Unlock()
db.remoteMaxSlice = v
}
// MaxInverseSlice returns the max inverse slice in the database according to this node.
func (db *DB) MaxInverseSlice() uint64 {
if db == nil {
return 0
}
db.mu.Lock()
defer db.mu.Unlock()
max := db.remoteMaxInverseSlice
for _, f := range db.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
func (db *DB) SetRemoteMaxInverseSlice(v uint64) {
db.mu.Lock()
defer db.mu.Unlock()
db.remoteMaxInverseSlice = v
}
// TimeQuantum returns the default time quantum for the database.
func (db *DB) TimeQuantum() TimeQuantum {
db.mu.Lock()
@ -386,48 +418,6 @@ func (db *DB) DeleteFrame(name string) error {
return nil
}
/*
// SetBit sets a bit for a given profile & bitmap.
// If a timestamp is specified then set all bits for the different quantum units.
func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) {
// Read frame.
f := db.Frame(name)
if f == nil {
return changed, ErrFrameNotFound
}
// If this is a non-time bit then simply set the bit on the frame.
if t == nil {
return f.SetBit(bitmapID, profileID)
}
// Determine quantum of frame. Set to the default quantum if it is unset.
q := f.TimeQuantum()
if q == "" {
q = db.TimeQuantum()
if err := f.SetTimeQuantum(q); err != nil {
return changed, err
}
}
// If a timestamp is specified then set bits across all frames for the quantum.
opt := f.Options()
for _, subname := range ViewsByTime(name, *t, q) {
f, err := db.CreateFrameIfNotExists(subname, opt)
if err != nil {
return changed, err
}
if c, err := f.SetBit(bitmapID, profileID); err != nil {
return changed, err
} else if c {
changed = true
}
}
return changed, nil
}
*/
type dbSlice []*DB
func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -486,12 +476,6 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo {
return dbs
}
func (db *DB) SetRemoteMaxSlice(newmax uint64) {
db.mu.Lock()
defer db.mu.Unlock()
db.remoteMaxSlice = newmax
}
// DBOptions represents options to set when initializing a db.
type DBOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`

View file

@ -93,6 +93,18 @@ func (f *Frame) MaxSlice() uint64 {
return view.MaxSlice()
}
// MaxInverseSlice returns the max inverse slice in the frame.
func (f *Frame) MaxInverseSlice() uint64 {
f.mu.Lock()
defer f.mu.Unlock()
view := f.views[ViewInverse]
if view == nil {
return 0
}
return view.MaxSlice()
}
// SetRowLabel sets the row labels. Persists to meta file on update.
func (f *Frame) SetRowLabel(v string) error {
f.mu.Lock()

View file

@ -292,7 +292,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) error {
ms := h.Index.MaxSlices()
var ms map[string]uint64
if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse {
ms = h.Index.MaxInverseSlices()
} else {
ms = h.Index.MaxSlices()
}
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
pb := &internal.MaxSlicesResponse{
MaxSlices: ms,

View file

@ -86,6 +86,46 @@ func TestHandler_MaxSlices(t *testing.T) {
}
}
// Ensure the handler can return the maxslice map for the inverse views.
func TestHandler_MaxSlices_Inverse(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
f0, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := f0.SetBit((1*SliceWidth)+1, 30, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit((1*SliceWidth)+2, 30, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit((3*SliceWidth)+4, 30, nil); err != nil {
t.Fatal(err)
}
f1, err := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := f1.SetBit((0*SliceWidth)+1, 40, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit((0*SliceWidth)+2, 40, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit((0*SliceWidth)+4, 40, nil); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Index = idx.Index
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()

View file

@ -125,6 +125,15 @@ func (i *Index) MaxSlices() map[string]uint64 {
return a
}
// MaxInverseSlices returns MaxInverseSlice map for all databases.
func (i *Index) MaxInverseSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, db := range i.DBs() {
a[db.Name()] = db.MaxInverseSlice()
}
return a
}
// Schema returns schema data for all databases and frames.
func (i *Index) Schema() []*DBInfo {
var a []*DBInfo