From 912fcb34649f1e41741734d400b769e980cbdff2 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 28 Mar 2017 13:01:04 -0600 Subject: [PATCH] Separate max slices for standard and inverse views. Previously only the standard max slice was available. This commit changes it so that the inverse max can be retrieved separately through the `DB` and `Frame` types as well as through the `HTTP` API and `Client`. --- client.go | 13 +++++++ db.go | 90 ++++++++++++++++++++----------------------------- frame.go | 12 +++++++ handler.go | 8 ++++- handler_test.go | 40 ++++++++++++++++++++++ index.go | 9 +++++ 6 files changed, 118 insertions(+), 54 deletions(-) diff --git a/client.go b/client.go index fa6e38cb1..29005c10d 100644 --- a/client.go +++ b/client.go @@ -47,11 +47,24 @@ func (c *Client) Host() string { return c.host } // MaxSliceByDatabase returns the number of slices on a server by database. func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByDatabase(ctx, false) +} + +// MaxInverseSliceByDatabase returns the number of inverse slices on a server by database. +func (c *Client) MaxInverseSliceByDatabase(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByDatabase(ctx, true) +} + +// maxSliceByDatabase returns the number of slices on a server by database. +func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. u := url.URL{ Scheme: "http", Host: c.host, Path: "/slices/max", + RawQuery: (&url.Values{ + "inverse": {strconv.FormatBool(inverse)}, + }).Encode(), } // Build request. diff --git a/db.go b/db.go index c728b674a..36fb38a94 100644 --- a/db.go +++ b/db.go @@ -37,7 +37,8 @@ type DB struct { frames map[string]*Frame // Max Slice on any node in the cluster, according to this node - remoteMaxSlice uint64 + remoteMaxSlice uint64 + remoteMaxInverseSlice uint64 // Profile attribute storage and cache profileAttrStore *AttrStore @@ -55,10 +56,12 @@ func NewDB(path, name string) (*DB, error) { } return &DB{ - path: path, - name: name, - frames: make(map[string]*Frame), - remoteMaxSlice: 0, + path: path, + name: name, + frames: make(map[string]*Frame), + + remoteMaxSlice: 0, + remoteMaxInverseSlice: 0, profileAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -240,6 +243,35 @@ func (db *DB) MaxSlice() uint64 { return max } +func (db *DB) SetRemoteMaxSlice(v uint64) { + db.mu.Lock() + defer db.mu.Unlock() + db.remoteMaxSlice = v +} + +// MaxInverseSlice returns the max inverse slice in the database according to this node. +func (db *DB) MaxInverseSlice() uint64 { + if db == nil { + return 0 + } + db.mu.Lock() + defer db.mu.Unlock() + + max := db.remoteMaxInverseSlice + for _, f := range db.frames { + if slice := f.MaxInverseSlice(); slice > max { + max = slice + } + } + return max +} + +func (db *DB) SetRemoteMaxInverseSlice(v uint64) { + db.mu.Lock() + defer db.mu.Unlock() + db.remoteMaxInverseSlice = v +} + // TimeQuantum returns the default time quantum for the database. func (db *DB) TimeQuantum() TimeQuantum { db.mu.Lock() @@ -386,48 +418,6 @@ func (db *DB) DeleteFrame(name string) error { return nil } -/* -// SetBit sets a bit for a given profile & bitmap. -// If a timestamp is specified then set all bits for the different quantum units. -func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { - // Read frame. - f := db.Frame(name) - if f == nil { - return changed, ErrFrameNotFound - } - - // If this is a non-time bit then simply set the bit on the frame. - if t == nil { - return f.SetBit(bitmapID, profileID) - } - - // Determine quantum of frame. Set to the default quantum if it is unset. - q := f.TimeQuantum() - if q == "" { - q = db.TimeQuantum() - if err := f.SetTimeQuantum(q); err != nil { - return changed, err - } - } - - // If a timestamp is specified then set bits across all frames for the quantum. - opt := f.Options() - for _, subname := range ViewsByTime(name, *t, q) { - f, err := db.CreateFrameIfNotExists(subname, opt) - if err != nil { - return changed, err - } - - if c, err := f.SetBit(bitmapID, profileID); err != nil { - return changed, err - } else if c { - changed = true - } - } - return changed, nil -} -*/ - type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -486,12 +476,6 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { return dbs } -func (db *DB) SetRemoteMaxSlice(newmax uint64) { - db.mu.Lock() - defer db.mu.Unlock() - db.remoteMaxSlice = newmax -} - // DBOptions represents options to set when initializing a db. type DBOptions struct { ColumnLabel string `json:"columnLabel,omitempty"` diff --git a/frame.go b/frame.go index 1d444a8bb..9d0fccf50 100644 --- a/frame.go +++ b/frame.go @@ -93,6 +93,18 @@ func (f *Frame) MaxSlice() uint64 { return view.MaxSlice() } +// MaxInverseSlice returns the max inverse slice in the frame. +func (f *Frame) MaxInverseSlice() uint64 { + f.mu.Lock() + defer f.mu.Unlock() + + view := f.views[ViewInverse] + if view == nil { + return 0 + } + return view.MaxSlice() +} + // SetRowLabel sets the row labels. Persists to meta file on update. func (f *Frame) SetRowLabel(v string) error { f.mu.Lock() diff --git a/handler.go b/handler.go index 03c56ad09..400d9c48d 100644 --- a/handler.go +++ b/handler.go @@ -292,7 +292,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) error { - ms := h.Index.MaxSlices() + var ms map[string]uint64 + if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { + ms = h.Index.MaxInverseSlices() + } else { + ms = h.Index.MaxSlices() + } + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { pb := &internal.MaxSlicesResponse{ MaxSlices: ms, diff --git a/handler_test.go b/handler_test.go index 2fdba657b..20f76de05 100644 --- a/handler_test.go +++ b/handler_test.go @@ -86,6 +86,46 @@ func TestHandler_MaxSlices(t *testing.T) { } } +// Ensure the handler can return the maxslice map for the inverse views. +func TestHandler_MaxSlices_Inverse(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + f0, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := f0.SetBit((1*SliceWidth)+1, 30, nil); err != nil { + t.Fatal(err) + } else if _, err := f0.SetBit((1*SliceWidth)+2, 30, nil); err != nil { + t.Fatal(err) + } else if _, err := f0.SetBit((3*SliceWidth)+4, 30, nil); err != nil { + t.Fatal(err) + } + + f1, err := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := f1.SetBit((0*SliceWidth)+1, 40, nil); err != nil { + t.Fatal(err) + } else if _, err := f1.SetBit((0*SliceWidth)+2, 40, nil); err != nil { + t.Fatal(err) + } else if _, err := f1.SetBit((0*SliceWidth)+4, 40, nil); err != nil { + t.Fatal(err) + } + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() diff --git a/index.go b/index.go index 3b45dd438..7597984ba 100644 --- a/index.go +++ b/index.go @@ -125,6 +125,15 @@ func (i *Index) MaxSlices() map[string]uint64 { return a } +// MaxInverseSlices returns MaxInverseSlice map for all databases. +func (i *Index) MaxInverseSlices() map[string]uint64 { + a := make(map[string]uint64) + for _, db := range i.DBs() { + a[db.Name()] = db.MaxInverseSlice() + } + return a +} + // Schema returns schema data for all databases and frames. func (i *Index) Schema() []*DBInfo { var a []*DBInfo