From eff08af1125ba9b4c5c0a63a2b256bae76f81ca5 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 20 Mar 2017 15:30:12 -0600 Subject: [PATCH 1/3] Separate physical data layout with views. Previously, multiple frames with different prefixes were used to separate different data layouts. This included separating standard row/column layouts from inverted column/row layouts as well as storing aggregate information for timestamp data. Unfortunately, this caused frame meta data to be copied between multiple frames and it made it difficult to keep these frames in sync. This commit separates these different physical layouts into `Views`. A `Frame` now has one or more views which represent each layout. Fragments have been moved from under the `Frame` to be contained within the `View`. There are two primary views: - `standard` - `inverse` If a frame has a time quantum, then views are generated for these each of the standard/inverse views. For example a time quantum of `YMDH` for the date `2000-01-02T00:00:00Z` would create the following views: - `standard_2000` - `inverse_2000` - `standard_200001` - `inverse_200001` - `standard_20000102` - `inverse_20000102` From the user's perspective, nothing should change in PQL. Different PQL statements will handle the appropriate view automatically. For example, `Bitmap()` and `Profile()` will fetch using the `standard` view or the `inverse` view, respectively. The `Range()` statement will lookup the appropriate time-based views automatically. --- client.go | 71 ++++++++-- client_test.go | 74 +++++----- cmd/backup.go | 3 +- cmd/restore.go | 3 +- ctl/backup.go | 7 +- ctl/restore.go | 3 +- db.go | 87 +++--------- db_test.go | 24 ++-- executor.go | 22 +-- executor_test.go | 190 +++++++++++++------------- fragment.go | 19 ++- fragment_test.go | 48 +++---- frame.go | 293 ++++++++++++++++++++++++++++++---------- frame_test.go | 47 ++++--- handler.go | 152 +++++++++++++++------ handler_test.go | 32 +++-- index.go | 79 +++++++---- index_test.go | 30 ++-- internal/internal.pb.go | 189 ++++++++++++++++---------- internal/internal.proto | 5 +- time.go | 32 ++--- time_test.go | 40 +++--- view.go | 249 ++++++++++++++++++++++++++++++++++ view_test.go | 80 +++++++++++ 24 files changed, 1213 insertions(+), 566 deletions(-) create mode 100644 view.go create mode 100644 view_test.go diff --git a/client.go b/client.go index e7d0eaf85..fa6e38cb1 100644 --- a/client.go +++ b/client.go @@ -430,7 +430,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string } // BackupTo backs up an entire frame from a cluster to w. -func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) error { +func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error { if db == "" { return ErrDatabaseRequired } else if frame == "" { @@ -448,7 +448,7 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er // Backup every slice to the tar file. for i := uint64(0); i <= maxSlices[db]; i++ { - if err := c.backupSliceTo(ctx, tw, db, frame, i); err != nil { + if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil { return err } } @@ -462,9 +462,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er } // backupSliceTo backs up a single slice to tw. -func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame string, slice uint64) error { +func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error { // Return error if unable to backup from any slice. - r, err := c.BackupSlice(ctx, db, frame, slice) + r, err := c.BackupSlice(ctx, db, frame, view, slice) if err != nil { return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err) } else if r == nil { @@ -500,7 +500,7 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame st // BackupSlice retrieves a streaming backup from a single slice. // This function tries slice owners until one succeeds. -func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64) (io.ReadCloser, error) { +func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, db, slice) if err != nil { @@ -509,7 +509,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64 // Try to backup slice from each one until successful. for _, i := range rand.Perm(len(nodes)) { - r, err := c.backupSliceNode(ctx, db, frame, slice, nodes[i]) + r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i]) if err == nil { return r, nil // successfully attached } else if err == ErrFragmentNotFound { @@ -523,7 +523,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64 return nil, fmt.Errorf("unable to connect to any owner") } -func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := url.URL{ Scheme: "http", Host: node.Host, @@ -531,6 +531,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -560,7 +561,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui } // RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string) error { +func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error { if db == "" { return ErrDatabaseRequired } else if frame == "" { @@ -592,14 +593,14 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string) } // Restore file to all nodes that own it. - if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, slice); err != nil { + if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil { return err } } } // restoreSliceFrom restores a single slice to all owning nodes. -func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame string, slice uint64) error { +func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, db, slice) if err != nil { @@ -615,6 +616,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -726,9 +728,52 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error return nil } +// FrameViews returns a list of view names for a frame. +func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) { + // Create URL & HTTP request. + u := url.URL{ + Scheme: "http", + Host: c.host, + Path: "/frame/views", + RawQuery: (&url.Values{ + "db": {db}, + "frame": {frame}, + }).Encode(), + } + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, ErrFrameNotFound + default: + body, _ := ioutil.ReadAll(resp.Body) + return nil, errors.New(string(body)) + } + + // Decode response. + var rsp getFrameViewsResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, err + } + return rsp.Views, nil +} + // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uint64) ([]FragmentBlock, error) { +func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -736,6 +781,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -771,10 +817,11 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin } // BlockData returns bitmap/profile id pairs for a block. -func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ DB: db, Frame: frame, + View: view, Slice: slice, Block: uint64(block), }) diff --git a/client_test.go b/client_test.go index 15af73d8a..f808514a0 100644 --- a/client_test.go +++ b/client_test.go @@ -61,33 +61,33 @@ func TestClient_MultiNode(t *testing.T) { } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(99, 1, 2, 3, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(1, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(22, 1, 2, 3, 4, 5) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(100, (SliceWidth*10)+10) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(21, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(100, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).RecalculateCache() - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).RecalculateCache() - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).RecalculateCache() + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache() + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache() + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -161,7 +161,7 @@ func TestClient_Import(t *testing.T) { defer idx.Close() // Load bitmap into cache to ensure cache gets updated. - f := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f.Bitmap(0) s := NewServer() @@ -195,10 +195,10 @@ func TestClient_BackupRestore(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(100, SliceWidth, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).MustSetBits(100, (5*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(200, 20000) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) s := NewServer() defer s.Close() @@ -211,7 +211,7 @@ func TestClient_BackupRestore(t *testing.T) { // Backup from frame. var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "d", "f"); err != nil { + if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil { t.Fatal(err) } @@ -219,21 +219,21 @@ func TestClient_BackupRestore(t *testing.T) { if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil { + if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { t.Fatal(err) } // Verify data. - if a := idx.Fragment("x", "y", 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -244,11 +244,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer idx.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := NewServer() defer s.Close() @@ -259,7 +259,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := MustNewClient(s.Host()) - blocks, err := c.FragmentBlocks(context.Background(), "d", "f", 0) + blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { @@ -271,7 +271,7 @@ func TestClient_FragmentBlocks(t *testing.T) { } // Verify data matches local blocks. - if a := idx.Fragment("d", "f", 0).Blocks(); !reflect.DeepEqual(a, blocks) { + if a := idx.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } diff --git a/cmd/backup.go b/cmd/backup.go index ac1b19d7f..24904cbb5 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -18,7 +18,7 @@ func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Use: "backup", Short: "Backup data from pilosa.", Long: ` -Backs up the database and frame from across the cluster into a single file. +Backs up the view from across the cluster into a single file. `, RunE: func(cmd *cobra.Command, args []string) error { if err := Backuper.Run(context.Background()); err != nil { @@ -31,6 +31,7 @@ Backs up the database and frame from across the cluster into a single file. flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.") flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.") + flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.") flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") return backupCmd diff --git a/cmd/restore.go b/cmd/restore.go index a4930f668..5179cbc0d 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -19,7 +19,7 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Use: "restore", Short: "Restore data to pilosa from a backup file.", Long: ` -Restores a frame to the cluster from a backup file. +Restores a view to the cluster from a backup file. `, RunE: func(cmd *cobra.Command, args []string) error { if err := Restorer.Run(context.Background()); err != nil { @@ -32,6 +32,7 @@ Restores a frame to the cluster from a backup file. flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.") flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.") + flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.") flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.") return restoreCmd diff --git a/ctl/backup.go b/ctl/backup.go index 061982aa8..6eeed5d4c 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -9,14 +9,15 @@ import ( "github.com/pilosa/pilosa" ) -// BackupCommand represents a command for backing up a frame. +// BackupCommand represents a command for backing up a view. type BackupCommand struct { // Destination host and port. Host string - // Name of the database & frame to backup. + // Name of the database, frame, view to backup. Database string Frame string + View string // Output file to write to. Path string @@ -53,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error { defer f.Close() // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil { + if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { return err } diff --git a/ctl/restore.go b/ctl/restore.go index c764c213c..56dfb3080 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -17,6 +17,7 @@ type RestoreCommand struct { // Name of the database & frame to backup. Database string Frame string + View string // Import file to read from. Path string @@ -53,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error { defer f.Close() // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil { + if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { return err } diff --git a/db.go b/db.go index fef92e755..c728b674a 100644 --- a/db.go +++ b/db.go @@ -386,6 +386,7 @@ func (db *DB) DeleteFrame(name string) error { return nil } +/* // SetBit sets a bit for a given profile & bitmap. // If a timestamp is specified then set all bits for the different quantum units. func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { @@ -411,7 +412,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha // If a timestamp is specified then set bits across all frames for the quantum. opt := f.Options() - for _, subname := range FramesByTime(name, *t, q) { + for _, subname := range ViewsByTime(name, *t, q) { f, err := db.CreateFrameIfNotExists(subname, opt) if err != nil { return changed, err @@ -425,67 +426,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha } return changed, nil } - -// Import bulk imports data. -func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { - // Read frame. - f := db.Frame(name) - if f == nil { - return ErrFrameNotFound - } - - // Determine quantum if timestamps are set. - var q TimeQuantum - if hasTime(timestamps) { - if q = f.TimeQuantum(); q == "" { - q = db.TimeQuantum() - if err := f.SetTimeQuantum(q); err != nil { - return err - } - } - - if q == "" { - return errors.New("time quantum not set in either database or frame") - } - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importData) - for i := range bitmapIDs { - bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] - slice := profileID / SliceWidth - - var names []string - if timestamp == nil { - names = []string{name} - } else { - names = FramesByTime(name, *timestamp, q) - } - - // Attach bit to each frame. - for _, name := range names { - key := importKey{Frame: name, Slice: slice} - data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, bitmapID) - data.ProfileIDs = append(data.ProfileIDs, profileID) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - frag, err := f.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return err - } - - if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { - return err - } - } - - return nil -} +*/ type dbSlice []*DB @@ -508,14 +449,19 @@ func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // MergeSchemas combines databases and frames from a and b into one schema. func MergeSchemas(a, b []*DBInfo) []*DBInfo { // Generate a map from both schemas. - m := make(map[string]map[string]struct{}) + m := make(map[string]map[string]map[string]struct{}) for _, dbs := range [][]*DBInfo{a, b} { for _, db := range dbs { if m[db.Name] == nil { - m[db.Name] = make(map[string]struct{}) + m[db.Name] = make(map[string]map[string]struct{}) } for _, frame := range db.Frames { - m[db.Name][frame.Name] = struct{}{} + if m[db.Name][frame.Name] == nil { + m[db.Name][frame.Name] = make(map[string]struct{}) + } + for _, view := range frame.Views { + m[db.Name][frame.Name][view.Name] = struct{}{} + } } } } @@ -524,8 +470,13 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { dbs := make([]*DBInfo, 0, len(m)) for db, frames := range m { di := &DBInfo{Name: db} - for frame := range frames { - di.Frames = append(di.Frames, &FrameInfo{Name: frame}) + for frame, views := range frames { + fi := &FrameInfo{Name: frame} + for view := range views { + fi.Views = append(fi.Views, &ViewInfo{Name: view}) + } + sort.Sort(viewInfoSlice(fi.Views)) + di.Frames = append(di.Frames, fi) } sort.Sort(frameInfoSlice(di.Frames)) dbs = append(dbs, di) @@ -557,7 +508,7 @@ func hasTime(a []*time.Time) bool { } type importKey struct { - Frame string + View string Slice uint64 } diff --git a/db_test.go b/db_test.go index 96120dc1e..5ce17478f 100644 --- a/db_test.go +++ b/db_test.go @@ -4,7 +4,6 @@ import ( "io/ioutil" "os" "testing" - "time" "github.com/pilosa/pilosa" ) @@ -26,11 +25,11 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) - } else if f != other { + } else if f.Frame != other.Frame { t.Fatal("frame mismatch") } - if f != db.Frame("f") { + if f.Frame != db.Frame("f") { t.Fatal("frame mismatch") } } @@ -130,13 +129,22 @@ func (db *DB) Reopen() error { return nil } -// MustSetBit sets a bit on the database. Panic on error. -func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) { - changed, err := db.SetBit(name, bitmapID, profileID, t) +// CreateFrame creates a frame with the given options. +func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := db.DB.CreateFrame(name, opt) if err != nil { - panic(err) + return nil, err } - return changed + return &Frame{Frame: f}, nil +} + +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := db.DB.CreateFrameIfNotExists(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil } // Ensure database can delete a frame. diff --git a/executor.go b/executor.go index 6e95346e1..9aff82cda 100644 --- a/executor.go +++ b/executor.go @@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + f := e.Index.Fragment(db, frame, ViewStandard, slice) if f == nil { return nil, nil } @@ -346,7 +346,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel) } - frag := e.Index.Fragment(db, frame, slice) + frag := e.Index.Fragment(db, frame, ViewStandard, slice) if frag == nil { return NewBitmap(), nil } @@ -421,8 +421,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call // Union bitmaps across all time-based subframes. bm := &Bitmap{} - for _, subframe := range FramesByTimeRange(frame, startTime, endTime, q) { - f := e.Index.Fragment(db, subframe, slice) + for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { + f := e.Index.Fragment(db, frame, view, slice) if f == nil { continue } @@ -525,12 +525,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - frag := e.Index.Fragment(db, frame, slice) - if frag == nil { - return false, nil - } - - val, err := frag.ClearBit(rowID, colID) + val, err := f.ClearBit(rowID, colID, nil) if err != nil { return false, err } else if val { @@ -601,12 +596,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - d := e.Index.DB(db) - if d == nil { - return false, ErrDatabaseNotFound - } - - val, err := d.SetBit(frame, rowID, colID, timestamp) + val, err := f.SetBit(rowID, colID, timestamp) if err != nil { return false, err } else if val { diff --git a/executor_test.go b/executor_test.go index eb362ca39..424aba12a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -15,8 +15,8 @@ import ( func TestExecutor_Execute_Bitmap(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) @@ -36,11 +36,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -54,7 +54,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil { @@ -66,13 +66,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -97,12 +97,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -116,7 +116,7 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil { @@ -130,9 +130,9 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { @@ -148,7 +148,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { defer idx.Close() e := NewExecutor(idx.Index, NewCluster(1)) - f := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if n := f.Bitmap(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -216,15 +216,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -241,13 +241,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0 & 1 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth) + // Set bits for bitmaps 0, 10, & 20 across two slices. + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -265,23 +265,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(0, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(0, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 4).SetBit(0, 4*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -300,19 +300,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -332,9 +332,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -355,9 +355,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -380,24 +380,26 @@ func TestExecutor_Execute_Range(t *testing.T) { // Create database. db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) // Create frame. - if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { t.Fatal(err) } // Set bits. - db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) - db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) - db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) - db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00")) - db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00")) - db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00")) + f.MustSetBit(1, 2, MustParseTimePtr("1999-12-31 00:00")) + f.MustSetBit(1, 3, MustParseTimePtr("2000-01-01 00:00")) + f.MustSetBit(1, 4, MustParseTimePtr("2000-01-02 00:00")) + f.MustSetBit(1, 5, MustParseTimePtr("2000-02-01 00:00")) + f.MustSetBit(1, 6, MustParseTimePtr("2001-01-01 00:00")) + f.MustSetBit(1, 7, MustParseTimePtr("2002-01-01 02:00")) - db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early - db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late - db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap + f.MustSetBit(1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early + f.MustSetBit(1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late + f.MustSetBit(10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { @@ -439,7 +441,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { @@ -466,8 +468,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { @@ -514,7 +516,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -547,17 +549,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. idx := MustOpenIndex() defer idx.Close() - idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) - oldQuantum := idx.DB("d").TimeQuantum() - defer func() { - // restore db quantum - idx.DB("d").SetTimeQuantum(oldQuantum) - }() - // need to set the quantum otherwise SetBit fails silently - idx.DB("d").SetTimeQuantum("Y") // Create frame. - if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) } @@ -567,7 +563,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f_2016", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -620,8 +616,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 1 & 3. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { diff --git a/fragment.go b/fragment.go index 75ed0a298..5a3bcbd63 100644 --- a/fragment.go +++ b/fragment.go @@ -58,6 +58,7 @@ type Fragment struct { // Composite identifiers db string frame string + view string slice uint64 // File-backed storage @@ -92,11 +93,12 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame string, slice uint64) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64) *Fragment { return &Fragment{ path: path, db: db, frame: frame, + view: view, slice: slice, LogOutput: ioutil.Discard, @@ -118,6 +120,9 @@ func (f *Fragment) DB() string { return f.db } // Frame returns the frame the fragment was initialized with. func (f *Fragment) Frame() string { return f.frame } +// View returns the view the fragment was initialized with. +func (f *Fragment) View() string { return f.view } + // Slice returns the slice the fragment was initialized with. func (f *Fragment) Slice() uint64 { return f.slice } @@ -362,7 +367,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) } // Write to storage. - if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -995,8 +999,8 @@ func track(start time.Time, name string, logger *log.Logger) { func (f *Fragment) snapshot() error { logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%d", f.db, f.frame, f.slice) - defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%d", f.db, f.frame, f.slice), logger) + logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice) + defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt @@ -1322,7 +1326,7 @@ func (s *FragmentSyncer) SyncFragment() error { if err != nil { return err } - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return err } @@ -1403,7 +1407,8 @@ func (s *FragmentSyncer) syncBlock(id int) error { } clients = append(clients, client) - bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), f.Slice(), id) + // Only sync the standard block. + bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) if err != nil { return err } @@ -1436,6 +1441,8 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Generate query with sets & clears. var buf bytes.Buffer + + // Only sync the standard block. for j := 0; j < len(set.ProfileIDs); j++ { fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j]) } diff --git a/fragment_test.go b/fragment_test.go index f99b83848..63315ab49 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -54,7 +54,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -81,7 +81,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -110,7 +110,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -139,7 +139,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the bitmaps 100, 101, & 102. @@ -161,7 +161,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter bitmaps when retrieving the top n bitmaps. func TestFragment_Top_Filter(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the bitmaps 100, 101, & 102. @@ -191,7 +191,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top bitmaps that intersect with an input bitmap. func TestFragment_TopN_Intersect(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input bitmap. @@ -221,7 +221,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input bitmap. @@ -258,7 +258,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top bitmaps when specified by ID. func TestFragment_TopN_BitmapIDs(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on various bitmaps. @@ -279,7 +279,7 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve checksum and set bits. @@ -298,7 +298,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve initial checksum. @@ -336,7 +336,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on a different block. @@ -354,7 +354,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -386,7 +386,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - f := MustOpenFragment("d", "f.n", 0) + f := MustOpenFragment("d", "f.n", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -418,7 +418,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := MustOpenFragment("d", "f", 0) + f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f0.Close() // Set and then clear bits on the fragment. @@ -443,7 +443,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := MustOpenFragment("d", "f", 0) + f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -476,7 +476,7 @@ func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlo func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) } func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Fill fragment. @@ -505,7 +505,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", 0) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -521,7 +521,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() f.MaxOpN = math.MaxInt32 @@ -558,7 +558,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(db, frame string, slice uint64) *Fragment { +func NewFragment(db, frame, view string, slice uint64) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -566,7 +566,7 @@ func NewFragment(db, frame string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, slice), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -574,8 +574,8 @@ func NewFragment(db, frame string, slice uint64) *Fragment { } // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(db, frame string, slice uint64) *Fragment { - f := NewFragment(db, frame, slice) +func MustOpenFragment(db, frame, view string, slice uint64) *Fragment { + f := NewFragment(db, frame, view, slice) if err := f.Open(); err != nil { panic(err) } @@ -597,7 +597,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index 8ff32f761..8fdeaa75e 100644 --- a/frame.go +++ b/frame.go @@ -1,13 +1,14 @@ package pilosa import ( + "errors" "fmt" "io" "io/ioutil" "os" "path/filepath" - "strconv" "sync" + "time" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -23,7 +24,7 @@ const ( DefaultRowLabel = "id" ) -// Frame represents a container for fragments. +// Frame represents a container for views. type Frame struct { mu sync.Mutex path string @@ -31,8 +32,7 @@ type Frame struct { name string timeQuantum TimeQuantum - // Fragments by slice. - fragments map[uint64]*Fragment + views map[string]*View // Bitmap attribute storage and cache bitmapAttrStore *AttrStore @@ -57,7 +57,7 @@ func NewFrame(path, db, name string) (*Frame, error) { db: db, name: name, - fragments: make(map[uint64]*Fragment), + views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), stats: NopStatsClient, @@ -86,8 +86,8 @@ func (f *Frame) MaxSlice() uint64 { defer f.mu.Unlock() var max uint64 - for slice := range f.fragments { - if slice > max { + for _, view := range f.views { + if slice := view.MaxSlice(); slice > max { max = slice } } @@ -143,7 +143,7 @@ func (f *Frame) Open() error { return err } - if err := f.openFragments(); err != nil { + if err := f.openViews(); err != nil { return err } @@ -160,10 +160,12 @@ func (f *Frame) Open() error { return nil } -// openFragments opens and initializes the fragments inside the frame. -func (f *Frame) openFragments() error { - file, err := os.Open(f.path) - if err != nil { +// openViews opens and initializes the views inside the frame. +func (f *Frame) openViews() error { + file, err := os.Open(filepath.Join(f.path, "views")) + if os.IsNotExist(err) { + return nil + } else if err != nil { return err } defer file.Close() @@ -174,22 +176,17 @@ func (f *Frame) openFragments() error { } for _, fi := range fis { - if fi.IsDir() { + if !fi.IsDir() { continue } - // Parse filename into integer. - slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) - if err != nil { - continue + name := filepath.Base(fi.Name()) + view := f.newView(f.ViewPath(name), name) + if err := view.Open(); err != nil { + return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) } - - frag := f.newFragment(f.FragmentPath(slice), slice) - if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) - } - frag.BitmapAttrStore = f.bitmapAttrStore - f.fragments[frag.Slice()] = frag + view.BitmapAttrStore = f.bitmapAttrStore + f.views[view.Name()] = view f.stats.Count("maxSlice", 1) } @@ -241,7 +238,7 @@ func (f *Frame) saveMeta() error { return nil } -// Close closes the frame and its fragments. +// Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() defer f.mu.Unlock() @@ -251,11 +248,11 @@ func (f *Frame) Close() error { _ = f.bitmapAttrStore.Close() } - // Close all fragments. - for _, frag := range f.fragments { - _ = frag.Close() + // Close all views. + for _, view := range f.views { + _ = view.Close() } - f.fragments = make(map[uint64]*Fragment) + f.views = make(map[string]*View) return nil } @@ -288,75 +285,232 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error { return nil } -// FragmentPath returns the path to a fragment in the frame. -func (f *Frame) FragmentPath(slice uint64) string { - return filepath.Join(f.path, strconv.FormatUint(slice, 10)) +// ViewPath returns the path to a view in the frame. +func (f *Frame) ViewPath(name string) string { + return filepath.Join(f.path, "views", name) } -// Fragment returns a fragment in the frame by slice. -func (f *Frame) Fragment(slice uint64) *Fragment { +// View returns a view in the frame by name. +func (f *Frame) View(name string) *View { f.mu.Lock() defer f.mu.Unlock() - return f.fragment(slice) + return f.view(name) } -func (f *Frame) fragment(slice uint64) *Fragment { return f.fragments[slice] } +func (f *Frame) view(name string) *View { return f.views[name] } -// Fragments returns a list of all fragments in the frame. -func (f *Frame) Fragments() []*Fragment { +// Views returns a list of all views in the frame. +func (f *Frame) Views() []*View { f.mu.Lock() defer f.mu.Unlock() - other := make([]*Fragment, 0, len(f.fragments)) - for _, fragment := range f.fragments { - other = append(other, fragment) + other := make([]*View, 0, len(f.views)) + for _, view := range f.views { + other = append(other, view) } return other } -// CreateFragmentIfNotExists returns a fragment in the frame by slice. -func (f *Frame) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { +func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { f.mu.Lock() defer f.mu.Unlock() - return f.createFragmentIfNotExists(slice) -} -func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) { - // Find fragment in cache first. - if frag := f.fragments[slice]; frag != nil { - return frag, nil + if view := f.views[name]; view != nil { + return view, nil } - // Initialize and open fragment. - frag := f.newFragment(f.FragmentPath(slice), slice) - if err := frag.Open(); err != nil { + view := f.newView(f.ViewPath(name), name) + if err := view.Open(); err != nil { return nil, err } - frag.BitmapAttrStore = f.bitmapAttrStore + view.BitmapAttrStore = f.bitmapAttrStore + f.views[view.Name()] = view - // Save to lookup. - f.fragments[slice] = frag - - f.stats.Count("maxSlice", 1) - - return frag, nil + return view, nil } -func (f *Frame) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, f.db, f.name, slice) - frag.LogOutput = f.LogOutput - frag.stats = f.stats.WithTags(fmt.Sprintf("slice:%d", slice)) - return frag +func (f *Frame) newView(path, name string) *View { + view := NewView(path, f.db, f.name, name) + view.LogOutput = f.LogOutput + view.BitmapAttrStore = f.bitmapAttrStore + view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name)) + return view } // SetBit sets a bit within the frame. -func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth - frag, err := f.CreateFragmentIfNotExists(slice) +func (f *Frame) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Set standard layout bits. + if v, err := f.setBit(ViewStandard, rowID, colID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + // Set inverse layout bits. + // NOTE: The row & col are transposed for the inverted view. + if v, err := f.setBit(ViewInverse, colID, rowID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + return changed, nil +} + +// setBit sets a bit for a given layout (default or inverted). +func (f *Frame) setBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) if err != nil { return changed, err } - return frag.SetBit(bitmapID, profileID) + + // Set non-time bit. + if v, err := view.SetBit(rowID, colID); err != nil { + return changed, err + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then set bits across all views for the quantum. + for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := view.SetBit(rowID, colID); err != nil { + return changed, err + } else if c { + changed = true + } + } + + return changed, nil +} + +// ClearBit clears a bit within the frame. +func (f *Frame) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Clear standard layout bits. + if v, err := f.clearBit(ViewStandard, rowID, colID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + // Clear inverse layout bits. + // NOTE: The row & col are transposed for the inverted view. + if v, err := f.clearBit(ViewInverse, colID, rowID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + return changed, nil +} + +// clearBit clears a bit for a given layout (default or inverted). +func (f *Frame) clearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) + if err != nil { + return changed, err + } + + // Clear non-time bit. + if v, err := view.ClearBit(rowID, colID); err != nil { + return changed, err + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then clear bits across all views for the quantum. + for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := view.ClearBit(rowID, colID); err != nil { + return changed, err + } else if c { + changed = true + } + } + + return changed, nil +} + +// Import bulk imports data. +func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { + // Determine quantum if timestamps are set. + q := f.TimeQuantum() + if hasTime(timestamps) && q == "" { + return errors.New("time quantum not set in either database or frame") + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range bitmapIDs { + bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + slice := profileID / SliceWidth + + var standard, inverse []string + if timestamp == nil { + standard = []string{ViewStandard} + inverse = []string{ViewInverse} + } else { + standard = ViewsByTime(ViewStandard, *timestamp, q) + inverse = ViewsByTime(ViewInverse, *timestamp, q) + } + + // Attach bit to each standard view. + for _, name := range standard { + key := importKey{View: name, Slice: slice} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, bitmapID) + data.ProfileIDs = append(data.ProfileIDs, profileID) + dataByFragment[key] = data + } + + // Attach reversed bits to each inverse view. + for _, name := range inverse { + key := importKey{View: name, Slice: slice} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed + data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + view, err := f.CreateViewIfNotExists(key.View) + if err != nil { + return err + } + + frag, err := view.CreateFragmentIfNotExists(key.Slice) + if err != nil { + return err + } + + if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + return err + } + } + + return nil } type frameSlice []*Frame @@ -367,7 +521,8 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FrameInfo represents schema information for a frame. type FrameInfo struct { - Name string `json:"name"` + Name string `json:"name"` + Views []*ViewInfo `json:"views,omitempty"` } type frameInfoSlice []*FrameInfo diff --git a/frame_test.go b/frame_test.go index 5b03cfb3e..8a86f1ee8 100644 --- a/frame_test.go +++ b/frame_test.go @@ -4,33 +4,34 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/pilosa/pilosa" ) -// Ensure frame can open and retrieve a fragment. -func TestFrame_CreateFragmentIfNotExists(t *testing.T) { +// Ensure frame can open and retrieve a view. +func TestFrame_CreateViewIfNotExists(t *testing.T) { f := MustOpenFrame() defer f.Close() - // Create fragment. - frag, err := f.CreateFragmentIfNotExists(100) + // Create view. + view, err := f.CreateViewIfNotExists("v") if err != nil { t.Fatal(err) - } else if frag == nil { - t.Fatal("expected fragment") + } else if view == nil { + t.Fatal("expected view") } - // Retrieve existing fragment. - frag2, err := f.CreateFragmentIfNotExists(100) + // Retrieve existing view. + view2, err := f.CreateViewIfNotExists("v") if err != nil { t.Fatal(err) - } else if frag != frag2 { - t.Fatal("fragment mismatch") + } else if view != view2 { + t.Fatal("view mismatch") } - if frag != f.Fragment(100) { - t.Fatal("fragment mismatch") + if view != f.View("v") { + t.Fatal("view mismatch") } } @@ -54,6 +55,17 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } } +func TestFrame_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + frame, err := pilosa.NewFrame(path, "d", ".meta") + if frame != nil { + t.Fatalf("unexpected frame name %s", err) + } +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame @@ -106,14 +118,11 @@ func (f *Frame) Reopen() error { return nil } -// NewFrame does not return a frame when name is invalid. -func TestFrame_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-frame-") +// MustSetBit sets a bit on the frame. Panic on error. +func (f *Frame) MustSetBit(bitmapID, profileID uint64, t *time.Time) (changed bool) { + changed, err := f.SetBit(bitmapID, profileID, t) if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "d", ".meta") - if frame != nil { - t.Fatalf("unexpected frame name %s", err) - } + return changed } diff --git a/handler.go b/handler.go index 1ccdf3711..03c56ad09 100644 --- a/handler.go +++ b/handler.go @@ -145,6 +145,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/frame/views": + switch r.Method { + case "GET": + h.handleGetFrameViews(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/frame/attr/diff": switch r.Method { case "POST": @@ -583,6 +590,35 @@ type patchFrameTimeQuantumRequest struct { type patchFrameTimeQuantumResponse struct{} +// handleGetFrameViews handles GET /frame/views request. +func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + db, frame := q.Get("db"), q.Get("frame") + + // Retrieve views. + f := h.Index.Frame(db, frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Fetch views. + views := f.Views() + names := make([]string, len(views)) + for i := range views { + names[i] = views[i].Name() + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type getFrameViewsResponse struct { + Views []string `json:"views,omitempty"` +} + // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -791,7 +827,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Find the correct fragment. + // Find the DB. h.logger().Println("importing:", req.DB, req.Frame, req.Slice) db := h.Index.DB(req.DB) if db == nil { @@ -800,8 +836,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } + // Retrieve frame. + f := db.Frame(req.Frame) + if f == nil { + h.logger().Printf("frame error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrFrameNotFound.Error()) + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + // Import into fragment. - err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps) + err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps) if err != nil { h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) return @@ -834,7 +878,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Parse query parameters. q := r.URL.Query() - db, frame := q.Get("db"), q.Get("frame") + db, frame, view := q.Get("db"), q.Get("frame"), q.Get("view") slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { @@ -850,7 +894,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Find the fragment. - f := h.Index.Fragment(db, frame, slice) + f := h.Index.Fragment(db, frame, view, slice) if f == nil { return } @@ -905,7 +949,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice) + f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -934,8 +978,15 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } + // Retrieve view. + view, err := f.CreateViewIfNotExists(q.Get("view")) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Retrieve fragment from frame. - frag, err := f.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(slice) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -961,7 +1012,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from index. - f := h.Index.Fragment(req.DB, req.Frame, req.Slice) + f := h.Index.Fragment(req.DB, req.Frame, req.View, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -997,7 +1048,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request } // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice) + f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1050,6 +1101,24 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) return } + // Retrieve frame. + f := h.Index.Frame(db, frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Retrieve list of all views. + views, err := client.FrameViews(r.Context(), db, frame) + if err != nil { + http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) + return + } + + for _, view := range views { + println("dbg/views", view) + } + // Loop over each slice and import it if this node owns it. //travis for slice := uint64(0); slice <= maxSlices[db]; slice++ { @@ -1058,39 +1127,42 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) continue } - // Retrieve frame. - f := h.Index.Frame(db, frame) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Otherwise retrieve the local fragment. - frag, err := f.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), db, frame, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return err + // Loop over view names. + for _, view := range views { + // Create view. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Otherwise retrieve the local fragment. + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Stream backup from remote node. + rd, err := client.BackupSlice(r.Context(), db, frame, view, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } else if rd == nil { + continue // slice doesn't exist + } + + // Restore to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return } } } diff --git a/handler_test.go b/handler_test.go index be88d6d9a..2fdba657b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -37,10 +37,14 @@ func TestHandler_Schema(t *testing.T) { d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) - if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } - if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { @@ -53,7 +57,7 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1"}]},{"name":"d1","frames":[{"name":"f0"}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"inverse"},{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -63,13 +67,13 @@ func TestHandler_MaxSlices(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d0", "f0", 3).MustSetBits(30, (3*SliceWidth)+4) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+8) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := NewHandler() h.Index = idx.Index @@ -654,11 +658,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer s.Close() // Set bits in the index. - f0 := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f0 := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) // Begin backing up from slice d/f/0. - resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&slice=0") + resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0") if err != nil { t.Fatal(err) } @@ -675,7 +679,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Restore backup to slice x/y/0. - if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil { + if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { t.Fatal(err) } else if resp.StatusCode != http.StatusOK { resp.Body.Close() @@ -685,9 +689,9 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Verify data is correctly restored. - f1 := idx.Fragment("x", "y", 0) + f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { - t.Fatal("fragment x/y/0 not created") + t.Fatal("fragment x/y/standard/0 not created") } else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { t.Fatalf("unexpected restored bits: %+v", bits) } diff --git a/index.go b/index.go index 07d13a030..3b45dd438 100644 --- a/index.go +++ b/index.go @@ -131,12 +131,17 @@ func (i *Index) Schema() []*DBInfo { for _, db := range i.DBs() { di := &DBInfo{Name: db.Name()} for _, frame := range db.Frames() { - di.Frames = append(di.Frames, &FrameInfo{ - Name: frame.Name(), - }) + fi := &FrameInfo{Name: frame.Name()} + for _, view := range frame.Views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + } + sort.Sort(viewInfoSlice(fi.Views)) + di.Frames = append(di.Frames, fi) } + sort.Sort(frameInfoSlice(di.Frames)) a = append(a, di) } + sort.Sort(dbInfoSlice(a)) return a } @@ -270,13 +275,22 @@ func (i *Index) Frame(db, name string) *Frame { return d.Frame(name) } -// Fragment returns the fragment for a database, frame & slice. -func (i *Index) Fragment(db, frame string, slice uint64) *Fragment { +// View returns the view for a database, frame, and name. +func (i *Index) View(db, frame, name string) *View { f := i.Frame(db, frame) if f == nil { return nil } - return f.Fragment(slice) + return f.View(name) +} + +// Fragment returns the fragment for a database, frame & slice. +func (i *Index) Fragment(db, frame, view string, slice uint64) *Fragment { + v := i.View(db, frame, view) + if v == nil { + return nil + } + return v.Fragment(slice) } // monitorCacheFlush periodically flushes all fragment caches sequentially. @@ -298,15 +312,17 @@ func (i *Index) monitorCacheFlush() { func (i *Index) flushCaches() { for _, db := range i.DBs() { for _, frame := range db.Frames() { - for _, fragment := range frame.Fragments() { - select { - case <-i.closing: - return - default: - } + for _, view := range frame.Views() { + for _, fragment := range view.Fragments() { + select { + case <-i.closing: + return + default: + } - if err := fragment.FlushCache(); err != nil { - i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + if err := fragment.FlushCache(); err != nil { + i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + } } } } @@ -362,20 +378,27 @@ func (s *IndexSyncer) SyncIndex() error { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } - for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { - // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { - continue - } - + for _, vi := range fi.Views { // Verify syncer has not closed. if s.IsClosing() { return nil } - // Sync fragment if own it. - if err := s.syncFragment(di.Name, fi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { + // Ignore slices that this host doesn't own. + if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { + continue + } + + // Verify syncer has not closed. + if s.IsClosing() { + return nil + } + + // Sync fragment if own it. + if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { + return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + } } } } @@ -477,15 +500,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error { +func (s *IndexSyncer) syncFragment(db, frame, view string, slice uint64) error { // Retrieve local frame. f := s.Index.Frame(db, frame) if f == nil { return ErrFrameNotFound } + // Ensure view exists locally. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + return err + } + // Ensure fragment exists locally. - frag, err := f.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return err } diff --git a/index_test.go b/index_test.go index dace25fcc..4a1be4cf2 100644 --- a/index_test.go +++ b/index_test.go @@ -18,11 +18,11 @@ func TestIndex_DeleteDB(t *testing.T) { defer idx.Close() // Write bits to separate databases. - f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0) + f0 := idx.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } - f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0) + f1 := idx.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0) if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } // Set data on the local index. - f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 20); err != nil { @@ -91,15 +91,15 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1) + f = idx0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1) if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } - idx0.MustCreateFragmentIfNotExists("y", "z", 0) + idx0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) // Set data on the remote index. - f = idx1.MustCreateFragmentIfNotExists("d", "f", 0) + f = idx1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) } else if _, err := f.SetBit(3, 10); err != nil { @@ -108,7 +108,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx1.MustCreateFragmentIfNotExists("y", "z", 3) + f = idx1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { @@ -134,7 +134,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Verify data is the same on both nodes. for i, idx := range []*Index{idx0, idx1} { - f := idx.Fragment("d", "f", 0) + f := idx.Fragment("d", "f", pilosa.ViewStandard, 0) if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) } else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { @@ -147,7 +147,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } - f = idx.Fragment("d", "f0", 1) + f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1) a := f.Bitmap(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) @@ -155,7 +155,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - f = idx.Fragment("y", "z", 3) + f = idx.Fragment("y", "z", pilosa.ViewStandard, 3) if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) } @@ -212,17 +212,21 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { if err != nil { panic(err) } - return &Frame{Frame: f} + return f } // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { +func (i *Index) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment { d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } - frag, err := f.CreateFragmentIfNotExists(slice) + v, err := f.CreateViewIfNotExists(view) + if err != nil { + panic(err) + } + frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { panic(err) } diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 85f4102bd..2314036cd 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -242,8 +242,9 @@ func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInter type BlockDataRequest struct { DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` + Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } @@ -907,15 +908,21 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) i += copy(dAtA[i:], m.Frame) } - if m.Slice != 0 { + if m.Block != 0 { dAtA[i] = 0x18 i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + } + if m.Slice != 0 { + dAtA[i] = 0x20 + i++ i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) } - if m.Block != 0 { - dAtA[i] = 0x20 + if len(m.View) > 0 { + dAtA[i] = 0x2a i++ - i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + i = encodeVarintInternal(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) } return i, nil } @@ -1329,11 +1336,15 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovInternal(uint64(l)) } + if m.Block != 0 { + n += 1 + sovInternal(uint64(m.Block)) + } if m.Slice != 0 { n += 1 + sovInternal(uint64(m.Slice)) } - if m.Block != 0 { - n += 1 + sovInternal(uint64(m.Block)) + l = len(m.View) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) } return n } @@ -3330,25 +3341,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { m.Frame = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) - } - m.Slice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowInternal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) } @@ -3367,6 +3359,54 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -3938,51 +3978,52 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 735 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd4, 0x4a, - 0x10, 0xbd, 0x3d, 0xb6, 0xe7, 0x51, 0x93, 0x8c, 0x26, 0xad, 0xb9, 0x57, 0x56, 0x74, 0x35, 0xb2, - 0x5a, 0xf7, 0x4a, 0x16, 0x12, 0x89, 0x14, 0x36, 0x08, 0x21, 0x21, 0x3c, 0x33, 0x51, 0x46, 0x90, - 0x28, 0xe9, 0x04, 0x76, 0x2c, 0x3a, 0x49, 0x93, 0x58, 0xf1, 0x63, 0xb0, 0xdb, 0xc0, 0x2c, 0x59, - 0xf0, 0x0f, 0x08, 0xbe, 0x00, 0xbe, 0x84, 0x25, 0x9f, 0x80, 0xc2, 0x8f, 0xa0, 0x7e, 0xf8, 0x11, - 0x10, 0x21, 0x0b, 0x76, 0xae, 0x53, 0x5d, 0xd5, 0x75, 0xfa, 0x54, 0x95, 0x61, 0x10, 0x26, 0x82, - 0x67, 0x09, 0x8b, 0x36, 0x16, 0x59, 0x2a, 0x52, 0xdc, 0x2d, 0x6d, 0xb2, 0x03, 0xad, 0x69, 0x80, - 0x3d, 0xe8, 0x1f, 0x85, 0x31, 0x3f, 0x28, 0x58, 0x22, 0x8a, 0xd8, 0x45, 0x1e, 0xf2, 0x7b, 0xb4, - 0x09, 0xc9, 0x13, 0x93, 0x34, 0x2a, 0xe2, 0xe4, 0x31, 0x3b, 0xe6, 0x91, 0xdb, 0xd2, 0x27, 0x1a, - 0x10, 0x99, 0x81, 0xb3, 0x9d, 0xb1, 0x98, 0xdf, 0x20, 0xd9, 0x3a, 0x74, 0x69, 0xfa, 0xaa, 0x99, - 0xa9, 0xb2, 0x49, 0x00, 0xed, 0x20, 0x14, 0x31, 0x5b, 0x60, 0x0c, 0x76, 0x10, 0x8a, 0xdc, 0x45, - 0x9e, 0xe5, 0xdb, 0x54, 0x7d, 0xe3, 0xff, 0xc0, 0x79, 0x28, 0x44, 0x96, 0xbb, 0x2d, 0xcf, 0xf2, - 0xfb, 0x5b, 0x83, 0x8d, 0x8a, 0x98, 0x84, 0xa9, 0x76, 0x92, 0x0d, 0xb0, 0xf7, 0x59, 0x98, 0xe1, - 0x21, 0x58, 0x8f, 0xf8, 0x52, 0x55, 0x60, 0x53, 0xf9, 0x89, 0x47, 0xe0, 0x4c, 0xd2, 0x22, 0x11, - 0xea, 0x5a, 0x9b, 0x6a, 0x83, 0x3c, 0x03, 0x2b, 0x08, 0x85, 0x2c, 0x4b, 0x5f, 0x3d, 0x9f, 0x9a, - 0x98, 0xca, 0xc6, 0xff, 0x42, 0x6f, 0x3f, 0x4b, 0x9f, 0x87, 0x11, 0x9f, 0x4f, 0x4d, 0x70, 0x0d, - 0x48, 0xaf, 0xe4, 0x97, 0x0b, 0x16, 0x2f, 0x5c, 0xcb, 0x43, 0xbe, 0x45, 0x6b, 0x80, 0x3c, 0x80, - 0x8e, 0x39, 0x8a, 0x07, 0xd0, 0xaa, 0x92, 0xb7, 0xe6, 0xd3, 0x1b, 0xf2, 0xf9, 0x84, 0xc0, 0x96, - 0x5f, 0x4d, 0x42, 0x3d, 0x4d, 0x08, 0x83, 0x7d, 0xb4, 0x5c, 0x70, 0x53, 0x92, 0xfa, 0x96, 0x02, - 0x1c, 0x8a, 0x2c, 0x4c, 0xce, 0x9e, 0xb2, 0xa8, 0xe0, 0xaa, 0x9e, 0x1e, 0x6d, 0x42, 0xb2, 0xde, - 0x27, 0x61, 0x22, 0xb4, 0xdf, 0xd6, 0x6c, 0x2a, 0x40, 0x7a, 0x83, 0x34, 0x8d, 0xb4, 0xd7, 0xf1, - 0x90, 0xdf, 0xa5, 0x35, 0x80, 0xc7, 0x00, 0xdb, 0x51, 0xca, 0x4c, 0x70, 0xdb, 0x43, 0x3e, 0xa2, - 0x0d, 0x84, 0x6c, 0x42, 0x47, 0xd6, 0xba, 0xcb, 0x16, 0x35, 0x3b, 0x74, 0x1d, 0xbb, 0xf7, 0x08, - 0x56, 0x0e, 0x0a, 0x9e, 0x2d, 0x29, 0x7f, 0x51, 0xf0, 0x5c, 0xc8, 0x47, 0x9a, 0x06, 0x86, 0xa4, - 0xec, 0xce, 0x11, 0x38, 0xca, 0x6f, 0x7a, 0x45, 0x1b, 0xf8, 0x1f, 0x68, 0x1f, 0x46, 0xe1, 0x09, - 0xcf, 0x5d, 0x4b, 0x35, 0x88, 0xb1, 0xa4, 0x8a, 0xe6, 0xb5, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b, - 0xbb, 0xd0, 0x29, 0xdb, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4, - 0xa5, 0xc6, 0x22, 0x6f, 0x10, 0xac, 0x9a, 0xe2, 0xf2, 0x45, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcb, - 0xb2, 0x52, 0x83, 0x59, 0x96, 0xe1, 0x4d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb, - 0x26, 0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xbb, 0x51, 0xa2, 0xa5, 0x22, 0xd6, 0xea, - 0x08, 0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0x8e, 0x88, 0x2a, 0xa2, - 0xbf, 0x35, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x11, 0x5a, 0x01, 0xb4, 0x67, 0x5a, 0x03, 0xed, 0x49, - 0x39, 0xe4, 0x58, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xf9, 0x46, 0x93, 0x73, 0x96, - 0x9c, 0xf1, 0x53, 0xf3, 0x7c, 0xa5, 0x49, 0x3e, 0x22, 0x58, 0x9d, 0xc7, 0x8b, 0x34, 0x13, 0xd7, - 0x28, 0xa5, 0x76, 0x40, 0xa9, 0x94, 0x5e, 0x08, 0x23, 0x70, 0x94, 0x36, 0xaa, 0x13, 0x6d, 0xaa, - 0x0d, 0xd5, 0x65, 0x66, 0xba, 0xa4, 0x50, 0x52, 0xc2, 0x1a, 0x90, 0x5d, 0x56, 0x8d, 0x57, 0xee, - 0x3a, 0xca, 0xdd, 0x40, 0xa4, 0xbf, 0x1a, 0xb0, 0xdc, 0x6d, 0x7b, 0x96, 0x6f, 0xd1, 0x06, 0x42, - 0x08, 0x0c, 0xca, 0x52, 0x7f, 0xa5, 0x1b, 0x39, 0x85, 0x61, 0x10, 0xa5, 0x27, 0x17, 0x53, 0x26, - 0xd8, 0x9f, 0x60, 0x34, 0x02, 0x47, 0xe5, 0x33, 0x13, 0xa5, 0x0d, 0x72, 0x00, 0x6b, 0x8d, 0x5b, - 0x4c, 0x31, 0x57, 0xc8, 0xa3, 0xeb, 0xc9, 0xb7, 0x7e, 0x24, 0x4f, 0xfe, 0x07, 0x67, 0xc2, 0x4e, - 0xce, 0x7f, 0x93, 0x86, 0x7c, 0x40, 0xb0, 0xb6, 0xcb, 0x5e, 0xeb, 0xb9, 0xa8, 0xae, 0xde, 0x81, - 0x5e, 0x05, 0x9a, 0xc1, 0xbc, 0x55, 0x77, 0xc2, 0x4f, 0xe7, 0x6b, 0x64, 0x96, 0x88, 0x6c, 0x49, - 0xeb, 0xe0, 0xf5, 0xfb, 0x30, 0xb8, 0xea, 0x94, 0x6f, 0x7c, 0x51, 0xef, 0xa7, 0x0b, 0xbd, 0x70, - 0x5f, 0xaa, 0x45, 0x61, 0x16, 0xae, 0x32, 0xee, 0xb5, 0xee, 0xa2, 0x60, 0xf8, 0xf9, 0x72, 0x8c, - 0xbe, 0x5c, 0x8e, 0xd1, 0xd7, 0xcb, 0x31, 0x7a, 0xf7, 0x6d, 0xfc, 0xd7, 0x71, 0x5b, 0xfd, 0x9c, - 0xee, 0x7c, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x22, 0xbf, 0xb0, 0xae, 0x06, 0x00, 0x00, + // 746 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6a, 0x14, 0x4b, + 0x10, 0x3e, 0xbd, 0x33, 0xb3, 0x3f, 0xb5, 0xc9, 0xb2, 0x69, 0x72, 0x0e, 0x43, 0x38, 0x2c, 0x43, + 0x73, 0x0e, 0x0c, 0x82, 0x09, 0xc4, 0x1b, 0x11, 0x41, 0x9c, 0xdd, 0x0d, 0x59, 0x34, 0x21, 0xe9, + 0xc4, 0xdc, 0x79, 0xd1, 0x89, 0x6d, 0x32, 0x64, 0x7e, 0xd6, 0x99, 0x1e, 0x93, 0xbd, 0xf4, 0xc2, + 0x77, 0x10, 0x7d, 0x02, 0x7d, 0x12, 0x2f, 0x7d, 0x04, 0x89, 0x2f, 0x22, 0xfd, 0x33, 0x3f, 0x51, + 0x8c, 0xf1, 0x6e, 0xea, 0xab, 0xae, 0xea, 0xfa, 0xea, 0xab, 0xea, 0x81, 0x41, 0x98, 0x08, 0x9e, + 0x25, 0x2c, 0x5a, 0x9f, 0x67, 0xa9, 0x48, 0x71, 0xb7, 0xb4, 0xc9, 0x36, 0xb4, 0x26, 0x01, 0xf6, + 0xa0, 0x7f, 0x18, 0xc6, 0x7c, 0xbf, 0x60, 0x89, 0x28, 0x62, 0x17, 0x79, 0xc8, 0xef, 0xd1, 0x26, + 0x24, 0x4f, 0x8c, 0xd3, 0xa8, 0x88, 0x93, 0xa7, 0xec, 0x98, 0x47, 0x6e, 0x4b, 0x9f, 0x68, 0x40, + 0x64, 0x0a, 0xce, 0x56, 0xc6, 0x62, 0x7e, 0x8b, 0x64, 0x6b, 0xd0, 0xa5, 0xe9, 0x45, 0x33, 0x53, + 0x65, 0x93, 0x00, 0xda, 0x41, 0x28, 0x62, 0x36, 0xc7, 0x18, 0xec, 0x20, 0x14, 0xb9, 0x8b, 0x3c, + 0xcb, 0xb7, 0xa9, 0xfa, 0xc6, 0xff, 0x81, 0xf3, 0x58, 0x88, 0x2c, 0x77, 0x5b, 0x9e, 0xe5, 0xf7, + 0x37, 0x07, 0xeb, 0x15, 0x31, 0x09, 0x53, 0xed, 0x24, 0xeb, 0x60, 0xef, 0xb1, 0x30, 0xc3, 0x43, + 0xb0, 0x9e, 0xf0, 0x85, 0xaa, 0xc0, 0xa6, 0xf2, 0x13, 0xaf, 0x82, 0x33, 0x4e, 0x8b, 0x44, 0xa8, + 0x6b, 0x6d, 0xaa, 0x0d, 0xf2, 0x1c, 0xac, 0x20, 0x14, 0xb2, 0x2c, 0x7d, 0xf5, 0x6c, 0x62, 0x62, + 0x2a, 0x1b, 0xff, 0x0b, 0xbd, 0xbd, 0x2c, 0x7d, 0x19, 0x46, 0x7c, 0x36, 0x31, 0xc1, 0x35, 0x20, + 0xbd, 0x92, 0x5f, 0x2e, 0x58, 0x3c, 0x77, 0x2d, 0x0f, 0xf9, 0x16, 0xad, 0x01, 0xf2, 0x08, 0x3a, + 0xe6, 0x28, 0x1e, 0x40, 0xab, 0x4a, 0xde, 0x9a, 0x4d, 0x6e, 0xc9, 0xe7, 0x13, 0x02, 0x5b, 0x7e, + 0x35, 0x09, 0xf5, 0x34, 0x21, 0x0c, 0xf6, 0xe1, 0x62, 0xce, 0x4d, 0x49, 0xea, 0x5b, 0x0a, 0x70, + 0x20, 0xb2, 0x30, 0x39, 0x3d, 0x62, 0x51, 0xc1, 0x55, 0x3d, 0x3d, 0xda, 0x84, 0x64, 0xbd, 0xcf, + 0xc2, 0x44, 0x68, 0xbf, 0xad, 0xd9, 0x54, 0x80, 0xf4, 0x06, 0x69, 0x1a, 0x69, 0xaf, 0xe3, 0x21, + 0xbf, 0x4b, 0x6b, 0x00, 0x8f, 0x00, 0xb6, 0xa2, 0x94, 0x99, 0xe0, 0xb6, 0x87, 0x7c, 0x44, 0x1b, + 0x08, 0xd9, 0x80, 0x8e, 0xac, 0x75, 0x87, 0xcd, 0x6b, 0x76, 0xe8, 0x26, 0x76, 0xef, 0x11, 0x2c, + 0xed, 0x17, 0x3c, 0x5b, 0x50, 0xfe, 0xaa, 0xe0, 0xb9, 0x90, 0x4d, 0x9a, 0x04, 0x86, 0xa4, 0x9c, + 0xce, 0x55, 0x70, 0x94, 0xdf, 0xcc, 0x8a, 0x36, 0xf0, 0x3f, 0xd0, 0x3e, 0x88, 0xc2, 0x13, 0x9e, + 0xbb, 0x96, 0x1a, 0x10, 0x63, 0x49, 0x15, 0x4d, 0xb7, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b, 0xbb, + 0xd0, 0x29, 0xc7, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4, 0xa5, + 0xc6, 0x22, 0x6f, 0x10, 0x2c, 0x9b, 0xe2, 0xf2, 0x79, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcd, 0xb2, + 0x52, 0x83, 0x69, 0x96, 0xe1, 0x0d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb, 0x26, + 0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xb7, 0x51, 0xa2, 0xa5, 0x22, 0x56, 0xea, 0x08, + 0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0xae, 0x88, 0x2a, 0xa2, 0xbf, + 0x39, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x15, 0x5a, 0x02, 0xb4, 0x6b, 0x46, 0x03, 0xed, 0x4a, 0x39, + 0xe4, 0x5a, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xd9, 0xa3, 0xf1, 0x19, 0x4b, 0x4e, + 0xf9, 0x0b, 0xd3, 0xbe, 0xd2, 0x24, 0x1f, 0x11, 0x2c, 0xcf, 0xe2, 0x79, 0x9a, 0x89, 0x1b, 0x94, + 0x52, 0x6f, 0x40, 0xa9, 0x94, 0x7e, 0x10, 0x56, 0xc1, 0x51, 0xda, 0xa8, 0x49, 0xb4, 0xa9, 0x36, + 0xd4, 0x94, 0x99, 0xed, 0x92, 0x42, 0x49, 0x09, 0x6b, 0x40, 0x4e, 0x59, 0xb5, 0x5e, 0xb9, 0xeb, + 0x28, 0x77, 0x03, 0x91, 0xfe, 0x6a, 0xc1, 0x72, 0xb7, 0xed, 0x59, 0xbe, 0x45, 0x1b, 0x08, 0x21, + 0x30, 0x28, 0x4b, 0xfd, 0x95, 0x6e, 0xe4, 0x12, 0x86, 0x41, 0x94, 0x9e, 0x9c, 0x4f, 0x98, 0x60, + 0x7f, 0xcc, 0x48, 0x45, 0x96, 0x8c, 0x94, 0x51, 0xf3, 0xb4, 0x9b, 0x3c, 0x31, 0xd8, 0x47, 0x21, + 0xbf, 0x30, 0x03, 0xa7, 0xbe, 0xc9, 0x3e, 0xac, 0x34, 0x6e, 0x36, 0x05, 0x5e, 0x6b, 0x08, 0xba, + 0xb9, 0x21, 0xad, 0x1f, 0x1b, 0x42, 0xfe, 0x07, 0x67, 0xcc, 0x4e, 0xce, 0x7e, 0x93, 0x86, 0x7c, + 0x40, 0xb0, 0xb2, 0xc3, 0x2e, 0xf5, 0xae, 0x54, 0x57, 0x6f, 0x43, 0xaf, 0x02, 0xcd, 0xb2, 0xde, + 0xa9, 0xa7, 0xe3, 0xa7, 0xf3, 0x35, 0x32, 0x4d, 0x44, 0xb6, 0xa0, 0x75, 0xf0, 0xda, 0x43, 0x18, + 0x5c, 0x77, 0xca, 0xbe, 0x9f, 0xd7, 0x6f, 0xd6, 0xb9, 0x7e, 0x84, 0x5f, 0xab, 0xc7, 0xc3, 0x3c, + 0xc2, 0xca, 0x78, 0xd0, 0xba, 0x8f, 0x82, 0xe1, 0xe7, 0xab, 0x11, 0xfa, 0x72, 0x35, 0x42, 0x5f, + 0xaf, 0x46, 0xe8, 0xdd, 0xb7, 0xd1, 0x5f, 0xc7, 0x6d, 0xf5, 0xc3, 0xba, 0xf7, 0x3d, 0x00, 0x00, + 0xff, 0xff, 0x1c, 0xb1, 0xe1, 0x29, 0xc2, 0x06, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 67000243c..e256b35b5 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -84,8 +84,9 @@ message ImportResponse { message BlockDataRequest { string DB = 1; string Frame = 2; - uint64 Slice = 3; - uint64 Block = 4; + string View = 5; + uint64 Slice = 4; + uint64 Block = 3; } message BlockDataResponse { diff --git a/time.go b/time.go index b117023ad..b2176285b 100644 --- a/time.go +++ b/time.go @@ -48,8 +48,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -// FrameByTimeUnit returns the frame name for time with a given quantum unit. -func FrameByTimeUnit(name string, t time.Time, unit rune) string { +// ViewByTimeUnit returns the view name for time with a given quantum unit. +func ViewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -64,21 +64,21 @@ func FrameByTimeUnit(name string, t time.Time, unit rune) string { } } -// FramesByTime returns a list of frames for a given timestamp. -func FramesByTime(name string, t time.Time, q TimeQuantum) []string { +// ViewsByTime returns a list of views for a given timestamp. +func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - frame := FrameByTimeUnit(name, t, unit) - if frame == "" { + view := ViewByTimeUnit(name, t, unit) + if view == "" { continue } - a = append(a, frame) + a = append(a, view) } return a } -// FramesByTimeRange returns a list of frames to traverse to query a time range. -func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// ViewsByTimeRange returns a list of views to traverse to query a time range. +func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -96,7 +96,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, FrameByTimeUnit(name, t, 'H')) + results = append(results, ViewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -107,7 +107,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, FrameByTimeUnit(name, t, 'D')) + results = append(results, ViewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -117,7 +117,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, FrameByTimeUnit(name, t, 'M')) + results = append(results, ViewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -133,16 +133,16 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'Y')) + results = append(results, ViewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'M')) + results = append(results, ViewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'D')) + results = append(results, ViewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, FrameByTimeUnit(name, t, 'H')) + results = append(results, ViewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_test.go index 28d86555d..6de3df364 100644 --- a/time_test.go +++ b/time_test.go @@ -25,45 +25,45 @@ func TestParseTimeQuantum(t *testing.T) { }) } -// Ensure generated frame name can be returned for a given time unit. -func TestFrameByTimeUnit(t *testing.T) { +// Ensure generated view name can be returned for a given time unit. +func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) } // Ensure all applicable frame names can be generated when mutating a time bit. -func TestFramesByTime(t *testing.T) { +func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D")) + a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } @@ -71,63 +71,63 @@ func TestFramesByTime(t *testing.T) { } // Ensure sets of frames can be returned for a given time range. -func TestFramesByTimeRange(t *testing.T) { +func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { t.Fatalf("unexpected frames: %#v", a) } diff --git a/view.go b/view.go new file mode 100644 index 000000000..2dae28bcb --- /dev/null +++ b/view.go @@ -0,0 +1,249 @@ +package pilosa + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "sync" +) + +// View layout modes. +const ( + ViewStandard = "standard" + ViewInverse = "inverse" +) + +// View represents a container for frame data. +type View struct { + mu sync.Mutex + path string + db string + frame string + name string + + // Fragments by slice. + fragments map[uint64]*Fragment + + stats StatsClient + + BitmapAttrStore *AttrStore + LogOutput io.Writer +} + +// NewView returns a new instance of View. +func NewView(path, db, frame, name string) *View { + return &View{ + path: path, + db: db, + frame: frame, + name: name, + + fragments: make(map[uint64]*Fragment), + + stats: NopStatsClient, + LogOutput: ioutil.Discard, + } +} + +// Name returns the name the view was initialized with. +func (v *View) Name() string { return v.name } + +// DB returns the database name the view was initialized with. +func (v *View) DB() string { return v.db } + +// Frame returns the frame name the view was initialized with. +func (v *View) Frame() string { return v.frame } + +// Path returns the path the view was initialized with. +func (v *View) Path() string { return v.path } + +// Open opens and initializes the view. +func (v *View) Open() error { + if err := func() error { + // Ensure the view's path exists. + if err := os.MkdirAll(v.path, 0777); err != nil { + return err + } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { + return err + } + + if err := v.openFragments(); err != nil { + return err + } + + return nil + }(); err != nil { + v.Close() + return err + } + + return nil +} + +// openFragments opens and initializes the fragments inside the view. +func (v *View) openFragments() error { + file, err := os.Open(filepath.Join(v.path, "fragments")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + defer file.Close() + + fis, err := file.Readdir(0) + if err != nil { + return err + } + + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // Parse filename into integer. + slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + if err != nil { + continue + } + + frag := v.newFragment(v.FragmentPath(slice), slice) + if err := frag.Open(); err != nil { + return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) + } + frag.BitmapAttrStore = v.BitmapAttrStore + v.fragments[frag.Slice()] = frag + + v.stats.Count("maxSlice", 1) + } + + return nil +} + +// Close closes the view and its fragments. +func (v *View) Close() error { + v.mu.Lock() + defer v.mu.Unlock() + + // Close all fragments. + for _, frag := range v.fragments { + _ = frag.Close() + } + v.fragments = make(map[uint64]*Fragment) + + return nil +} + +// MaxSlice returns the max slice in the view. +func (v *View) MaxSlice() uint64 { + v.mu.Lock() + defer v.mu.Unlock() + + var max uint64 + for slice := range v.fragments { + if slice > max { + max = slice + } + } + return max +} + +// FragmentPath returns the path to a fragment in the view. +func (v *View) FragmentPath(slice uint64) string { + return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) +} + +// Fragment returns a fragment in the view by slice. +func (v *View) Fragment(slice uint64) *Fragment { + v.mu.Lock() + defer v.mu.Unlock() + return v.fragment(slice) +} + +func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } + +// Fragments returns a list of all fragments in the view. +func (v *View) Fragments() []*Fragment { + v.mu.Lock() + defer v.mu.Unlock() + + other := make([]*Fragment, 0, len(v.fragments)) + for _, fragment := range v.fragments { + other = append(other, fragment) + } + return other +} + +// CreateFragmentIfNotExists returns a fragment in the view by slice. +func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { + v.mu.Lock() + defer v.mu.Unlock() + return v.createFragmentIfNotExists(slice) +} + +func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { + // Find fragment in cache first. + if frag := v.fragments[slice]; frag != nil { + return frag, nil + } + + // Initialize and open fragment. + frag := v.newFragment(v.FragmentPath(slice), slice) + if err := frag.Open(); err != nil { + return nil, err + } + frag.BitmapAttrStore = v.BitmapAttrStore + + // Save to lookup. + v.fragments[slice] = frag + + v.stats.Count("maxSlice", 1) + + return frag, nil +} + +func (v *View) newFragment(path string, slice uint64) *Fragment { + frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag.LogOutput = v.LogOutput + frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) + return frag +} + +// SetBit sets a bit within the view. +func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := profileID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetBit(bitmapID, profileID) +} + +// ClearBit clears a bit within the view. +func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := profileID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.ClearBit(bitmapID, profileID) +} + +type viewSlice []*View + +func (p viewSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p viewSlice) Len() int { return len(p) } +func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } + +// ViewInfo represents schema information for a view. +type ViewInfo struct { + Name string `json:"name"` +} + +type viewInfoSlice []*ViewInfo + +func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p viewInfoSlice) Len() int { return len(p) } +func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } diff --git a/view_test.go b/view_test.go new file mode 100644 index 000000000..87b3e5826 --- /dev/null +++ b/view_test.go @@ -0,0 +1,80 @@ +package pilosa_test + +import ( + "io/ioutil" + "os" + + "github.com/pilosa/pilosa" +) + +// View is a test wrapper for pilosa.View. +type View struct { + *pilosa.View + BitmapAttrStore *AttrStore +} + +// NewView returns a new instance of View with a temporary path. +func NewView(db, frame, name string) *View { + file, err := ioutil.TempFile("", "pilosa-view-") + if err != nil { + panic(err) + } + file.Close() + + v := &View{ + View: pilosa.NewView(file.Name(), db, frame, name), + BitmapAttrStore: MustOpenAttrStore(), + } + v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + return v +} + +// MustOpenView creates and opens an view at a temporary path. Panic on error. +func MustOpenView(db, frame, name string) *View { + v := NewView(db, frame, name) + if err := v.Open(); err != nil { + panic(err) + } + return v +} + +// Close closes the view and removes all underlying data. +func (v *View) Close() error { + defer os.Remove(v.Path()) + defer v.BitmapAttrStore.Close() + return v.View.Close() +} + +// Reopen closes the view and reopens it as a new instance. +func (v *View) Reopen() error { + path := v.Path() + if err := v.View.Close(); err != nil { + return err + } + + v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name()) + v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + if err := v.Open(); err != nil { + return err + } + return nil +} + +// MustSetBits sets bits on a bitmap. Panic on error. +// This function does not accept a timestamp or quantum. +func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if _, err := v.SetBit(bitmapID, profileID); err != nil { + panic(err) + } + } +} + +// MustClearBits clears bits on a bitmap. Panic on error. +func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if _, err := v.ClearBit(bitmapID, profileID); err != nil { + panic(err) + } + } +} From 3a710032401cb78672381fb8af10b91d0c324e50 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 24 Mar 2017 13:40:21 -0600 Subject: [PATCH 2/3] Re-sort inverted bits on import. --- frame.go | 37 ++++++++++++++++++++++++++++--------- view.go | 6 ++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/frame.go b/frame.go index 8fdeaa75e..1d444a8bb 100644 --- a/frame.go +++ b/frame.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "sync" "time" @@ -85,13 +86,11 @@ func (f *Frame) MaxSlice() uint64 { f.mu.Lock() defer f.mu.Unlock() - var max uint64 - for _, view := range f.views { - if slice := view.MaxSlice(); slice > max { - max = slice - } + view := f.views[ViewStandard] + if view == nil { + return 0 } - return max + return view.MaxSlice() } // SetRowLabel sets the row labels. Persists to meta file on update. @@ -463,7 +462,6 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) dataByFragment := make(map[importKey]importData) for i := range bitmapIDs { bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] - slice := profileID / SliceWidth var standard, inverse []string if timestamp == nil { @@ -476,7 +474,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Attach bit to each standard view. for _, name := range standard { - key := importKey{View: name, Slice: slice} + key := importKey{View: name, Slice: profileID / SliceWidth} data := dataByFragment[key] data.BitmapIDs = append(data.BitmapIDs, bitmapID) data.ProfileIDs = append(data.ProfileIDs, profileID) @@ -485,7 +483,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Attach reversed bits to each inverse view. for _, name := range inverse { - key := importKey{View: name, Slice: slice} + key := importKey{View: name, Slice: bitmapID / SliceWidth} data := dataByFragment[key] data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed @@ -495,6 +493,14 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Import into each fragment. for key, data := range dataByFragment { + // Re-sort data for inverse views. + if IsViewInverted(key.View) { + sort.Sort(importBitSet{ + bitmapIDs: data.BitmapIDs, + profileIDs: data.ProfileIDs, + }) + } + view, err := f.CreateViewIfNotExists(key.View) if err != nil { return err @@ -535,3 +541,16 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` } + +// importBitSet represents slices of row and column ids. +// This is used to sort data during import. +type importBitSet struct { + bitmapIDs, profileIDs []uint64 +} + +func (p importBitSet) Swap(i, j int) { + p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i] + p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i] +} +func (p importBitSet) Len() int { return len(p.bitmapIDs) } +func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] } diff --git a/view.go b/view.go index 2dae28bcb..b53cea6aa 100644 --- a/view.go +++ b/view.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" ) @@ -231,6 +232,11 @@ func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) { return frag.ClearBit(bitmapID, profileID) } +// IsViewInverted returns true if the view is used for storing an inverted representation. +func IsViewInverted(name string) bool { + return strings.HasPrefix(name, ViewInverse) +} + type viewSlice []*View func (p viewSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } From 071f1f265291de5143a8f9d8daa031f905664814 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 24 Mar 2017 14:01:01 -0600 Subject: [PATCH 3/3] Fix rebase. --- fragment_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index 63315ab49..875b7c8bf 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -661,7 +661,7 @@ func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint6 } func TestFragment_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) @@ -683,7 +683,7 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3)