diff --git a/client.go b/client.go index e7d0eaf85..fa6e38cb1 100644 --- a/client.go +++ b/client.go @@ -430,7 +430,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string } // BackupTo backs up an entire frame from a cluster to w. -func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) error { +func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error { if db == "" { return ErrDatabaseRequired } else if frame == "" { @@ -448,7 +448,7 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er // Backup every slice to the tar file. for i := uint64(0); i <= maxSlices[db]; i++ { - if err := c.backupSliceTo(ctx, tw, db, frame, i); err != nil { + if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil { return err } } @@ -462,9 +462,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er } // backupSliceTo backs up a single slice to tw. -func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame string, slice uint64) error { +func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error { // Return error if unable to backup from any slice. - r, err := c.BackupSlice(ctx, db, frame, slice) + r, err := c.BackupSlice(ctx, db, frame, view, slice) if err != nil { return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err) } else if r == nil { @@ -500,7 +500,7 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame st // BackupSlice retrieves a streaming backup from a single slice. // This function tries slice owners until one succeeds. -func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64) (io.ReadCloser, error) { +func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, db, slice) if err != nil { @@ -509,7 +509,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64 // Try to backup slice from each one until successful. for _, i := range rand.Perm(len(nodes)) { - r, err := c.backupSliceNode(ctx, db, frame, slice, nodes[i]) + r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i]) if err == nil { return r, nil // successfully attached } else if err == ErrFragmentNotFound { @@ -523,7 +523,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64 return nil, fmt.Errorf("unable to connect to any owner") } -func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := url.URL{ Scheme: "http", Host: node.Host, @@ -531,6 +531,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -560,7 +561,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui } // RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string) error { +func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error { if db == "" { return ErrDatabaseRequired } else if frame == "" { @@ -592,14 +593,14 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string) } // Restore file to all nodes that own it. - if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, slice); err != nil { + if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil { return err } } } // restoreSliceFrom restores a single slice to all owning nodes. -func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame string, slice uint64) error { +func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error { // Retrieve a list of nodes that own the slice. nodes, err := c.FragmentNodes(ctx, db, slice) if err != nil { @@ -615,6 +616,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -726,9 +728,52 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error return nil } +// FrameViews returns a list of view names for a frame. +func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) { + // Create URL & HTTP request. + u := url.URL{ + Scheme: "http", + Host: c.host, + Path: "/frame/views", + RawQuery: (&url.Values{ + "db": {db}, + "frame": {frame}, + }).Encode(), + } + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, ErrFrameNotFound + default: + body, _ := ioutil.ReadAll(resp.Body) + return nil, errors.New(string(body)) + } + + // Decode response. + var rsp getFrameViewsResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, err + } + return rsp.Views, nil +} + // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uint64) ([]FragmentBlock, error) { +func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -736,6 +781,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin RawQuery: url.Values{ "db": {db}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } @@ -771,10 +817,11 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin } // BlockData returns bitmap/profile id pairs for a block. -func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ DB: db, Frame: frame, + View: view, Slice: slice, Block: uint64(block), }) diff --git a/client_test.go b/client_test.go index 15af73d8a..f808514a0 100644 --- a/client_test.go +++ b/client_test.go @@ -61,33 +61,33 @@ func TestClient_MultiNode(t *testing.T) { } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(99, 1, 2, 3, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(1, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(22, 1, 2, 3, 4, 5) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4) + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(100, (SliceWidth*10)+10) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(21, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(100, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).RecalculateCache() - idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).RecalculateCache() - idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).RecalculateCache() + idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache() + idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache() + idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -161,7 +161,7 @@ func TestClient_Import(t *testing.T) { defer idx.Close() // Load bitmap into cache to ensure cache gets updated. - f := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f.Bitmap(0) s := NewServer() @@ -195,10 +195,10 @@ func TestClient_BackupRestore(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(100, SliceWidth, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).MustSetBits(100, (5*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(200, 20000) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) s := NewServer() defer s.Close() @@ -211,7 +211,7 @@ func TestClient_BackupRestore(t *testing.T) { // Backup from frame. var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "d", "f"); err != nil { + if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil { t.Fatal(err) } @@ -219,21 +219,21 @@ func TestClient_BackupRestore(t *testing.T) { if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil { + if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { t.Fatal(err) } // Verify data. - if a := idx.Fragment("x", "y", 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -244,11 +244,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer idx.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := NewServer() defer s.Close() @@ -259,7 +259,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := MustNewClient(s.Host()) - blocks, err := c.FragmentBlocks(context.Background(), "d", "f", 0) + blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { @@ -271,7 +271,7 @@ func TestClient_FragmentBlocks(t *testing.T) { } // Verify data matches local blocks. - if a := idx.Fragment("d", "f", 0).Blocks(); !reflect.DeepEqual(a, blocks) { + if a := idx.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } diff --git a/cmd/backup.go b/cmd/backup.go index ac1b19d7f..24904cbb5 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -18,7 +18,7 @@ func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Use: "backup", Short: "Backup data from pilosa.", Long: ` -Backs up the database and frame from across the cluster into a single file. +Backs up the view from across the cluster into a single file. `, RunE: func(cmd *cobra.Command, args []string) error { if err := Backuper.Run(context.Background()); err != nil { @@ -31,6 +31,7 @@ Backs up the database and frame from across the cluster into a single file. flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.") flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.") + flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.") flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") return backupCmd diff --git a/cmd/restore.go b/cmd/restore.go index a4930f668..5179cbc0d 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -19,7 +19,7 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command Use: "restore", Short: "Restore data to pilosa from a backup file.", Long: ` -Restores a frame to the cluster from a backup file. +Restores a view to the cluster from a backup file. `, RunE: func(cmd *cobra.Command, args []string) error { if err := Restorer.Run(context.Background()); err != nil { @@ -32,6 +32,7 @@ Restores a frame to the cluster from a backup file. flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.") flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.") + flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.") flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.") return restoreCmd diff --git a/ctl/backup.go b/ctl/backup.go index 061982aa8..6eeed5d4c 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -9,14 +9,15 @@ import ( "github.com/pilosa/pilosa" ) -// BackupCommand represents a command for backing up a frame. +// BackupCommand represents a command for backing up a view. type BackupCommand struct { // Destination host and port. Host string - // Name of the database & frame to backup. + // Name of the database, frame, view to backup. Database string Frame string + View string // Output file to write to. Path string @@ -53,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error { defer f.Close() // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil { + if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { return err } diff --git a/ctl/restore.go b/ctl/restore.go index c764c213c..56dfb3080 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -17,6 +17,7 @@ type RestoreCommand struct { // Name of the database & frame to backup. Database string Frame string + View string // Import file to read from. Path string @@ -53,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error { defer f.Close() // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil { + if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { return err } diff --git a/db.go b/db.go index fef92e755..c728b674a 100644 --- a/db.go +++ b/db.go @@ -386,6 +386,7 @@ func (db *DB) DeleteFrame(name string) error { return nil } +/* // SetBit sets a bit for a given profile & bitmap. // If a timestamp is specified then set all bits for the different quantum units. func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { @@ -411,7 +412,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha // If a timestamp is specified then set bits across all frames for the quantum. opt := f.Options() - for _, subname := range FramesByTime(name, *t, q) { + for _, subname := range ViewsByTime(name, *t, q) { f, err := db.CreateFrameIfNotExists(subname, opt) if err != nil { return changed, err @@ -425,67 +426,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha } return changed, nil } - -// Import bulk imports data. -func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { - // Read frame. - f := db.Frame(name) - if f == nil { - return ErrFrameNotFound - } - - // Determine quantum if timestamps are set. - var q TimeQuantum - if hasTime(timestamps) { - if q = f.TimeQuantum(); q == "" { - q = db.TimeQuantum() - if err := f.SetTimeQuantum(q); err != nil { - return err - } - } - - if q == "" { - return errors.New("time quantum not set in either database or frame") - } - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importData) - for i := range bitmapIDs { - bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] - slice := profileID / SliceWidth - - var names []string - if timestamp == nil { - names = []string{name} - } else { - names = FramesByTime(name, *timestamp, q) - } - - // Attach bit to each frame. - for _, name := range names { - key := importKey{Frame: name, Slice: slice} - data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, bitmapID) - data.ProfileIDs = append(data.ProfileIDs, profileID) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - frag, err := f.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return err - } - - if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { - return err - } - } - - return nil -} +*/ type dbSlice []*DB @@ -508,14 +449,19 @@ func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // MergeSchemas combines databases and frames from a and b into one schema. func MergeSchemas(a, b []*DBInfo) []*DBInfo { // Generate a map from both schemas. - m := make(map[string]map[string]struct{}) + m := make(map[string]map[string]map[string]struct{}) for _, dbs := range [][]*DBInfo{a, b} { for _, db := range dbs { if m[db.Name] == nil { - m[db.Name] = make(map[string]struct{}) + m[db.Name] = make(map[string]map[string]struct{}) } for _, frame := range db.Frames { - m[db.Name][frame.Name] = struct{}{} + if m[db.Name][frame.Name] == nil { + m[db.Name][frame.Name] = make(map[string]struct{}) + } + for _, view := range frame.Views { + m[db.Name][frame.Name][view.Name] = struct{}{} + } } } } @@ -524,8 +470,13 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { dbs := make([]*DBInfo, 0, len(m)) for db, frames := range m { di := &DBInfo{Name: db} - for frame := range frames { - di.Frames = append(di.Frames, &FrameInfo{Name: frame}) + for frame, views := range frames { + fi := &FrameInfo{Name: frame} + for view := range views { + fi.Views = append(fi.Views, &ViewInfo{Name: view}) + } + sort.Sort(viewInfoSlice(fi.Views)) + di.Frames = append(di.Frames, fi) } sort.Sort(frameInfoSlice(di.Frames)) dbs = append(dbs, di) @@ -557,7 +508,7 @@ func hasTime(a []*time.Time) bool { } type importKey struct { - Frame string + View string Slice uint64 } diff --git a/db_test.go b/db_test.go index 96120dc1e..5ce17478f 100644 --- a/db_test.go +++ b/db_test.go @@ -4,7 +4,6 @@ import ( "io/ioutil" "os" "testing" - "time" "github.com/pilosa/pilosa" ) @@ -26,11 +25,11 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) - } else if f != other { + } else if f.Frame != other.Frame { t.Fatal("frame mismatch") } - if f != db.Frame("f") { + if f.Frame != db.Frame("f") { t.Fatal("frame mismatch") } } @@ -130,13 +129,22 @@ func (db *DB) Reopen() error { return nil } -// MustSetBit sets a bit on the database. Panic on error. -func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) { - changed, err := db.SetBit(name, bitmapID, profileID, t) +// CreateFrame creates a frame with the given options. +func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := db.DB.CreateFrame(name, opt) if err != nil { - panic(err) + return nil, err } - return changed + return &Frame{Frame: f}, nil +} + +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := db.DB.CreateFrameIfNotExists(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil } // Ensure database can delete a frame. diff --git a/executor.go b/executor.go index 6e95346e1..9aff82cda 100644 --- a/executor.go +++ b/executor.go @@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + f := e.Index.Fragment(db, frame, ViewStandard, slice) if f == nil { return nil, nil } @@ -346,7 +346,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel) } - frag := e.Index.Fragment(db, frame, slice) + frag := e.Index.Fragment(db, frame, ViewStandard, slice) if frag == nil { return NewBitmap(), nil } @@ -421,8 +421,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call // Union bitmaps across all time-based subframes. bm := &Bitmap{} - for _, subframe := range FramesByTimeRange(frame, startTime, endTime, q) { - f := e.Index.Fragment(db, subframe, slice) + for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { + f := e.Index.Fragment(db, frame, view, slice) if f == nil { continue } @@ -525,12 +525,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - frag := e.Index.Fragment(db, frame, slice) - if frag == nil { - return false, nil - } - - val, err := frag.ClearBit(rowID, colID) + val, err := f.ClearBit(rowID, colID, nil) if err != nil { return false, err } else if val { @@ -601,12 +596,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - d := e.Index.DB(db) - if d == nil { - return false, ErrDatabaseNotFound - } - - val, err := d.SetBit(frame, rowID, colID, timestamp) + val, err := f.SetBit(rowID, colID, timestamp) if err != nil { return false, err } else if val { diff --git a/executor_test.go b/executor_test.go index eb362ca39..424aba12a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -15,8 +15,8 @@ import ( func TestExecutor_Execute_Bitmap(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) @@ -36,11 +36,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -54,7 +54,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil { @@ -66,13 +66,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -97,12 +97,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { @@ -116,7 +116,7 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0) + idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil { @@ -130,9 +130,9 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { @@ -148,7 +148,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { defer idx.Close() e := NewExecutor(idx.Index, NewCluster(1)) - f := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if n := f.Bitmap(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -216,15 +216,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -241,13 +241,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0 & 1 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth) + // Set bits for bitmaps 0, 10, & 20 across two slices. + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -265,23 +265,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(0, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(0, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 4).SetBit(0, 4*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -300,19 +300,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -332,9 +332,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -355,9 +355,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) @@ -380,24 +380,26 @@ func TestExecutor_Execute_Range(t *testing.T) { // Create database. db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) // Create frame. - if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { t.Fatal(err) } // Set bits. - db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) - db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) - db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) - db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00")) - db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00")) - db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00")) + f.MustSetBit(1, 2, MustParseTimePtr("1999-12-31 00:00")) + f.MustSetBit(1, 3, MustParseTimePtr("2000-01-01 00:00")) + f.MustSetBit(1, 4, MustParseTimePtr("2000-01-02 00:00")) + f.MustSetBit(1, 5, MustParseTimePtr("2000-02-01 00:00")) + f.MustSetBit(1, 6, MustParseTimePtr("2001-01-01 00:00")) + f.MustSetBit(1, 7, MustParseTimePtr("2002-01-01 02:00")) - db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early - db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late - db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap + f.MustSetBit(1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early + f.MustSetBit(1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late + f.MustSetBit(10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { @@ -439,7 +441,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { @@ -466,8 +468,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { @@ -514,7 +516,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -547,17 +549,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. idx := MustOpenIndex() defer idx.Close() - idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) - oldQuantum := idx.DB("d").TimeQuantum() - defer func() { - // restore db quantum - idx.DB("d").SetTimeQuantum(oldQuantum) - }() - // need to set the quantum otherwise SetBit fails silently - idx.DB("d").SetTimeQuantum("Y") // Create frame. - if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) } @@ -567,7 +563,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f_2016", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -620,8 +616,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 1 & 3. idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) e := NewExecutor(idx.Index, c) if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { diff --git a/fragment.go b/fragment.go index 75ed0a298..5a3bcbd63 100644 --- a/fragment.go +++ b/fragment.go @@ -58,6 +58,7 @@ type Fragment struct { // Composite identifiers db string frame string + view string slice uint64 // File-backed storage @@ -92,11 +93,12 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame string, slice uint64) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64) *Fragment { return &Fragment{ path: path, db: db, frame: frame, + view: view, slice: slice, LogOutput: ioutil.Discard, @@ -118,6 +120,9 @@ func (f *Fragment) DB() string { return f.db } // Frame returns the frame the fragment was initialized with. func (f *Fragment) Frame() string { return f.frame } +// View returns the view the fragment was initialized with. +func (f *Fragment) View() string { return f.view } + // Slice returns the slice the fragment was initialized with. func (f *Fragment) Slice() uint64 { return f.slice } @@ -362,7 +367,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) } // Write to storage. - if changed, err = f.storage.Add(pos); err != nil { return false, err } @@ -995,8 +999,8 @@ func track(start time.Time, name string, logger *log.Logger) { func (f *Fragment) snapshot() error { logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%d", f.db, f.frame, f.slice) - defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%d", f.db, f.frame, f.slice), logger) + logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice) + defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt @@ -1322,7 +1326,7 @@ func (s *FragmentSyncer) SyncFragment() error { if err != nil { return err } - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return err } @@ -1403,7 +1407,8 @@ func (s *FragmentSyncer) syncBlock(id int) error { } clients = append(clients, client) - bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), f.Slice(), id) + // Only sync the standard block. + bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) if err != nil { return err } @@ -1436,6 +1441,8 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Generate query with sets & clears. var buf bytes.Buffer + + // Only sync the standard block. for j := 0; j < len(set.ProfileIDs); j++ { fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j]) } diff --git a/fragment_test.go b/fragment_test.go index f99b83848..875b7c8bf 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -54,7 +54,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -81,7 +81,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -110,7 +110,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -139,7 +139,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the bitmaps 100, 101, & 102. @@ -161,7 +161,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter bitmaps when retrieving the top n bitmaps. func TestFragment_Top_Filter(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the bitmaps 100, 101, & 102. @@ -191,7 +191,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top bitmaps that intersect with an input bitmap. func TestFragment_TopN_Intersect(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input bitmap. @@ -221,7 +221,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input bitmap. @@ -258,7 +258,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top bitmaps when specified by ID. func TestFragment_TopN_BitmapIDs(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on various bitmaps. @@ -279,7 +279,7 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve checksum and set bits. @@ -298,7 +298,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve initial checksum. @@ -336,7 +336,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on a different block. @@ -354,7 +354,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -386,7 +386,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - f := MustOpenFragment("d", "f.n", 0) + f := MustOpenFragment("d", "f.n", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -418,7 +418,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := MustOpenFragment("d", "f", 0) + f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f0.Close() // Set and then clear bits on the fragment. @@ -443,7 +443,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := MustOpenFragment("d", "f", 0) + f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -476,7 +476,7 @@ func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlo func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) } func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() // Fill fragment. @@ -505,7 +505,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", 0) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -521,7 +521,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() f.MaxOpN = math.MaxInt32 @@ -558,7 +558,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(db, frame string, slice uint64) *Fragment { +func NewFragment(db, frame, view string, slice uint64) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -566,7 +566,7 @@ func NewFragment(db, frame string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, slice), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -574,8 +574,8 @@ func NewFragment(db, frame string, slice uint64) *Fragment { } // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(db, frame string, slice uint64) *Fragment { - f := NewFragment(db, frame, slice) +func MustOpenFragment(db, frame, view string, slice uint64) *Fragment { + f := NewFragment(db, frame, view, slice) if err := f.Open(); err != nil { panic(err) } @@ -597,7 +597,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err @@ -661,7 +661,7 @@ func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint6 } func TestFragment_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) @@ -683,7 +683,7 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", 0) + f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) diff --git a/frame.go b/frame.go index 8ff32f761..1d444a8bb 100644 --- a/frame.go +++ b/frame.go @@ -1,13 +1,15 @@ package pilosa import ( + "errors" "fmt" "io" "io/ioutil" "os" "path/filepath" - "strconv" + "sort" "sync" + "time" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -23,7 +25,7 @@ const ( DefaultRowLabel = "id" ) -// Frame represents a container for fragments. +// Frame represents a container for views. type Frame struct { mu sync.Mutex path string @@ -31,8 +33,7 @@ type Frame struct { name string timeQuantum TimeQuantum - // Fragments by slice. - fragments map[uint64]*Fragment + views map[string]*View // Bitmap attribute storage and cache bitmapAttrStore *AttrStore @@ -57,7 +58,7 @@ func NewFrame(path, db, name string) (*Frame, error) { db: db, name: name, - fragments: make(map[uint64]*Fragment), + views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), stats: NopStatsClient, @@ -85,13 +86,11 @@ func (f *Frame) MaxSlice() uint64 { f.mu.Lock() defer f.mu.Unlock() - var max uint64 - for slice := range f.fragments { - if slice > max { - max = slice - } + view := f.views[ViewStandard] + if view == nil { + return 0 } - return max + return view.MaxSlice() } // SetRowLabel sets the row labels. Persists to meta file on update. @@ -143,7 +142,7 @@ func (f *Frame) Open() error { return err } - if err := f.openFragments(); err != nil { + if err := f.openViews(); err != nil { return err } @@ -160,10 +159,12 @@ func (f *Frame) Open() error { return nil } -// openFragments opens and initializes the fragments inside the frame. -func (f *Frame) openFragments() error { - file, err := os.Open(f.path) - if err != nil { +// openViews opens and initializes the views inside the frame. +func (f *Frame) openViews() error { + file, err := os.Open(filepath.Join(f.path, "views")) + if os.IsNotExist(err) { + return nil + } else if err != nil { return err } defer file.Close() @@ -174,22 +175,17 @@ func (f *Frame) openFragments() error { } for _, fi := range fis { - if fi.IsDir() { + if !fi.IsDir() { continue } - // Parse filename into integer. - slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) - if err != nil { - continue + name := filepath.Base(fi.Name()) + view := f.newView(f.ViewPath(name), name) + if err := view.Open(); err != nil { + return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) } - - frag := f.newFragment(f.FragmentPath(slice), slice) - if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) - } - frag.BitmapAttrStore = f.bitmapAttrStore - f.fragments[frag.Slice()] = frag + view.BitmapAttrStore = f.bitmapAttrStore + f.views[view.Name()] = view f.stats.Count("maxSlice", 1) } @@ -241,7 +237,7 @@ func (f *Frame) saveMeta() error { return nil } -// Close closes the frame and its fragments. +// Close closes the frame and its views. func (f *Frame) Close() error { f.mu.Lock() defer f.mu.Unlock() @@ -251,11 +247,11 @@ func (f *Frame) Close() error { _ = f.bitmapAttrStore.Close() } - // Close all fragments. - for _, frag := range f.fragments { - _ = frag.Close() + // Close all views. + for _, view := range f.views { + _ = view.Close() } - f.fragments = make(map[uint64]*Fragment) + f.views = make(map[string]*View) return nil } @@ -288,75 +284,239 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error { return nil } -// FragmentPath returns the path to a fragment in the frame. -func (f *Frame) FragmentPath(slice uint64) string { - return filepath.Join(f.path, strconv.FormatUint(slice, 10)) +// ViewPath returns the path to a view in the frame. +func (f *Frame) ViewPath(name string) string { + return filepath.Join(f.path, "views", name) } -// Fragment returns a fragment in the frame by slice. -func (f *Frame) Fragment(slice uint64) *Fragment { +// View returns a view in the frame by name. +func (f *Frame) View(name string) *View { f.mu.Lock() defer f.mu.Unlock() - return f.fragment(slice) + return f.view(name) } -func (f *Frame) fragment(slice uint64) *Fragment { return f.fragments[slice] } +func (f *Frame) view(name string) *View { return f.views[name] } -// Fragments returns a list of all fragments in the frame. -func (f *Frame) Fragments() []*Fragment { +// Views returns a list of all views in the frame. +func (f *Frame) Views() []*View { f.mu.Lock() defer f.mu.Unlock() - other := make([]*Fragment, 0, len(f.fragments)) - for _, fragment := range f.fragments { - other = append(other, fragment) + other := make([]*View, 0, len(f.views)) + for _, view := range f.views { + other = append(other, view) } return other } -// CreateFragmentIfNotExists returns a fragment in the frame by slice. -func (f *Frame) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { +func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { f.mu.Lock() defer f.mu.Unlock() - return f.createFragmentIfNotExists(slice) -} -func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) { - // Find fragment in cache first. - if frag := f.fragments[slice]; frag != nil { - return frag, nil + if view := f.views[name]; view != nil { + return view, nil } - // Initialize and open fragment. - frag := f.newFragment(f.FragmentPath(slice), slice) - if err := frag.Open(); err != nil { + view := f.newView(f.ViewPath(name), name) + if err := view.Open(); err != nil { return nil, err } - frag.BitmapAttrStore = f.bitmapAttrStore + view.BitmapAttrStore = f.bitmapAttrStore + f.views[view.Name()] = view - // Save to lookup. - f.fragments[slice] = frag - - f.stats.Count("maxSlice", 1) - - return frag, nil + return view, nil } -func (f *Frame) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, f.db, f.name, slice) - frag.LogOutput = f.LogOutput - frag.stats = f.stats.WithTags(fmt.Sprintf("slice:%d", slice)) - return frag +func (f *Frame) newView(path, name string) *View { + view := NewView(path, f.db, f.name, name) + view.LogOutput = f.LogOutput + view.BitmapAttrStore = f.bitmapAttrStore + view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name)) + return view } // SetBit sets a bit within the frame. -func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth - frag, err := f.CreateFragmentIfNotExists(slice) +func (f *Frame) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Set standard layout bits. + if v, err := f.setBit(ViewStandard, rowID, colID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + // Set inverse layout bits. + // NOTE: The row & col are transposed for the inverted view. + if v, err := f.setBit(ViewInverse, colID, rowID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + return changed, nil +} + +// setBit sets a bit for a given layout (default or inverted). +func (f *Frame) setBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) if err != nil { return changed, err } - return frag.SetBit(bitmapID, profileID) + + // Set non-time bit. + if v, err := view.SetBit(rowID, colID); err != nil { + return changed, err + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then set bits across all views for the quantum. + for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := view.SetBit(rowID, colID); err != nil { + return changed, err + } else if c { + changed = true + } + } + + return changed, nil +} + +// ClearBit clears a bit within the frame. +func (f *Frame) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Clear standard layout bits. + if v, err := f.clearBit(ViewStandard, rowID, colID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + // Clear inverse layout bits. + // NOTE: The row & col are transposed for the inverted view. + if v, err := f.clearBit(ViewInverse, colID, rowID, t); err != nil { + return changed, err + } else if v { + changed = v + } + + return changed, nil +} + +// clearBit clears a bit for a given layout (default or inverted). +func (f *Frame) clearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) + if err != nil { + return changed, err + } + + // Clear non-time bit. + if v, err := view.ClearBit(rowID, colID); err != nil { + return changed, err + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then clear bits across all views for the quantum. + for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := view.ClearBit(rowID, colID); err != nil { + return changed, err + } else if c { + changed = true + } + } + + return changed, nil +} + +// Import bulk imports data. +func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { + // Determine quantum if timestamps are set. + q := f.TimeQuantum() + if hasTime(timestamps) && q == "" { + return errors.New("time quantum not set in either database or frame") + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range bitmapIDs { + bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + + var standard, inverse []string + if timestamp == nil { + standard = []string{ViewStandard} + inverse = []string{ViewInverse} + } else { + standard = ViewsByTime(ViewStandard, *timestamp, q) + inverse = ViewsByTime(ViewInverse, *timestamp, q) + } + + // Attach bit to each standard view. + for _, name := range standard { + key := importKey{View: name, Slice: profileID / SliceWidth} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, bitmapID) + data.ProfileIDs = append(data.ProfileIDs, profileID) + dataByFragment[key] = data + } + + // Attach reversed bits to each inverse view. + for _, name := range inverse { + key := importKey{View: name, Slice: bitmapID / SliceWidth} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed + data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + // Re-sort data for inverse views. + if IsViewInverted(key.View) { + sort.Sort(importBitSet{ + bitmapIDs: data.BitmapIDs, + profileIDs: data.ProfileIDs, + }) + } + + view, err := f.CreateViewIfNotExists(key.View) + if err != nil { + return err + } + + frag, err := view.CreateFragmentIfNotExists(key.Slice) + if err != nil { + return err + } + + if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + return err + } + } + + return nil } type frameSlice []*Frame @@ -367,7 +527,8 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // FrameInfo represents schema information for a frame. type FrameInfo struct { - Name string `json:"name"` + Name string `json:"name"` + Views []*ViewInfo `json:"views,omitempty"` } type frameInfoSlice []*FrameInfo @@ -380,3 +541,16 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` } + +// importBitSet represents slices of row and column ids. +// This is used to sort data during import. +type importBitSet struct { + bitmapIDs, profileIDs []uint64 +} + +func (p importBitSet) Swap(i, j int) { + p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i] + p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i] +} +func (p importBitSet) Len() int { return len(p.bitmapIDs) } +func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] } diff --git a/frame_test.go b/frame_test.go index 5b03cfb3e..8a86f1ee8 100644 --- a/frame_test.go +++ b/frame_test.go @@ -4,33 +4,34 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/pilosa/pilosa" ) -// Ensure frame can open and retrieve a fragment. -func TestFrame_CreateFragmentIfNotExists(t *testing.T) { +// Ensure frame can open and retrieve a view. +func TestFrame_CreateViewIfNotExists(t *testing.T) { f := MustOpenFrame() defer f.Close() - // Create fragment. - frag, err := f.CreateFragmentIfNotExists(100) + // Create view. + view, err := f.CreateViewIfNotExists("v") if err != nil { t.Fatal(err) - } else if frag == nil { - t.Fatal("expected fragment") + } else if view == nil { + t.Fatal("expected view") } - // Retrieve existing fragment. - frag2, err := f.CreateFragmentIfNotExists(100) + // Retrieve existing view. + view2, err := f.CreateViewIfNotExists("v") if err != nil { t.Fatal(err) - } else if frag != frag2 { - t.Fatal("fragment mismatch") + } else if view != view2 { + t.Fatal("view mismatch") } - if frag != f.Fragment(100) { - t.Fatal("fragment mismatch") + if view != f.View("v") { + t.Fatal("view mismatch") } } @@ -54,6 +55,17 @@ func TestFrame_SetTimeQuantum(t *testing.T) { } } +func TestFrame_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + frame, err := pilosa.NewFrame(path, "d", ".meta") + if frame != nil { + t.Fatalf("unexpected frame name %s", err) + } +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame @@ -106,14 +118,11 @@ func (f *Frame) Reopen() error { return nil } -// NewFrame does not return a frame when name is invalid. -func TestFrame_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-frame-") +// MustSetBit sets a bit on the frame. Panic on error. +func (f *Frame) MustSetBit(bitmapID, profileID uint64, t *time.Time) (changed bool) { + changed, err := f.SetBit(bitmapID, profileID, t) if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "d", ".meta") - if frame != nil { - t.Fatalf("unexpected frame name %s", err) - } + return changed } diff --git a/handler.go b/handler.go index 1ccdf3711..03c56ad09 100644 --- a/handler.go +++ b/handler.go @@ -145,6 +145,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/frame/views": + switch r.Method { + case "GET": + h.handleGetFrameViews(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/frame/attr/diff": switch r.Method { case "POST": @@ -583,6 +590,35 @@ type patchFrameTimeQuantumRequest struct { type patchFrameTimeQuantumResponse struct{} +// handleGetFrameViews handles GET /frame/views request. +func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + db, frame := q.Get("db"), q.Get("frame") + + // Retrieve views. + f := h.Index.Frame(db, frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Fetch views. + views := f.Views() + names := make([]string, len(views)) + for i := range views { + names[i] = views[i].Name() + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type getFrameViewsResponse struct { + Views []string `json:"views,omitempty"` +} + // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -791,7 +827,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Find the correct fragment. + // Find the DB. h.logger().Println("importing:", req.DB, req.Frame, req.Slice) db := h.Index.DB(req.DB) if db == nil { @@ -800,8 +836,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } + // Retrieve frame. + f := db.Frame(req.Frame) + if f == nil { + h.logger().Printf("frame error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrFrameNotFound.Error()) + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + // Import into fragment. - err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps) + err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps) if err != nil { h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) return @@ -834,7 +878,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Parse query parameters. q := r.URL.Query() - db, frame := q.Get("db"), q.Get("frame") + db, frame, view := q.Get("db"), q.Get("frame"), q.Get("view") slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { @@ -850,7 +894,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Find the fragment. - f := h.Index.Fragment(db, frame, slice) + f := h.Index.Fragment(db, frame, view, slice) if f == nil { return } @@ -905,7 +949,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice) + f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -934,8 +978,15 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } + // Retrieve view. + view, err := f.CreateViewIfNotExists(q.Get("view")) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Retrieve fragment from frame. - frag, err := f.CreateFragmentIfNotExists(slice) + frag, err := view.CreateFragmentIfNotExists(slice) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -961,7 +1012,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from index. - f := h.Index.Fragment(req.DB, req.Frame, req.Slice) + f := h.Index.Fragment(req.DB, req.Frame, req.View, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -997,7 +1048,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request } // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice) + f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1050,6 +1101,24 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) return } + // Retrieve frame. + f := h.Index.Frame(db, frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Retrieve list of all views. + views, err := client.FrameViews(r.Context(), db, frame) + if err != nil { + http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) + return + } + + for _, view := range views { + println("dbg/views", view) + } + // Loop over each slice and import it if this node owns it. //travis for slice := uint64(0); slice <= maxSlices[db]; slice++ { @@ -1058,39 +1127,42 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) continue } - // Retrieve frame. - f := h.Index.Frame(db, frame) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Otherwise retrieve the local fragment. - frag, err := f.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), db, frame, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return err + // Loop over view names. + for _, view := range views { + // Create view. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Otherwise retrieve the local fragment. + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Stream backup from remote node. + rd, err := client.BackupSlice(r.Context(), db, frame, view, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } else if rd == nil { + continue // slice doesn't exist + } + + // Restore to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return } } } diff --git a/handler_test.go b/handler_test.go index be88d6d9a..2fdba657b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -37,10 +37,14 @@ func TestHandler_Schema(t *testing.T) { d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) - if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } - if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := f.SetBit(0, 0, nil); err != nil { t.Fatal(err) } if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { @@ -53,7 +57,7 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1"}]},{"name":"d1","frames":[{"name":"f0"}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"inverse"},{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -63,13 +67,13 @@ func TestHandler_MaxSlices(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d0", "f0", 3).MustSetBits(30, (3*SliceWidth)+4) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+8) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := NewHandler() h.Index = idx.Index @@ -654,11 +658,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer s.Close() // Set bits in the index. - f0 := idx.MustCreateFragmentIfNotExists("d", "f", 0) + f0 := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) // Begin backing up from slice d/f/0. - resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&slice=0") + resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0") if err != nil { t.Fatal(err) } @@ -675,7 +679,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Restore backup to slice x/y/0. - if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil { + if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { t.Fatal(err) } else if resp.StatusCode != http.StatusOK { resp.Body.Close() @@ -685,9 +689,9 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Verify data is correctly restored. - f1 := idx.Fragment("x", "y", 0) + f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { - t.Fatal("fragment x/y/0 not created") + t.Fatal("fragment x/y/standard/0 not created") } else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { t.Fatalf("unexpected restored bits: %+v", bits) } diff --git a/index.go b/index.go index 07d13a030..3b45dd438 100644 --- a/index.go +++ b/index.go @@ -131,12 +131,17 @@ func (i *Index) Schema() []*DBInfo { for _, db := range i.DBs() { di := &DBInfo{Name: db.Name()} for _, frame := range db.Frames() { - di.Frames = append(di.Frames, &FrameInfo{ - Name: frame.Name(), - }) + fi := &FrameInfo{Name: frame.Name()} + for _, view := range frame.Views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + } + sort.Sort(viewInfoSlice(fi.Views)) + di.Frames = append(di.Frames, fi) } + sort.Sort(frameInfoSlice(di.Frames)) a = append(a, di) } + sort.Sort(dbInfoSlice(a)) return a } @@ -270,13 +275,22 @@ func (i *Index) Frame(db, name string) *Frame { return d.Frame(name) } -// Fragment returns the fragment for a database, frame & slice. -func (i *Index) Fragment(db, frame string, slice uint64) *Fragment { +// View returns the view for a database, frame, and name. +func (i *Index) View(db, frame, name string) *View { f := i.Frame(db, frame) if f == nil { return nil } - return f.Fragment(slice) + return f.View(name) +} + +// Fragment returns the fragment for a database, frame & slice. +func (i *Index) Fragment(db, frame, view string, slice uint64) *Fragment { + v := i.View(db, frame, view) + if v == nil { + return nil + } + return v.Fragment(slice) } // monitorCacheFlush periodically flushes all fragment caches sequentially. @@ -298,15 +312,17 @@ func (i *Index) monitorCacheFlush() { func (i *Index) flushCaches() { for _, db := range i.DBs() { for _, frame := range db.Frames() { - for _, fragment := range frame.Fragments() { - select { - case <-i.closing: - return - default: - } + for _, view := range frame.Views() { + for _, fragment := range view.Fragments() { + select { + case <-i.closing: + return + default: + } - if err := fragment.FlushCache(); err != nil { - i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + if err := fragment.FlushCache(); err != nil { + i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + } } } } @@ -362,20 +378,27 @@ func (s *IndexSyncer) SyncIndex() error { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } - for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { - // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { - continue - } - + for _, vi := range fi.Views { // Verify syncer has not closed. if s.IsClosing() { return nil } - // Sync fragment if own it. - if err := s.syncFragment(di.Name, fi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { + // Ignore slices that this host doesn't own. + if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { + continue + } + + // Verify syncer has not closed. + if s.IsClosing() { + return nil + } + + // Sync fragment if own it. + if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { + return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + } } } } @@ -477,15 +500,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error { +func (s *IndexSyncer) syncFragment(db, frame, view string, slice uint64) error { // Retrieve local frame. f := s.Index.Frame(db, frame) if f == nil { return ErrFrameNotFound } + // Ensure view exists locally. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + return err + } + // Ensure fragment exists locally. - frag, err := f.CreateFragmentIfNotExists(slice) + frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return err } diff --git a/index_test.go b/index_test.go index dace25fcc..4a1be4cf2 100644 --- a/index_test.go +++ b/index_test.go @@ -18,11 +18,11 @@ func TestIndex_DeleteDB(t *testing.T) { defer idx.Close() // Write bits to separate databases. - f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0) + f0 := idx.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } - f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0) + f1 := idx.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0) if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } // Set data on the local index. - f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) + f := idx0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 20); err != nil { @@ -91,15 +91,15 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1) + f = idx0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1) if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } - idx0.MustCreateFragmentIfNotExists("y", "z", 0) + idx0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) // Set data on the remote index. - f = idx1.MustCreateFragmentIfNotExists("d", "f", 0) + f = idx1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) } else if _, err := f.SetBit(3, 10); err != nil { @@ -108,7 +108,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx1.MustCreateFragmentIfNotExists("y", "z", 3) + f = idx1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { @@ -134,7 +134,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Verify data is the same on both nodes. for i, idx := range []*Index{idx0, idx1} { - f := idx.Fragment("d", "f", 0) + f := idx.Fragment("d", "f", pilosa.ViewStandard, 0) if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) } else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { @@ -147,7 +147,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } - f = idx.Fragment("d", "f0", 1) + f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1) a := f.Bitmap(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) @@ -155,7 +155,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - f = idx.Fragment("y", "z", 3) + f = idx.Fragment("y", "z", pilosa.ViewStandard, 3) if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) } @@ -212,17 +212,21 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { if err != nil { panic(err) } - return &Frame{Frame: f} + return f } // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { +func (i *Index) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment { d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } - frag, err := f.CreateFragmentIfNotExists(slice) + v, err := f.CreateViewIfNotExists(view) + if err != nil { + panic(err) + } + frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { panic(err) } diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 85f4102bd..2314036cd 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -242,8 +242,9 @@ func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInter type BlockDataRequest struct { DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` + View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` + Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } @@ -907,15 +908,21 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) i += copy(dAtA[i:], m.Frame) } - if m.Slice != 0 { + if m.Block != 0 { dAtA[i] = 0x18 i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + } + if m.Slice != 0 { + dAtA[i] = 0x20 + i++ i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) } - if m.Block != 0 { - dAtA[i] = 0x20 + if len(m.View) > 0 { + dAtA[i] = 0x2a i++ - i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + i = encodeVarintInternal(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) } return i, nil } @@ -1329,11 +1336,15 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovInternal(uint64(l)) } + if m.Block != 0 { + n += 1 + sovInternal(uint64(m.Block)) + } if m.Slice != 0 { n += 1 + sovInternal(uint64(m.Slice)) } - if m.Block != 0 { - n += 1 + sovInternal(uint64(m.Block)) + l = len(m.View) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) } return n } @@ -3330,25 +3341,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { m.Frame = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) - } - m.Slice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowInternal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) } @@ -3367,6 +3359,54 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -3938,51 +3978,52 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 735 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd4, 0x4a, - 0x10, 0xbd, 0x3d, 0xb6, 0xe7, 0x51, 0x93, 0x8c, 0x26, 0xad, 0xb9, 0x57, 0x56, 0x74, 0x35, 0xb2, - 0x5a, 0xf7, 0x4a, 0x16, 0x12, 0x89, 0x14, 0x36, 0x08, 0x21, 0x21, 0x3c, 0x33, 0x51, 0x46, 0x90, - 0x28, 0xe9, 0x04, 0x76, 0x2c, 0x3a, 0x49, 0x93, 0x58, 0xf1, 0x63, 0xb0, 0xdb, 0xc0, 0x2c, 0x59, - 0xf0, 0x0f, 0x08, 0xbe, 0x00, 0xbe, 0x84, 0x25, 0x9f, 0x80, 0xc2, 0x8f, 0xa0, 0x7e, 0xf8, 0x11, - 0x10, 0x21, 0x0b, 0x76, 0xae, 0x53, 0x5d, 0xd5, 0x75, 0xfa, 0x54, 0x95, 0x61, 0x10, 0x26, 0x82, - 0x67, 0x09, 0x8b, 0x36, 0x16, 0x59, 0x2a, 0x52, 0xdc, 0x2d, 0x6d, 0xb2, 0x03, 0xad, 0x69, 0x80, - 0x3d, 0xe8, 0x1f, 0x85, 0x31, 0x3f, 0x28, 0x58, 0x22, 0x8a, 0xd8, 0x45, 0x1e, 0xf2, 0x7b, 0xb4, - 0x09, 0xc9, 0x13, 0x93, 0x34, 0x2a, 0xe2, 0xe4, 0x31, 0x3b, 0xe6, 0x91, 0xdb, 0xd2, 0x27, 0x1a, - 0x10, 0x99, 0x81, 0xb3, 0x9d, 0xb1, 0x98, 0xdf, 0x20, 0xd9, 0x3a, 0x74, 0x69, 0xfa, 0xaa, 0x99, - 0xa9, 0xb2, 0x49, 0x00, 0xed, 0x20, 0x14, 0x31, 0x5b, 0x60, 0x0c, 0x76, 0x10, 0x8a, 0xdc, 0x45, - 0x9e, 0xe5, 0xdb, 0x54, 0x7d, 0xe3, 0xff, 0xc0, 0x79, 0x28, 0x44, 0x96, 0xbb, 0x2d, 0xcf, 0xf2, - 0xfb, 0x5b, 0x83, 0x8d, 0x8a, 0x98, 0x84, 0xa9, 0x76, 0x92, 0x0d, 0xb0, 0xf7, 0x59, 0x98, 0xe1, - 0x21, 0x58, 0x8f, 0xf8, 0x52, 0x55, 0x60, 0x53, 0xf9, 0x89, 0x47, 0xe0, 0x4c, 0xd2, 0x22, 0x11, - 0xea, 0x5a, 0x9b, 0x6a, 0x83, 0x3c, 0x03, 0x2b, 0x08, 0x85, 0x2c, 0x4b, 0x5f, 0x3d, 0x9f, 0x9a, - 0x98, 0xca, 0xc6, 0xff, 0x42, 0x6f, 0x3f, 0x4b, 0x9f, 0x87, 0x11, 0x9f, 0x4f, 0x4d, 0x70, 0x0d, - 0x48, 0xaf, 0xe4, 0x97, 0x0b, 0x16, 0x2f, 0x5c, 0xcb, 0x43, 0xbe, 0x45, 0x6b, 0x80, 0x3c, 0x80, - 0x8e, 0x39, 0x8a, 0x07, 0xd0, 0xaa, 0x92, 0xb7, 0xe6, 0xd3, 0x1b, 0xf2, 0xf9, 0x84, 0xc0, 0x96, - 0x5f, 0x4d, 0x42, 0x3d, 0x4d, 0x08, 0x83, 0x7d, 0xb4, 0x5c, 0x70, 0x53, 0x92, 0xfa, 0x96, 0x02, - 0x1c, 0x8a, 0x2c, 0x4c, 0xce, 0x9e, 0xb2, 0xa8, 0xe0, 0xaa, 0x9e, 0x1e, 0x6d, 0x42, 0xb2, 0xde, - 0x27, 0x61, 0x22, 0xb4, 0xdf, 0xd6, 0x6c, 0x2a, 0x40, 0x7a, 0x83, 0x34, 0x8d, 0xb4, 0xd7, 0xf1, - 0x90, 0xdf, 0xa5, 0x35, 0x80, 0xc7, 0x00, 0xdb, 0x51, 0xca, 0x4c, 0x70, 0xdb, 0x43, 0x3e, 0xa2, - 0x0d, 0x84, 0x6c, 0x42, 0x47, 0xd6, 0xba, 0xcb, 0x16, 0x35, 0x3b, 0x74, 0x1d, 0xbb, 0xf7, 0x08, - 0x56, 0x0e, 0x0a, 0x9e, 0x2d, 0x29, 0x7f, 0x51, 0xf0, 0x5c, 0xc8, 0x47, 0x9a, 0x06, 0x86, 0xa4, - 0xec, 0xce, 0x11, 0x38, 0xca, 0x6f, 0x7a, 0x45, 0x1b, 0xf8, 0x1f, 0x68, 0x1f, 0x46, 0xe1, 0x09, - 0xcf, 0x5d, 0x4b, 0x35, 0x88, 0xb1, 0xa4, 0x8a, 0xe6, 0xb5, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b, - 0xbb, 0xd0, 0x29, 0xdb, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4, - 0xa5, 0xc6, 0x22, 0x6f, 0x10, 0xac, 0x9a, 0xe2, 0xf2, 0x45, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcb, - 0xb2, 0x52, 0x83, 0x59, 0x96, 0xe1, 0x4d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb, - 0x26, 0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xbb, 0x51, 0xa2, 0xa5, 0x22, 0xd6, 0xea, - 0x08, 0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0x8e, 0x88, 0x2a, 0xa2, - 0xbf, 0x35, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x11, 0x5a, 0x01, 0xb4, 0x67, 0x5a, 0x03, 0xed, 0x49, - 0x39, 0xe4, 0x58, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xf9, 0x46, 0x93, 0x73, 0x96, - 0x9c, 0xf1, 0x53, 0xf3, 0x7c, 0xa5, 0x49, 0x3e, 0x22, 0x58, 0x9d, 0xc7, 0x8b, 0x34, 0x13, 0xd7, - 0x28, 0xa5, 0x76, 0x40, 0xa9, 0x94, 0x5e, 0x08, 0x23, 0x70, 0x94, 0x36, 0xaa, 0x13, 0x6d, 0xaa, - 0x0d, 0xd5, 0x65, 0x66, 0xba, 0xa4, 0x50, 0x52, 0xc2, 0x1a, 0x90, 0x5d, 0x56, 0x8d, 0x57, 0xee, - 0x3a, 0xca, 0xdd, 0x40, 0xa4, 0xbf, 0x1a, 0xb0, 0xdc, 0x6d, 0x7b, 0x96, 0x6f, 0xd1, 0x06, 0x42, - 0x08, 0x0c, 0xca, 0x52, 0x7f, 0xa5, 0x1b, 0x39, 0x85, 0x61, 0x10, 0xa5, 0x27, 0x17, 0x53, 0x26, - 0xd8, 0x9f, 0x60, 0x34, 0x02, 0x47, 0xe5, 0x33, 0x13, 0xa5, 0x0d, 0x72, 0x00, 0x6b, 0x8d, 0x5b, - 0x4c, 0x31, 0x57, 0xc8, 0xa3, 0xeb, 0xc9, 0xb7, 0x7e, 0x24, 0x4f, 0xfe, 0x07, 0x67, 0xc2, 0x4e, - 0xce, 0x7f, 0x93, 0x86, 0x7c, 0x40, 0xb0, 0xb6, 0xcb, 0x5e, 0xeb, 0xb9, 0xa8, 0xae, 0xde, 0x81, - 0x5e, 0x05, 0x9a, 0xc1, 0xbc, 0x55, 0x77, 0xc2, 0x4f, 0xe7, 0x6b, 0x64, 0x96, 0x88, 0x6c, 0x49, - 0xeb, 0xe0, 0xf5, 0xfb, 0x30, 0xb8, 0xea, 0x94, 0x6f, 0x7c, 0x51, 0xef, 0xa7, 0x0b, 0xbd, 0x70, - 0x5f, 0xaa, 0x45, 0x61, 0x16, 0xae, 0x32, 0xee, 0xb5, 0xee, 0xa2, 0x60, 0xf8, 0xf9, 0x72, 0x8c, - 0xbe, 0x5c, 0x8e, 0xd1, 0xd7, 0xcb, 0x31, 0x7a, 0xf7, 0x6d, 0xfc, 0xd7, 0x71, 0x5b, 0xfd, 0x9c, - 0xee, 0x7c, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x22, 0xbf, 0xb0, 0xae, 0x06, 0x00, 0x00, + // 746 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6a, 0x14, 0x4b, + 0x10, 0x3e, 0xbd, 0x33, 0xb3, 0x3f, 0xb5, 0xc9, 0xb2, 0x69, 0x72, 0x0e, 0x43, 0x38, 0x2c, 0x43, + 0x73, 0x0e, 0x0c, 0x82, 0x09, 0xc4, 0x1b, 0x11, 0x41, 0x9c, 0xdd, 0x0d, 0x59, 0x34, 0x21, 0xe9, + 0xc4, 0xdc, 0x79, 0xd1, 0x89, 0x6d, 0x32, 0x64, 0x7e, 0xd6, 0x99, 0x1e, 0x93, 0xbd, 0xf4, 0xc2, + 0x77, 0x10, 0x7d, 0x02, 0x7d, 0x12, 0x2f, 0x7d, 0x04, 0x89, 0x2f, 0x22, 0xfd, 0x33, 0x3f, 0x51, + 0x8c, 0xf1, 0x6e, 0xea, 0xab, 0xae, 0xea, 0xfa, 0xea, 0xab, 0xea, 0x81, 0x41, 0x98, 0x08, 0x9e, + 0x25, 0x2c, 0x5a, 0x9f, 0x67, 0xa9, 0x48, 0x71, 0xb7, 0xb4, 0xc9, 0x36, 0xb4, 0x26, 0x01, 0xf6, + 0xa0, 0x7f, 0x18, 0xc6, 0x7c, 0xbf, 0x60, 0x89, 0x28, 0x62, 0x17, 0x79, 0xc8, 0xef, 0xd1, 0x26, + 0x24, 0x4f, 0x8c, 0xd3, 0xa8, 0x88, 0x93, 0xa7, 0xec, 0x98, 0x47, 0x6e, 0x4b, 0x9f, 0x68, 0x40, + 0x64, 0x0a, 0xce, 0x56, 0xc6, 0x62, 0x7e, 0x8b, 0x64, 0x6b, 0xd0, 0xa5, 0xe9, 0x45, 0x33, 0x53, + 0x65, 0x93, 0x00, 0xda, 0x41, 0x28, 0x62, 0x36, 0xc7, 0x18, 0xec, 0x20, 0x14, 0xb9, 0x8b, 0x3c, + 0xcb, 0xb7, 0xa9, 0xfa, 0xc6, 0xff, 0x81, 0xf3, 0x58, 0x88, 0x2c, 0x77, 0x5b, 0x9e, 0xe5, 0xf7, + 0x37, 0x07, 0xeb, 0x15, 0x31, 0x09, 0x53, 0xed, 0x24, 0xeb, 0x60, 0xef, 0xb1, 0x30, 0xc3, 0x43, + 0xb0, 0x9e, 0xf0, 0x85, 0xaa, 0xc0, 0xa6, 0xf2, 0x13, 0xaf, 0x82, 0x33, 0x4e, 0x8b, 0x44, 0xa8, + 0x6b, 0x6d, 0xaa, 0x0d, 0xf2, 0x1c, 0xac, 0x20, 0x14, 0xb2, 0x2c, 0x7d, 0xf5, 0x6c, 0x62, 0x62, + 0x2a, 0x1b, 0xff, 0x0b, 0xbd, 0xbd, 0x2c, 0x7d, 0x19, 0x46, 0x7c, 0x36, 0x31, 0xc1, 0x35, 0x20, + 0xbd, 0x92, 0x5f, 0x2e, 0x58, 0x3c, 0x77, 0x2d, 0x0f, 0xf9, 0x16, 0xad, 0x01, 0xf2, 0x08, 0x3a, + 0xe6, 0x28, 0x1e, 0x40, 0xab, 0x4a, 0xde, 0x9a, 0x4d, 0x6e, 0xc9, 0xe7, 0x13, 0x02, 0x5b, 0x7e, + 0x35, 0x09, 0xf5, 0x34, 0x21, 0x0c, 0xf6, 0xe1, 0x62, 0xce, 0x4d, 0x49, 0xea, 0x5b, 0x0a, 0x70, + 0x20, 0xb2, 0x30, 0x39, 0x3d, 0x62, 0x51, 0xc1, 0x55, 0x3d, 0x3d, 0xda, 0x84, 0x64, 0xbd, 0xcf, + 0xc2, 0x44, 0x68, 0xbf, 0xad, 0xd9, 0x54, 0x80, 0xf4, 0x06, 0x69, 0x1a, 0x69, 0xaf, 0xe3, 0x21, + 0xbf, 0x4b, 0x6b, 0x00, 0x8f, 0x00, 0xb6, 0xa2, 0x94, 0x99, 0xe0, 0xb6, 0x87, 0x7c, 0x44, 0x1b, + 0x08, 0xd9, 0x80, 0x8e, 0xac, 0x75, 0x87, 0xcd, 0x6b, 0x76, 0xe8, 0x26, 0x76, 0xef, 0x11, 0x2c, + 0xed, 0x17, 0x3c, 0x5b, 0x50, 0xfe, 0xaa, 0xe0, 0xb9, 0x90, 0x4d, 0x9a, 0x04, 0x86, 0xa4, 0x9c, + 0xce, 0x55, 0x70, 0x94, 0xdf, 0xcc, 0x8a, 0x36, 0xf0, 0x3f, 0xd0, 0x3e, 0x88, 0xc2, 0x13, 0x9e, + 0xbb, 0x96, 0x1a, 0x10, 0x63, 0x49, 0x15, 0x4d, 0xb7, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b, 0xbb, + 0xd0, 0x29, 0xc7, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4, 0xa5, + 0xc6, 0x22, 0x6f, 0x10, 0x2c, 0x9b, 0xe2, 0xf2, 0x79, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcd, 0xb2, + 0x52, 0x83, 0x69, 0x96, 0xe1, 0x0d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb, 0x26, + 0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xb7, 0x51, 0xa2, 0xa5, 0x22, 0x56, 0xea, 0x08, + 0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0xae, 0x88, 0x2a, 0xa2, 0xbf, + 0x39, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x15, 0x5a, 0x02, 0xb4, 0x6b, 0x46, 0x03, 0xed, 0x4a, 0x39, + 0xe4, 0x5a, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xd9, 0xa3, 0xf1, 0x19, 0x4b, 0x4e, + 0xf9, 0x0b, 0xd3, 0xbe, 0xd2, 0x24, 0x1f, 0x11, 0x2c, 0xcf, 0xe2, 0x79, 0x9a, 0x89, 0x1b, 0x94, + 0x52, 0x6f, 0x40, 0xa9, 0x94, 0x7e, 0x10, 0x56, 0xc1, 0x51, 0xda, 0xa8, 0x49, 0xb4, 0xa9, 0x36, + 0xd4, 0x94, 0x99, 0xed, 0x92, 0x42, 0x49, 0x09, 0x6b, 0x40, 0x4e, 0x59, 0xb5, 0x5e, 0xb9, 0xeb, + 0x28, 0x77, 0x03, 0x91, 0xfe, 0x6a, 0xc1, 0x72, 0xb7, 0xed, 0x59, 0xbe, 0x45, 0x1b, 0x08, 0x21, + 0x30, 0x28, 0x4b, 0xfd, 0x95, 0x6e, 0xe4, 0x12, 0x86, 0x41, 0x94, 0x9e, 0x9c, 0x4f, 0x98, 0x60, + 0x7f, 0xcc, 0x48, 0x45, 0x96, 0x8c, 0x94, 0x51, 0xf3, 0xb4, 0x9b, 0x3c, 0x31, 0xd8, 0x47, 0x21, + 0xbf, 0x30, 0x03, 0xa7, 0xbe, 0xc9, 0x3e, 0xac, 0x34, 0x6e, 0x36, 0x05, 0x5e, 0x6b, 0x08, 0xba, + 0xb9, 0x21, 0xad, 0x1f, 0x1b, 0x42, 0xfe, 0x07, 0x67, 0xcc, 0x4e, 0xce, 0x7e, 0x93, 0x86, 0x7c, + 0x40, 0xb0, 0xb2, 0xc3, 0x2e, 0xf5, 0xae, 0x54, 0x57, 0x6f, 0x43, 0xaf, 0x02, 0xcd, 0xb2, 0xde, + 0xa9, 0xa7, 0xe3, 0xa7, 0xf3, 0x35, 0x32, 0x4d, 0x44, 0xb6, 0xa0, 0x75, 0xf0, 0xda, 0x43, 0x18, + 0x5c, 0x77, 0xca, 0xbe, 0x9f, 0xd7, 0x6f, 0xd6, 0xb9, 0x7e, 0x84, 0x5f, 0xab, 0xc7, 0xc3, 0x3c, + 0xc2, 0xca, 0x78, 0xd0, 0xba, 0x8f, 0x82, 0xe1, 0xe7, 0xab, 0x11, 0xfa, 0x72, 0x35, 0x42, 0x5f, + 0xaf, 0x46, 0xe8, 0xdd, 0xb7, 0xd1, 0x5f, 0xc7, 0x6d, 0xf5, 0xc3, 0xba, 0xf7, 0x3d, 0x00, 0x00, + 0xff, 0xff, 0x1c, 0xb1, 0xe1, 0x29, 0xc2, 0x06, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 67000243c..e256b35b5 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -84,8 +84,9 @@ message ImportResponse { message BlockDataRequest { string DB = 1; string Frame = 2; - uint64 Slice = 3; - uint64 Block = 4; + string View = 5; + uint64 Slice = 4; + uint64 Block = 3; } message BlockDataResponse { diff --git a/time.go b/time.go index b117023ad..b2176285b 100644 --- a/time.go +++ b/time.go @@ -48,8 +48,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -// FrameByTimeUnit returns the frame name for time with a given quantum unit. -func FrameByTimeUnit(name string, t time.Time, unit rune) string { +// ViewByTimeUnit returns the view name for time with a given quantum unit. +func ViewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -64,21 +64,21 @@ func FrameByTimeUnit(name string, t time.Time, unit rune) string { } } -// FramesByTime returns a list of frames for a given timestamp. -func FramesByTime(name string, t time.Time, q TimeQuantum) []string { +// ViewsByTime returns a list of views for a given timestamp. +func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - frame := FrameByTimeUnit(name, t, unit) - if frame == "" { + view := ViewByTimeUnit(name, t, unit) + if view == "" { continue } - a = append(a, frame) + a = append(a, view) } return a } -// FramesByTimeRange returns a list of frames to traverse to query a time range. -func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// ViewsByTimeRange returns a list of views to traverse to query a time range. +func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -96,7 +96,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, FrameByTimeUnit(name, t, 'H')) + results = append(results, ViewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -107,7 +107,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, FrameByTimeUnit(name, t, 'D')) + results = append(results, ViewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -117,7 +117,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, FrameByTimeUnit(name, t, 'M')) + results = append(results, ViewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -133,16 +133,16 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'Y')) + results = append(results, ViewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'M')) + results = append(results, ViewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, FrameByTimeUnit(name, t, 'D')) + results = append(results, ViewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, FrameByTimeUnit(name, t, 'H')) + results = append(results, ViewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_test.go index 28d86555d..6de3df364 100644 --- a/time_test.go +++ b/time_test.go @@ -25,45 +25,45 @@ func TestParseTimeQuantum(t *testing.T) { }) } -// Ensure generated frame name can be returned for a given time unit. -func TestFrameByTimeUnit(t *testing.T) { +// Ensure generated view name can be returned for a given time unit. +func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) } // Ensure all applicable frame names can be generated when mutating a time bit. -func TestFramesByTime(t *testing.T) { +func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D")) + a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } @@ -71,63 +71,63 @@ func TestFramesByTime(t *testing.T) { } // Ensure sets of frames can be returned for a given time range. -func TestFramesByTimeRange(t *testing.T) { +func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { t.Fatalf("unexpected frames: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { t.Fatalf("unexpected frames: %#v", a) } diff --git a/view.go b/view.go new file mode 100644 index 000000000..b53cea6aa --- /dev/null +++ b/view.go @@ -0,0 +1,255 @@ +package pilosa + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" +) + +// View layout modes. +const ( + ViewStandard = "standard" + ViewInverse = "inverse" +) + +// View represents a container for frame data. +type View struct { + mu sync.Mutex + path string + db string + frame string + name string + + // Fragments by slice. + fragments map[uint64]*Fragment + + stats StatsClient + + BitmapAttrStore *AttrStore + LogOutput io.Writer +} + +// NewView returns a new instance of View. +func NewView(path, db, frame, name string) *View { + return &View{ + path: path, + db: db, + frame: frame, + name: name, + + fragments: make(map[uint64]*Fragment), + + stats: NopStatsClient, + LogOutput: ioutil.Discard, + } +} + +// Name returns the name the view was initialized with. +func (v *View) Name() string { return v.name } + +// DB returns the database name the view was initialized with. +func (v *View) DB() string { return v.db } + +// Frame returns the frame name the view was initialized with. +func (v *View) Frame() string { return v.frame } + +// Path returns the path the view was initialized with. +func (v *View) Path() string { return v.path } + +// Open opens and initializes the view. +func (v *View) Open() error { + if err := func() error { + // Ensure the view's path exists. + if err := os.MkdirAll(v.path, 0777); err != nil { + return err + } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { + return err + } + + if err := v.openFragments(); err != nil { + return err + } + + return nil + }(); err != nil { + v.Close() + return err + } + + return nil +} + +// openFragments opens and initializes the fragments inside the view. +func (v *View) openFragments() error { + file, err := os.Open(filepath.Join(v.path, "fragments")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + defer file.Close() + + fis, err := file.Readdir(0) + if err != nil { + return err + } + + for _, fi := range fis { + if fi.IsDir() { + continue + } + + // Parse filename into integer. + slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + if err != nil { + continue + } + + frag := v.newFragment(v.FragmentPath(slice), slice) + if err := frag.Open(); err != nil { + return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) + } + frag.BitmapAttrStore = v.BitmapAttrStore + v.fragments[frag.Slice()] = frag + + v.stats.Count("maxSlice", 1) + } + + return nil +} + +// Close closes the view and its fragments. +func (v *View) Close() error { + v.mu.Lock() + defer v.mu.Unlock() + + // Close all fragments. + for _, frag := range v.fragments { + _ = frag.Close() + } + v.fragments = make(map[uint64]*Fragment) + + return nil +} + +// MaxSlice returns the max slice in the view. +func (v *View) MaxSlice() uint64 { + v.mu.Lock() + defer v.mu.Unlock() + + var max uint64 + for slice := range v.fragments { + if slice > max { + max = slice + } + } + return max +} + +// FragmentPath returns the path to a fragment in the view. +func (v *View) FragmentPath(slice uint64) string { + return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) +} + +// Fragment returns a fragment in the view by slice. +func (v *View) Fragment(slice uint64) *Fragment { + v.mu.Lock() + defer v.mu.Unlock() + return v.fragment(slice) +} + +func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } + +// Fragments returns a list of all fragments in the view. +func (v *View) Fragments() []*Fragment { + v.mu.Lock() + defer v.mu.Unlock() + + other := make([]*Fragment, 0, len(v.fragments)) + for _, fragment := range v.fragments { + other = append(other, fragment) + } + return other +} + +// CreateFragmentIfNotExists returns a fragment in the view by slice. +func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) { + v.mu.Lock() + defer v.mu.Unlock() + return v.createFragmentIfNotExists(slice) +} + +func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { + // Find fragment in cache first. + if frag := v.fragments[slice]; frag != nil { + return frag, nil + } + + // Initialize and open fragment. + frag := v.newFragment(v.FragmentPath(slice), slice) + if err := frag.Open(); err != nil { + return nil, err + } + frag.BitmapAttrStore = v.BitmapAttrStore + + // Save to lookup. + v.fragments[slice] = frag + + v.stats.Count("maxSlice", 1) + + return frag, nil +} + +func (v *View) newFragment(path string, slice uint64) *Fragment { + frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag.LogOutput = v.LogOutput + frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) + return frag +} + +// SetBit sets a bit within the view. +func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := profileID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetBit(bitmapID, profileID) +} + +// ClearBit clears a bit within the view. +func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := profileID / SliceWidth + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.ClearBit(bitmapID, profileID) +} + +// IsViewInverted returns true if the view is used for storing an inverted representation. +func IsViewInverted(name string) bool { + return strings.HasPrefix(name, ViewInverse) +} + +type viewSlice []*View + +func (p viewSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p viewSlice) Len() int { return len(p) } +func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } + +// ViewInfo represents schema information for a view. +type ViewInfo struct { + Name string `json:"name"` +} + +type viewInfoSlice []*ViewInfo + +func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p viewInfoSlice) Len() int { return len(p) } +func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } diff --git a/view_test.go b/view_test.go new file mode 100644 index 000000000..87b3e5826 --- /dev/null +++ b/view_test.go @@ -0,0 +1,80 @@ +package pilosa_test + +import ( + "io/ioutil" + "os" + + "github.com/pilosa/pilosa" +) + +// View is a test wrapper for pilosa.View. +type View struct { + *pilosa.View + BitmapAttrStore *AttrStore +} + +// NewView returns a new instance of View with a temporary path. +func NewView(db, frame, name string) *View { + file, err := ioutil.TempFile("", "pilosa-view-") + if err != nil { + panic(err) + } + file.Close() + + v := &View{ + View: pilosa.NewView(file.Name(), db, frame, name), + BitmapAttrStore: MustOpenAttrStore(), + } + v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + return v +} + +// MustOpenView creates and opens an view at a temporary path. Panic on error. +func MustOpenView(db, frame, name string) *View { + v := NewView(db, frame, name) + if err := v.Open(); err != nil { + panic(err) + } + return v +} + +// Close closes the view and removes all underlying data. +func (v *View) Close() error { + defer os.Remove(v.Path()) + defer v.BitmapAttrStore.Close() + return v.View.Close() +} + +// Reopen closes the view and reopens it as a new instance. +func (v *View) Reopen() error { + path := v.Path() + if err := v.View.Close(); err != nil { + return err + } + + v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name()) + v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + if err := v.Open(); err != nil { + return err + } + return nil +} + +// MustSetBits sets bits on a bitmap. Panic on error. +// This function does not accept a timestamp or quantum. +func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if _, err := v.SetBit(bitmapID, profileID); err != nil { + panic(err) + } + } +} + +// MustClearBits clears bits on a bitmap. Panic on error. +func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) { + for _, profileID := range profileIDs { + if _, err := v.ClearBit(bitmapID, profileID); err != nil { + panic(err) + } + } +}