scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-12 15:51:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Separate physical data layout with views.

Browse source 
Previously, multiple frames with different prefixes were used to separate
different data layouts. This included separating standard row/column
layouts from inverted column/row layouts as well as storing aggregate
information for timestamp data.

Unfortunately, this caused frame meta data to be copied between multiple
frames and it made it difficult to keep these frames in sync.

This commit separates these different physical layouts into `Views`.
A `Frame` now has one or more views which represent each layout.
Fragments have been moved from under the `Frame` to be contained
within the `View`.

There are two primary views:

- `standard`
- `inverse`

If a frame has a time quantum, then views are generated for these
each of the standard/inverse views. For example a time quantum
of `YMDH` for the date `2000-01-02T00:00:00Z` would create the
following views:

- `standard_2000`
- `inverse_2000`
- `standard_200001`
- `inverse_200001`
- `standard_20000102`
- `inverse_20000102`

From the user's perspective, nothing should change in PQL. Different
PQL statements will handle the appropriate view automatically. For
example, `Bitmap()` and `Profile()` will fetch using the `standard`
view or the `inverse` view, respectively. The `Range()` statement
will lookup the appropriate time-based views automatically.
This commit is contained in:
Ben Johnson 2017-03-20 15:30:12 -06:00
parent 74b5f99e15
commit eff08af112
24 changed files with 1213 additions and 566 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
client.go
View file

@ -430,7 +430,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string
}
// BackupTo backs up an entire frame from a cluster to w.
func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) error {
func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error {
if db == "" {
return ErrDatabaseRequired
} else if frame == "" {
@ -448,7 +448,7 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[db]; i++ {
if err := c.backupSliceTo(ctx, tw, db, frame, i); err != nil {
if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil {
return err
}
}
@ -462,9 +462,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er
}
// backupSliceTo backs up a single slice to tw.
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame string, slice uint64) error {
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error {
// Return error if unable to backup from any slice.
r, err := c.BackupSlice(ctx, db, frame, slice)
r, err := c.BackupSlice(ctx, db, frame, view, slice)
if err != nil {
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
} else if r == nil {
@ -500,7 +500,7 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame st
// BackupSlice retrieves a streaming backup from a single slice.
// This function tries slice owners until one succeeds.
func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64) (io.ReadCloser, error) {
func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
if err != nil {
@ -509,7 +509,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64
// Try to backup slice from each one until successful.
for _, i := range rand.Perm(len(nodes)) {
r, err := c.backupSliceNode(ctx, db, frame, slice, nodes[i])
r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i])
if err == nil {
return r, nil // successfully attached
} else if err == ErrFragmentNotFound {
@ -523,7 +523,7 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame string, slice uint64
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice uint64, node *Node) (io.ReadCloser, error) {
func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
u := url.URL{
Scheme: "http",
Host: node.Host,
@ -531,6 +531,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui
RawQuery: url.Values{
"db": {db},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode(),
}
@ -560,7 +561,7 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame string, slice ui
}
// RestoreFrom restores a frame from a backup file to an entire cluster.
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string) error {
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error {
if db == "" {
return ErrDatabaseRequired
} else if frame == "" {
@ -592,14 +593,14 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame string)
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, slice); err != nil {
if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil {
return err
}
}
}
// restoreSliceFrom restores a single slice to all owning nodes.
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame string, slice uint64) error {
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
if err != nil {
@ -615,6 +616,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str
RawQuery: url.Values{
"db": {db},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode(),
}
@ -726,9 +728,52 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error
return nil
}
// FrameViews returns a list of view names for a frame.
func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) {
// Create URL & HTTP request.
u := url.URL{
Scheme: "http",
Host: c.host,
Path: "/frame/views",
RawQuery: (&url.Values{
"db": {db},
"frame": {frame},
}).Encode(),
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Handle response based on status code.
switch resp.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, ErrFrameNotFound
default:
body, _ := ioutil.ReadAll(resp.Body)
return nil, errors.New(string(body))
}
// Decode response.
var rsp getFrameViewsResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
}
return rsp.Views, nil
}
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uint64) ([]FragmentBlock, error) {
func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
@ -736,6 +781,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin
RawQuery: url.Values{
"db": {db},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode(),
}
@ -771,10 +817,11 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin
}
// BlockData returns bitmap/profile id pairs for a block.
func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) {
func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
DB: db,
Frame: frame,
View: view,
Slice: slice,
Block: uint64(block),
})

74
client_test.go
View file

@ -61,33 +61,33 @@ func TestClient_MultiNode(t *testing.T) {
}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(99, 1, 2, 3, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(1, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).MustSetBits(22, 1, 2, 3, 4, 5)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4)
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(100, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(21, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(100, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
idx[0].MustCreateFragmentIfNotExists("d", "f.n", 0).RecalculateCache()
idx[1].MustCreateFragmentIfNotExists("d", "f.n", 10).RecalculateCache()
idx[2].MustCreateFragmentIfNotExists("d", "f.n", 6).RecalculateCache()
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache()
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache()
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
@ -161,7 +161,7 @@ func TestClient_Import(t *testing.T) {
defer idx.Close()
// Load bitmap into cache to ensure cache gets updated.
f := idx.MustCreateFragmentIfNotExists("d", "f", 0)
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f.Bitmap(0)
s := NewServer()
@ -195,10 +195,10 @@ func TestClient_BackupRestore(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(100, SliceWidth, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 5).MustSetBits(100, (5*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(200, 20000)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
s := NewServer()
defer s.Close()
@ -211,7 +211,7 @@ func TestClient_BackupRestore(t *testing.T) {
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "d", "f"); err != nil {
if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
@ -219,21 +219,21 @@ func TestClient_BackupRestore(t *testing.T) {
if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil {
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Verify data.
if a := idx.Fragment("x", "y", 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
@ -244,11 +244,11 @@ func TestClient_FragmentBlocks(t *testing.T) {
defer idx.Close()
// Set two bits on blocks 0 & 3.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
s := NewServer()
defer s.Close()
@ -259,7 +259,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
// Retrieve blocks.
c := MustNewClient(s.Host())
blocks, err := c.FragmentBlocks(context.Background(), "d", "f", 0)
blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0)
if err != nil {
t.Fatal(err)
} else if len(blocks) != 2 {
@ -271,7 +271,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
}
// Verify data matches local blocks.
if a := idx.Fragment("d", "f", 0).Blocks(); !reflect.DeepEqual(a, blocks) {
if a := idx.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
}
}

3
cmd/backup.go
View file

@ -18,7 +18,7 @@ func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Use: "backup",
Short: "Backup data from pilosa.",
Long: `
Backs up the database and frame from across the cluster into a single file.
Backs up the view from across the cluster into a single file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Backuper.Run(context.Background()); err != nil {
@ -31,6 +31,7 @@ Backs up the database and frame from across the cluster into a single file.
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.")
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
return backupCmd

3
cmd/restore.go
View file

@ -19,7 +19,7 @@ func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
Use: "restore",
Short: "Restore data to pilosa from a backup file.",
Long: `
Restores a frame to the cluster from a backup file.
Restores a view to the cluster from a backup file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Restorer.Run(context.Background()); err != nil {
@ -32,6 +32,7 @@ Restores a frame to the cluster from a backup file.
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.")
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.")
return restoreCmd

7
ctl/backup.go
View file

@ -9,14 +9,15 @@ import (
"github.com/pilosa/pilosa"
)
// BackupCommand represents a command for backing up a frame.
// BackupCommand represents a command for backing up a view.
type BackupCommand struct {
// Destination host and port.
Host string
// Name of the database & frame to backup.
// Name of the database, frame, view to backup.
Database string
Frame string
View string
// Output file to write to.
Path string
@ -53,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error {
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame); err != nil {
if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
return err
}

3
ctl/restore.go
View file

@ -17,6 +17,7 @@ type RestoreCommand struct {
// Name of the database & frame to backup.
Database string
Frame string
View string
// Import file to read from.
Path string
@ -53,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame); err != nil {
if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
return err
}

87
db.go
View file

@ -386,6 +386,7 @@ func (db *DB) DeleteFrame(name string) error {
return nil
}
/*
// SetBit sets a bit for a given profile & bitmap.
// If a timestamp is specified then set all bits for the different quantum units.
func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) {
@ -411,7 +412,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
// If a timestamp is specified then set bits across all frames for the quantum.
opt := f.Options()
for _, subname := range FramesByTime(name, *t, q) {
for _, subname := range ViewsByTime(name, *t, q) {
f, err := db.CreateFrameIfNotExists(subname, opt)
if err != nil {
return changed, err
@ -425,67 +426,7 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
}
return changed, nil
}
// Import bulk imports data.
func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
// Read frame.
f := db.Frame(name)
if f == nil {
return ErrFrameNotFound
}
// Determine quantum if timestamps are set.
var q TimeQuantum
if hasTime(timestamps) {
if q = f.TimeQuantum(); q == "" {
q = db.TimeQuantum()
if err := f.SetTimeQuantum(q); err != nil {
return err
}
}
if q == "" {
return errors.New("time quantum not set in either database or frame")
}
}
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range bitmapIDs {
bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i]
slice := profileID / SliceWidth
var names []string
if timestamp == nil {
names = []string{name}
} else {
names = FramesByTime(name, *timestamp, q)
}
// Attach bit to each frame.
for _, name := range names {
key := importKey{Frame: name, Slice: slice}
data := dataByFragment[key]
data.BitmapIDs = append(data.BitmapIDs, bitmapID)
data.ProfileIDs = append(data.ProfileIDs, profileID)
dataByFragment[key] = data
}
}
// Import into each fragment.
for key, data := range dataByFragment {
frag, err := f.CreateFragmentIfNotExists(key.Slice)
if err != nil {
return err
}
if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
return err
}
}
return nil
}
*/
type dbSlice []*DB
@ -508,14 +449,19 @@ func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// MergeSchemas combines databases and frames from a and b into one schema.
func MergeSchemas(a, b []*DBInfo) []*DBInfo {
// Generate a map from both schemas.
m := make(map[string]map[string]struct{})
m := make(map[string]map[string]map[string]struct{})
for _, dbs := range [][]*DBInfo{a, b} {
for _, db := range dbs {
if m[db.Name] == nil {
m[db.Name] = make(map[string]struct{})
m[db.Name] = make(map[string]map[string]struct{})
}
for _, frame := range db.Frames {
m[db.Name][frame.Name] = struct{}{}
if m[db.Name][frame.Name] == nil {
m[db.Name][frame.Name] = make(map[string]struct{})
}
for _, view := range frame.Views {
m[db.Name][frame.Name][view.Name] = struct{}{}
}
}
}
}
@ -524,8 +470,13 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo {
dbs := make([]*DBInfo, 0, len(m))
for db, frames := range m {
di := &DBInfo{Name: db}
for frame := range frames {
di.Frames = append(di.Frames, &FrameInfo{Name: frame})
for frame, views := range frames {
fi := &FrameInfo{Name: frame}
for view := range views {
fi.Views = append(fi.Views, &ViewInfo{Name: view})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
dbs = append(dbs, di)
@ -557,7 +508,7 @@ func hasTime(a []*time.Time) bool {
}
type importKey struct {
Frame string
View string
Slice uint64
}

24
db_test.go
View file

@ -4,7 +4,6 @@ import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/pilosa/pilosa"
)
@ -26,11 +25,11 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) {
other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f != other {
} else if f.Frame != other.Frame {
t.Fatal("frame mismatch")
}
if f != db.Frame("f") {
if f.Frame != db.Frame("f") {
t.Fatal("frame mismatch")
}
}
@ -130,13 +129,22 @@ func (db *DB) Reopen() error {
return nil
}
// MustSetBit sets a bit on the database. Panic on error.
func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) {
changed, err := db.SetBit(name, bitmapID, profileID, t)
// CreateFrame creates a frame with the given options.
func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := db.DB.CreateFrame(name, opt)
if err != nil {
panic(err)
return nil, err
}
return changed
return &Frame{Frame: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := db.DB.CreateFrameIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// Ensure database can delete a frame.

22
executor.go
View file

@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
frame = DefaultFrame
}
f := e.Index.Fragment(db, frame, slice)
f := e.Index.Fragment(db, frame, ViewStandard, slice)
if f == nil {
return nil, nil
}
@ -346,7 +346,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel)
}
frag := e.Index.Fragment(db, frame, slice)
frag := e.Index.Fragment(db, frame, ViewStandard, slice)
if frag == nil {
return NewBitmap(), nil
}
@ -421,8 +421,8 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
// Union bitmaps across all time-based subframes.
bm := &Bitmap{}
for _, subframe := range FramesByTimeRange(frame, startTime, endTime, q) {
f := e.Index.Fragment(db, subframe, slice)
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Index.Fragment(db, frame, view, slice)
if f == nil {
continue
}
@ -525,12 +525,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
for _, node := range e.Cluster.FragmentNodes(db, slice) {
// Update locally if host matches.
if node.Host == e.Host {
frag := e.Index.Fragment(db, frame, slice)
if frag == nil {
return false, nil
}
val, err := frag.ClearBit(rowID, colID)
val, err := f.ClearBit(rowID, colID, nil)
if err != nil {
return false, err
} else if val {
@ -601,12 +596,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
for _, node := range e.Cluster.FragmentNodes(db, slice) {
// Update locally if host matches.
if node.Host == e.Host {
d := e.Index.DB(db)
if d == nil {
return false, ErrDatabaseNotFound
}
val, err := d.SetBit(frame, rowID, colID, timestamp)
val, err := f.SetBit(rowID, colID, timestamp)
if err != nil {
return false, err
} else if val {

190
executor_test.go
View file

@ -15,8 +15,8 @@ import (
func TestExecutor_Execute_Bitmap(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
@ -36,11 +36,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
func TestExecutor_Execute_Difference(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 4)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
@ -54,7 +54,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil {
@ -66,13 +66,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
func TestExecutor_Execute_Intersect(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 1)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
@ -97,12 +97,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
func TestExecutor_Execute_Union(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
@ -116,7 +116,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
func TestExecutor_Execute_Empty_Union(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(10, 0)
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil {
@ -130,9 +130,9 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
func TestExecutor_Execute_Count(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
@ -148,7 +148,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
defer idx.Close()
e := NewExecutor(idx.Index, NewCluster(1))
f := idx.MustCreateFragmentIfNotExists("d", "f", 0)
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
if n := f.Bitmap(11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
@ -216,15 +216,15 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
@ -241,13 +241,13 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0 & 1 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth)
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
@ -265,23 +265,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(0, 2*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(0, 3*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 4).SetBit(0, 4*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(1, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(2, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 2).SetBit(3, 2*SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 3).SetBit(4, 3*SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
@ -300,19 +300,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
// Create an intersecting bitmap.
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
@ -332,9 +332,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
@ -355,9 +355,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
//
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
@ -380,24 +380,26 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Create database.
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
db.SetTimeQuantum(pilosa.TimeQuantum("YMDH"))
// Create frame.
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
}
// Set bits.
db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00"))
db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00"))
db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00"))
db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00"))
db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00"))
db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00"))
f.MustSetBit(1, 2, MustParseTimePtr("1999-12-31 00:00"))
f.MustSetBit(1, 3, MustParseTimePtr("2000-01-01 00:00"))
f.MustSetBit(1, 4, MustParseTimePtr("2000-01-02 00:00"))
f.MustSetBit(1, 5, MustParseTimePtr("2000-02-01 00:00"))
f.MustSetBit(1, 6, MustParseTimePtr("2001-01-01 00:00"))
f.MustSetBit(1, 7, MustParseTimePtr("2002-01-01 02:00"))
db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early
db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late
db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap
f.MustSetBit(1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early
f.MustSetBit(1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late
f.MustSetBit(10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
@ -439,7 +441,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// The local node owns slice 1.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
@ -466,8 +468,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Create local executor data. The local node owns slice 1.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
@ -514,7 +516,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
// Verify that one bit is set on both node's index.
if n := idx.MustCreateFragmentIfNotExists("d", "f", 0).Bitmap(10).Count(); n != 1 {
if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -547,17 +549,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create local executor data.
idx := MustOpenIndex()
defer idx.Close()
idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
oldQuantum := idx.DB("d").TimeQuantum()
defer func() {
// restore db quantum
idx.DB("d").SetTimeQuantum(oldQuantum)
}()
// need to set the quantum otherwise SetBit fails silently
idx.DB("d").SetTimeQuantum("Y")
// Create frame.
if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := f.SetTimeQuantum("Y"); err != nil {
t.Fatal(err)
}
@ -567,7 +563,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
// Verify that one bit is set on both node's index.
if n := idx.MustCreateFragmentIfNotExists("d", "f_2016", 0).Bitmap(10).Count(); n != 1 {
if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -620,8 +616,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create local executor data on slice 1 & 3.
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(30, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {

19
fragment.go
View file

@ -58,6 +58,7 @@ type Fragment struct {
// Composite identifiers
db string
frame string
view string
slice uint64
// File-backed storage
@ -92,11 +93,12 @@ type Fragment struct {
}
// NewFragment returns a new instance of Fragment.
func NewFragment(path, db, frame string, slice uint64) *Fragment {
func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
return &Fragment{
path: path,
db: db,
frame: frame,
view: view,
slice: slice,
LogOutput: ioutil.Discard,
@ -118,6 +120,9 @@ func (f *Fragment) DB() string { return f.db }
// Frame returns the frame the fragment was initialized with.
func (f *Fragment) Frame() string { return f.frame }
// View returns the view the fragment was initialized with.
func (f *Fragment) View() string { return f.view }
// Slice returns the slice the fragment was initialized with.
func (f *Fragment) Slice() uint64 { return f.slice }
@ -362,7 +367,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error)
}
// Write to storage.
if changed, err = f.storage.Add(pos); err != nil {
return false, err
}
@ -995,8 +999,8 @@ func track(start time.Time, name string, logger *log.Logger) {
func (f *Fragment) snapshot() error {
logger := f.logger()
logger.Printf("fragment: snapshotting %s/%s/%d", f.db, f.frame, f.slice)
defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%d", f.db, f.frame, f.slice), logger)
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice)
defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
@ -1322,7 +1326,7 @@ func (s *FragmentSyncer) SyncFragment() error {
if err != nil {
return err
}
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.Slice())
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return err
}
@ -1403,7 +1407,8 @@ func (s *FragmentSyncer) syncBlock(id int) error {
}
clients = append(clients, client)
bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), f.Slice(), id)
// Only sync the standard block.
bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id)
if err != nil {
return err
}
@ -1436,6 +1441,8 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Generate query with sets & clears.
var buf bytes.Buffer
// Only sync the standard block.
for j := 0; j < len(set.ProfileIDs); j++ {
fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j])
}

48
fragment_test.go
View file

@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth
// Ensure a fragment can set a bit and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -54,7 +54,7 @@ func TestFragment_SetBit(t *testing.T) {
// Ensure a fragment can clear a set bit.
func TestFragment_ClearBit(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set and then clear bits on the fragment.
@ -81,7 +81,7 @@ func TestFragment_ClearBit(t *testing.T) {
// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set and then clear bits on the fragment.
@ -110,7 +110,7 @@ func TestFragment_Snapshot(t *testing.T) {
// Ensure a fragment can iterate over all bits in order.
func TestFragment_ForEachBit(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -139,7 +139,7 @@ func TestFragment_ForEachBit(t *testing.T) {
// Ensure a fragment can return the top n results.
func TestFragment_Top(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
@ -161,7 +161,7 @@ func TestFragment_Top(t *testing.T) {
// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps.
func TestFragment_Top_Filter(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
@ -191,7 +191,7 @@ func TestFragment_Top_Filter(t *testing.T) {
// Ensure a fragment can return top bitmaps that intersect with an input bitmap.
func TestFragment_TopN_Intersect(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input bitmap.
@ -221,7 +221,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
t.Skip("short mode")
}
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input bitmap.
@ -258,7 +258,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
// Ensure a fragment can return top bitmaps when specified by ID.
func TestFragment_TopN_BitmapIDs(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on various bitmaps.
@ -279,7 +279,7 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) {
// Ensure fragment can return a checksum for its blocks.
func TestFragment_Checksum(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Retrieve checksum and set bits.
@ -298,7 +298,7 @@ func TestFragment_Checksum(t *testing.T) {
// Ensure fragment can return a checksum for a given block.
func TestFragment_Blocks(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Retrieve initial checksum.
@ -336,7 +336,7 @@ func TestFragment_Blocks(t *testing.T) {
// Ensure fragment returns an empty checksum if no data exists for a block.
func TestFragment_Blocks_Empty(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on a different block.
@ -354,7 +354,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_LRUCache_Persistence(t *testing.T) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -386,7 +386,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_RankCache_Persistence(t *testing.T) {
f := MustOpenFragment("d", "f.n", 0)
f := MustOpenFragment("d", "f.n", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -418,7 +418,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := MustOpenFragment("d", "f", 0)
f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f0.Close()
// Set and then clear bits on the fragment.
@ -443,7 +443,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Read into another fragment.
f1 := MustOpenFragment("d", "f", 0)
f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
if rn, err := f1.ReadFrom(&buf); err != nil {
t.Fatal(err)
} else if wn != rn {
@ -476,7 +476,7 @@ func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlo
func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) }
func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Fill fragment.
@ -505,7 +505,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
// Open the fragment specified by the path.
f := pilosa.NewFragment(*FragmentPath, "d", "f", 0)
f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@ -521,7 +521,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
func BenchmarkFragment_IntersectionCount(b *testing.B) {
f := MustOpenFragment("d", "f", 0)
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
f.MaxOpN = math.MaxInt32
@ -558,7 +558,7 @@ type Fragment struct {
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(db, frame string, slice uint64) *Fragment {
func NewFragment(db, frame, view string, slice uint64) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
@ -566,7 +566,7 @@ func NewFragment(db, frame string, slice uint64) *Fragment {
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), db, frame, slice),
Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice),
BitmapAttrStore: MustOpenAttrStore(),
}
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
@ -574,8 +574,8 @@ func NewFragment(db, frame string, slice uint64) *Fragment {
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(db, frame string, slice uint64) *Fragment {
f := NewFragment(db, frame, slice)
func MustOpenFragment(db, frame, view string, slice uint64) *Fragment {
f := NewFragment(db, frame, view, slice)
if err := f.Open(); err != nil {
panic(err)
}
@ -597,7 +597,7 @@ func (f *Fragment) Reopen() error {
return err
}
f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.Slice())
f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice())
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
if err := f.Open(); err != nil {
return err

293
frame.go
View file

@ -1,13 +1,14 @@
package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
@ -23,7 +24,7 @@ const (
DefaultRowLabel = "id"
)
// Frame represents a container for fragments.
// Frame represents a container for views.
type Frame struct {
mu sync.Mutex
path string
@ -31,8 +32,7 @@ type Frame struct {
name string
timeQuantum TimeQuantum
// Fragments by slice.
fragments map[uint64]*Fragment
views map[string]*View
// Bitmap attribute storage and cache
bitmapAttrStore *AttrStore
@ -57,7 +57,7 @@ func NewFrame(path, db, name string) (*Frame, error) {
db: db,
name: name,
fragments: make(map[uint64]*Fragment),
views: make(map[string]*View),
bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")),
stats: NopStatsClient,
@ -86,8 +86,8 @@ func (f *Frame) MaxSlice() uint64 {
defer f.mu.Unlock()
var max uint64
for slice := range f.fragments {
if slice > max {
for _, view := range f.views {
if slice := view.MaxSlice(); slice > max {
max = slice
}
}
@ -143,7 +143,7 @@ func (f *Frame) Open() error {
return err
}
if err := f.openFragments(); err != nil {
if err := f.openViews(); err != nil {
return err
}
@ -160,10 +160,12 @@ func (f *Frame) Open() error {
return nil
}
// openFragments opens and initializes the fragments inside the frame.
func (f *Frame) openFragments() error {
file, err := os.Open(f.path)
if err != nil {
// openViews opens and initializes the views inside the frame.
func (f *Frame) openViews() error {
file, err := os.Open(filepath.Join(f.path, "views"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return err
}
defer file.Close()
@ -174,22 +176,17 @@ func (f *Frame) openFragments() error {
}
for _, fi := range fis {
if fi.IsDir() {
if !fi.IsDir() {
continue
}
// Parse filename into integer.
slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
continue
name := filepath.Base(fi.Name())
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err)
}
frag := f.newFragment(f.FragmentPath(slice), slice)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
}
frag.BitmapAttrStore = f.bitmapAttrStore
f.fragments[frag.Slice()] = frag
view.BitmapAttrStore = f.bitmapAttrStore
f.views[view.Name()] = view
f.stats.Count("maxSlice", 1)
}
@ -241,7 +238,7 @@ func (f *Frame) saveMeta() error {
return nil
}
// Close closes the frame and its fragments.
// Close closes the frame and its views.
func (f *Frame) Close() error {
f.mu.Lock()
defer f.mu.Unlock()
@ -251,11 +248,11 @@ func (f *Frame) Close() error {
_ = f.bitmapAttrStore.Close()
}
// Close all fragments.
for _, frag := range f.fragments {
_ = frag.Close()
// Close all views.
for _, view := range f.views {
_ = view.Close()
}
f.fragments = make(map[uint64]*Fragment)
f.views = make(map[string]*View)
return nil
}
@ -288,75 +285,232 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error {
return nil
}
// FragmentPath returns the path to a fragment in the frame.
func (f *Frame) FragmentPath(slice uint64) string {
return filepath.Join(f.path, strconv.FormatUint(slice, 10))
// ViewPath returns the path to a view in the frame.
func (f *Frame) ViewPath(name string) string {
return filepath.Join(f.path, "views", name)
}
// Fragment returns a fragment in the frame by slice.
func (f *Frame) Fragment(slice uint64) *Fragment {
// View returns a view in the frame by name.
func (f *Frame) View(name string) *View {
f.mu.Lock()
defer f.mu.Unlock()
return f.fragment(slice)
return f.view(name)
}
func (f *Frame) fragment(slice uint64) *Fragment { return f.fragments[slice] }
func (f *Frame) view(name string) *View { return f.views[name] }
// Fragments returns a list of all fragments in the frame.
func (f *Frame) Fragments() []*Fragment {
// Views returns a list of all views in the frame.
func (f *Frame) Views() []*View {
f.mu.Lock()
defer f.mu.Unlock()
other := make([]*Fragment, 0, len(f.fragments))
for _, fragment := range f.fragments {
other = append(other, fragment)
other := make([]*View, 0, len(f.views))
for _, view := range f.views {
other = append(other, view)
}
return other
}
// CreateFragmentIfNotExists returns a fragment in the frame by slice.
func (f *Frame) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.createFragmentIfNotExists(slice)
}
func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Find fragment in cache first.
if frag := f.fragments[slice]; frag != nil {
return frag, nil
if view := f.views[name]; view != nil {
return view, nil
}
// Initialize and open fragment.
frag := f.newFragment(f.FragmentPath(slice), slice)
if err := frag.Open(); err != nil {
view := f.newView(f.ViewPath(name), name)
if err := view.Open(); err != nil {
return nil, err
}
frag.BitmapAttrStore = f.bitmapAttrStore
view.BitmapAttrStore = f.bitmapAttrStore
f.views[view.Name()] = view
// Save to lookup.
f.fragments[slice] = frag
f.stats.Count("maxSlice", 1)
return frag, nil
return view, nil
}
func (f *Frame) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, f.db, f.name, slice)
frag.LogOutput = f.LogOutput
frag.stats = f.stats.WithTags(fmt.Sprintf("slice:%d", slice))
return frag
func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.db, f.name, name)
view.LogOutput = f.LogOutput
view.BitmapAttrStore = f.bitmapAttrStore
view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name))
return view
}
// SetBit sets a bit within the frame.
func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
slice := profileID / SliceWidth
frag, err := f.CreateFragmentIfNotExists(slice)
func (f *Frame) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Set standard layout bits.
if v, err := f.setBit(ViewStandard, rowID, colID, t); err != nil {
return changed, err
} else if v {
changed = v
}
// Set inverse layout bits.
// NOTE: The row & col are transposed for the inverted view.
if v, err := f.setBit(ViewInverse, colID, rowID, t); err != nil {
return changed, err
} else if v {
changed = v
}
return changed, nil
}
// setBit sets a bit for a given layout (default or inverted).
func (f *Frame) setBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Retrieve view. Exit if it doesn't exist.
view, err := f.CreateViewIfNotExists(name)
if err != nil {
return changed, err
}
return frag.SetBit(bitmapID, profileID)
// Set non-time bit.
if v, err := view.SetBit(rowID, colID); err != nil {
return changed, err
} else if v {
changed = v
}
// Exit early if no timestamp is specified.
if t == nil {
return changed, nil
}
// If a timestamp is specified then set bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, err
}
if c, err := view.SetBit(rowID, colID); err != nil {
return changed, err
} else if c {
changed = true
}
}
return changed, nil
}
// ClearBit clears a bit within the frame.
func (f *Frame) ClearBit(rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Clear standard layout bits.
if v, err := f.clearBit(ViewStandard, rowID, colID, t); err != nil {
return changed, err
} else if v {
changed = v
}
// Clear inverse layout bits.
// NOTE: The row & col are transposed for the inverted view.
if v, err := f.clearBit(ViewInverse, colID, rowID, t); err != nil {
return changed, err
} else if v {
changed = v
}
return changed, nil
}
// clearBit clears a bit for a given layout (default or inverted).
func (f *Frame) clearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Retrieve view. Exit if it doesn't exist.
view, err := f.CreateViewIfNotExists(name)
if err != nil {
return changed, err
}
// Clear non-time bit.
if v, err := view.ClearBit(rowID, colID); err != nil {
return changed, err
} else if v {
changed = v
}
// Exit early if no timestamp is specified.
if t == nil {
return changed, nil
}
// If a timestamp is specified then clear bits across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
return changed, err
}
if c, err := view.ClearBit(rowID, colID); err != nil {
return changed, err
} else if c {
changed = true
}
}
return changed, nil
}
// Import bulk imports data.
func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
// Determine quantum if timestamps are set.
q := f.TimeQuantum()
if hasTime(timestamps) && q == "" {
return errors.New("time quantum not set in either database or frame")
}
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range bitmapIDs {
bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i]
slice := profileID / SliceWidth
var standard, inverse []string
if timestamp == nil {
standard = []string{ViewStandard}
inverse = []string{ViewInverse}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
inverse = ViewsByTime(ViewInverse, *timestamp, q)
}
// Attach bit to each standard view.
for _, name := range standard {
key := importKey{View: name, Slice: slice}
data := dataByFragment[key]
data.BitmapIDs = append(data.BitmapIDs, bitmapID)
data.ProfileIDs = append(data.ProfileIDs, profileID)
dataByFragment[key] = data
}
// Attach reversed bits to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: slice}
data := dataByFragment[key]
data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed
data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed
dataByFragment[key] = data
}
}
// Import into each fragment.
for key, data := range dataByFragment {
view, err := f.CreateViewIfNotExists(key.View)
if err != nil {
return err
}
frag, err := view.CreateFragmentIfNotExists(key.Slice)
if err != nil {
return err
}
if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
return err
}
}
return nil
}
type frameSlice []*Frame
@ -367,7 +521,8 @@ func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// FrameInfo represents schema information for a frame.
type FrameInfo struct {
Name string `json:"name"`
Name string `json:"name"`
Views []*ViewInfo `json:"views,omitempty"`
}
type frameInfoSlice []*FrameInfo

47
frame_test.go
View file

@ -4,33 +4,34 @@ import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/pilosa/pilosa"
)
// Ensure frame can open and retrieve a fragment.
func TestFrame_CreateFragmentIfNotExists(t *testing.T) {
// Ensure frame can open and retrieve a view.
func TestFrame_CreateViewIfNotExists(t *testing.T) {
f := MustOpenFrame()
defer f.Close()
// Create fragment.
frag, err := f.CreateFragmentIfNotExists(100)
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if frag == nil {
t.Fatal("expected fragment")
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing fragment.
frag2, err := f.CreateFragmentIfNotExists(100)
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if frag != frag2 {
t.Fatal("fragment mismatch")
} else if view != view2 {
t.Fatal("view mismatch")
}
if frag != f.Fragment(100) {
t.Fatal("fragment mismatch")
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
@ -54,6 +55,17 @@ func TestFrame_SetTimeQuantum(t *testing.T) {
}
}
func TestFrame_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "d", ".meta")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
}
// Frame represents a test wrapper for pilosa.Frame.
type Frame struct {
*pilosa.Frame
@ -106,14 +118,11 @@ func (f *Frame) Reopen() error {
return nil
}
// NewFrame does not return a frame when name is invalid.
func TestFrame_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-frame-")
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(bitmapID, profileID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(bitmapID, profileID, t)
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "d", ".meta")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
return changed
}

152
handler.go
View file

@ -145,6 +145,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
case "/frame/views":
switch r.Method {
case "GET":
h.handleGetFrameViews(w, r)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
case "/frame/attr/diff":
switch r.Method {
case "POST":
@ -583,6 +590,35 @@ type patchFrameTimeQuantumRequest struct {
type patchFrameTimeQuantumResponse struct{}
// handleGetFrameViews handles GET /frame/views request.
func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
db, frame := q.Get("db"), q.Get("frame")
// Retrieve views.
f := h.Index.Frame(db, frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Fetch views.
views := f.Views()
names := make([]string, len(views))
for i := range views {
names[i] = views[i].Name()
}
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameViewsResponse{Views: names}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type getFrameViewsResponse struct {
Views []string `json:"views,omitempty"`
}
// handlePostFrameAttrDiff handles POST /frame/attr/diff requests.
func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) {
// Decode request.
@ -791,7 +827,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
// Find the correct fragment.
// Find the DB.
h.logger().Println("importing:", req.DB, req.Frame, req.Slice)
db := h.Index.DB(req.DB)
if db == nil {
@ -800,8 +836,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
return
}
// Retrieve frame.
f := db.Frame(req.Frame)
if f == nil {
h.logger().Printf("frame error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrFrameNotFound.Error())
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Import into fragment.
err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps)
err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps)
if err != nil {
h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err)
return
@ -834,7 +878,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
// Parse query parameters.
q := r.URL.Query()
db, frame := q.Get("db"), q.Get("frame")
db, frame, view := q.Get("db"), q.Get("frame"), q.Get("view")
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
@ -850,7 +894,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
}
// Find the fragment.
f := h.Index.Fragment(db, frame, slice)
f := h.Index.Fragment(db, frame, view, slice)
if f == nil {
return
}
@ -905,7 +949,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
}
// Retrieve fragment from index.
f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice)
f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice)
if f == nil {
http.Error(w, "fragment not found", http.StatusNotFound)
return
@ -934,8 +978,15 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
return
}
// Retrieve view.
view, err := f.CreateViewIfNotExists(q.Get("view"))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Retrieve fragment from frame.
frag, err := f.CreateFragmentIfNotExists(slice)
frag, err := view.CreateFragmentIfNotExists(slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -961,7 +1012,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
}
// Retrieve fragment from index.
f := h.Index.Fragment(req.DB, req.Frame, req.Slice)
f := h.Index.Fragment(req.DB, req.Frame, req.View, req.Slice)
if f == nil {
http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound)
return
@ -997,7 +1048,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
}
// Retrieve fragment from index.
f := h.Index.Fragment(q.Get("db"), q.Get("frame"), slice)
f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice)
if f == nil {
http.Error(w, "fragment not found", http.StatusNotFound)
return
@ -1050,6 +1101,24 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
return
}
// Retrieve frame.
f := h.Index.Frame(db, frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Retrieve list of all views.
views, err := client.FrameViews(r.Context(), db, frame)
if err != nil {
http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError)
return
}
for _, view := range views {
println("dbg/views", view)
}
// Loop over each slice and import it if this node owns it.
//travis
for slice := uint64(0); slice <= maxSlices[db]; slice++ {
@ -1058,39 +1127,42 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
continue
}
// Retrieve frame.
f := h.Index.Frame(db, frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Otherwise retrieve the local fragment.
frag, err := f.CreateFragmentIfNotExists(slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Stream backup from remote node.
rd, err := client.BackupSlice(r.Context(), db, frame, slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
} else if rd == nil {
continue // slice doesn't exist
}
// Restore to local frame and always close reader.
if err := func() error {
defer rd.Close()
if _, err := frag.ReadFrom(rd); err != nil {
return err
// Loop over view names.
for _, view := range views {
// Create view.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Otherwise retrieve the local fragment.
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Stream backup from remote node.
rd, err := client.BackupSlice(r.Context(), db, frame, view, slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
} else if rd == nil {
continue // slice doesn't exist
}
// Restore to local frame and always close reader.
if err := func() error {
defer rd.Close()
if _, err := frag.ReadFrom(rd); err != nil {
return err
}
return nil
}(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
return nil
}(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}

32
handler_test.go
View file

@ -37,10 +37,14 @@ func TestHandler_Schema(t *testing.T) {
d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{})
if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
@ -53,7 +57,7 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1"}]},{"name":"d1","frames":[{"name":"f0"}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"inverse"},{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -63,13 +67,13 @@ func TestHandler_MaxSlices(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d0", "f0", 3).MustSetBits(30, (3*SliceWidth)+4)
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+8)
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
h := NewHandler()
h.Index = idx.Index
@ -654,11 +658,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
defer s.Close()
// Set bits in the index.
f0 := idx.MustCreateFragmentIfNotExists("d", "f", 0)
f0 := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f0.MustSetBits(100, 1, 2, 3)
// Begin backing up from slice d/f/0.
resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&slice=0")
resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0")
if err != nil {
t.Fatal(err)
}
@ -675,7 +679,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
}
// Restore backup to slice x/y/0.
if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil {
if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
@ -685,9 +689,9 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
}
// Verify data is correctly restored.
f1 := idx.Fragment("x", "y", 0)
f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/0 not created")
t.Fatal("fragment x/y/standard/0 not created")
} else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored bits: %+v", bits)
}

79
index.go
View file

@ -131,12 +131,17 @@ func (i *Index) Schema() []*DBInfo {
for _, db := range i.DBs() {
di := &DBInfo{Name: db.Name()}
for _, frame := range db.Frames() {
di.Frames = append(di.Frames, &FrameInfo{
Name: frame.Name(),
})
fi := &FrameInfo{Name: frame.Name()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
a = append(a, di)
}
sort.Sort(dbInfoSlice(a))
return a
}
@ -270,13 +275,22 @@ func (i *Index) Frame(db, name string) *Frame {
return d.Frame(name)
}
// Fragment returns the fragment for a database, frame & slice.
func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
// View returns the view for a database, frame, and name.
func (i *Index) View(db, frame, name string) *View {
f := i.Frame(db, frame)
if f == nil {
return nil
}
return f.Fragment(slice)
return f.View(name)
}
// Fragment returns the fragment for a database, frame & slice.
func (i *Index) Fragment(db, frame, view string, slice uint64) *Fragment {
v := i.View(db, frame, view)
if v == nil {
return nil
}
return v.Fragment(slice)
}
// monitorCacheFlush periodically flushes all fragment caches sequentially.
@ -298,15 +312,17 @@ func (i *Index) monitorCacheFlush() {
func (i *Index) flushCaches() {
for _, db := range i.DBs() {
for _, frame := range db.Frames() {
for _, fragment := range frame.Fragments() {
select {
case <-i.closing:
return
default:
}
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
select {
case <-i.closing:
return
default:
}
if err := fragment.FlushCache(); err != nil {
i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
if err := fragment.FlushCache(); err != nil {
i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
}
}
}
}
@ -362,20 +378,27 @@ func (s *IndexSyncer) SyncIndex() error {
return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err)
}
for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) {
continue
}
for _, vi := range fi.Views {
// Verify syncer has not closed.
if s.IsClosing() {
return nil
}
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) {
continue
}
// Verify syncer has not closed.
if s.IsClosing() {
return nil
}
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
}
}
}
}
@ -477,15 +500,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error {
func (s *IndexSyncer) syncFragment(db, frame, view string, slice uint64) error {
// Retrieve local frame.
f := s.Index.Frame(db, frame)
if f == nil {
return ErrFrameNotFound
}
// Ensure view exists locally.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
return err
}
// Ensure fragment exists locally.
frag, err := f.CreateFragmentIfNotExists(slice)
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return err
}

30
index_test.go
View file

@ -18,11 +18,11 @@ func TestIndex_DeleteDB(t *testing.T) {
defer idx.Close()
// Write bits to separate databases.
f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0)
f0 := idx.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0)
if _, err := f0.SetBit(100, 200); err != nil {
t.Fatal(err)
}
f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0)
f1 := idx.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0)
if _, err := f1.SetBit(100, 200); err != nil {
t.Fatal(err)
}
@ -80,7 +80,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
}
// Set data on the local index.
f := idx0.MustCreateFragmentIfNotExists("d", "f", 0)
f := idx0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
@ -91,15 +91,15 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
t.Fatal(err)
}
f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1)
f = idx0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
idx0.MustCreateFragmentIfNotExists("y", "z", 0)
idx0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
// Set data on the remote index.
f = idx1.MustCreateFragmentIfNotExists("d", "f", 0)
f = idx1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
@ -108,7 +108,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
t.Fatal(err)
}
f = idx1.MustCreateFragmentIfNotExists("y", "z", 3)
f = idx1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
@ -134,7 +134,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
// Verify data is the same on both nodes.
for i, idx := range []*Index{idx0, idx1} {
f := idx.Fragment("d", "f", 0)
f := idx.Fragment("d", "f", pilosa.ViewStandard, 0)
if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
@ -147,7 +147,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = idx.Fragment("d", "f0", 1)
f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1)
a := f.Bitmap(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
@ -155,7 +155,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
f = idx.Fragment("y", "z", 3)
f = idx.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
@ -212,17 +212,21 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
if err != nil {
panic(err)
}
return &Frame{Frame: f}
return f
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment {
func (i *Index) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment {
d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{})
f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
frag, err := f.CreateFragmentIfNotExists(slice)
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}

189
internal/internal.pb.go
View file

@ -242,8 +242,9 @@ func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInter
type BlockDataRequest struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"`
View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"`
Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"`
Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"`
}
func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} }
@ -907,15 +908,21 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
}
if m.Slice != 0 {
if m.Block != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintInternal(dAtA, i, uint64(m.Block))
}
if m.Slice != 0 {
dAtA[i] = 0x20
i++
i = encodeVarintInternal(dAtA, i, uint64(m.Slice))
}
if m.Block != 0 {
dAtA[i] = 0x20
if len(m.View) > 0 {
dAtA[i] = 0x2a
i++
i = encodeVarintInternal(dAtA, i, uint64(m.Block))
i = encodeVarintInternal(dAtA, i, uint64(len(m.View)))
i += copy(dAtA[i:], m.View)
}
return i, nil
}
@ -1329,11 +1336,15 @@ func (m *BlockDataRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
if m.Block != 0 {
n += 1 + sovInternal(uint64(m.Block))
}
if m.Slice != 0 {
n += 1 + sovInternal(uint64(m.Slice))
}
if m.Block != 0 {
n += 1 + sovInternal(uint64(m.Block))
l = len(m.View)
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
return n
}
@ -3330,25 +3341,6 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
m.Frame = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowInternal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType)
}
@ -3367,6 +3359,54 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowInternal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field View", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowInternal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthInternal
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.View = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipInternal(dAtA[iNdEx:])
@ -3938,51 +3978,52 @@ var (
func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) }
var fileDescriptorInternal = []byte{
// 735 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd4, 0x4a,
0x10, 0xbd, 0x3d, 0xb6, 0xe7, 0x51, 0x93, 0x8c, 0x26, 0xad, 0xb9, 0x57, 0x56, 0x74, 0x35, 0xb2,
0x5a, 0xf7, 0x4a, 0x16, 0x12, 0x89, 0x14, 0x36, 0x08, 0x21, 0x21, 0x3c, 0x33, 0x51, 0x46, 0x90,
0x28, 0xe9, 0x04, 0x76, 0x2c, 0x3a, 0x49, 0x93, 0x58, 0xf1, 0x63, 0xb0, 0xdb, 0xc0, 0x2c, 0x59,
0xf0, 0x0f, 0x08, 0xbe, 0x00, 0xbe, 0x84, 0x25, 0x9f, 0x80, 0xc2, 0x8f, 0xa0, 0x7e, 0xf8, 0x11,
0x10, 0x21, 0x0b, 0x76, 0xae, 0x53, 0x5d, 0xd5, 0x75, 0xfa, 0x54, 0x95, 0x61, 0x10, 0x26, 0x82,
0x67, 0x09, 0x8b, 0x36, 0x16, 0x59, 0x2a, 0x52, 0xdc, 0x2d, 0x6d, 0xb2, 0x03, 0xad, 0x69, 0x80,
0x3d, 0xe8, 0x1f, 0x85, 0x31, 0x3f, 0x28, 0x58, 0x22, 0x8a, 0xd8, 0x45, 0x1e, 0xf2, 0x7b, 0xb4,
0x09, 0xc9, 0x13, 0x93, 0x34, 0x2a, 0xe2, 0xe4, 0x31, 0x3b, 0xe6, 0x91, 0xdb, 0xd2, 0x27, 0x1a,
0x10, 0x99, 0x81, 0xb3, 0x9d, 0xb1, 0x98, 0xdf, 0x20, 0xd9, 0x3a, 0x74, 0x69, 0xfa, 0xaa, 0x99,
0xa9, 0xb2, 0x49, 0x00, 0xed, 0x20, 0x14, 0x31, 0x5b, 0x60, 0x0c, 0x76, 0x10, 0x8a, 0xdc, 0x45,
0x9e, 0xe5, 0xdb, 0x54, 0x7d, 0xe3, 0xff, 0xc0, 0x79, 0x28, 0x44, 0x96, 0xbb, 0x2d, 0xcf, 0xf2,
0xfb, 0x5b, 0x83, 0x8d, 0x8a, 0x98, 0x84, 0xa9, 0x76, 0x92, 0x0d, 0xb0, 0xf7, 0x59, 0x98, 0xe1,
0x21, 0x58, 0x8f, 0xf8, 0x52, 0x55, 0x60, 0x53, 0xf9, 0x89, 0x47, 0xe0, 0x4c, 0xd2, 0x22, 0x11,
0xea, 0x5a, 0x9b, 0x6a, 0x83, 0x3c, 0x03, 0x2b, 0x08, 0x85, 0x2c, 0x4b, 0x5f, 0x3d, 0x9f, 0x9a,
0x98, 0xca, 0xc6, 0xff, 0x42, 0x6f, 0x3f, 0x4b, 0x9f, 0x87, 0x11, 0x9f, 0x4f, 0x4d, 0x70, 0x0d,
0x48, 0xaf, 0xe4, 0x97, 0x0b, 0x16, 0x2f, 0x5c, 0xcb, 0x43, 0xbe, 0x45, 0x6b, 0x80, 0x3c, 0x80,
0x8e, 0x39, 0x8a, 0x07, 0xd0, 0xaa, 0x92, 0xb7, 0xe6, 0xd3, 0x1b, 0xf2, 0xf9, 0x84, 0xc0, 0x96,
0x5f, 0x4d, 0x42, 0x3d, 0x4d, 0x08, 0x83, 0x7d, 0xb4, 0x5c, 0x70, 0x53, 0x92, 0xfa, 0x96, 0x02,
0x1c, 0x8a, 0x2c, 0x4c, 0xce, 0x9e, 0xb2, 0xa8, 0xe0, 0xaa, 0x9e, 0x1e, 0x6d, 0x42, 0xb2, 0xde,
0x27, 0x61, 0x22, 0xb4, 0xdf, 0xd6, 0x6c, 0x2a, 0x40, 0x7a, 0x83, 0x34, 0x8d, 0xb4, 0xd7, 0xf1,
0x90, 0xdf, 0xa5, 0x35, 0x80, 0xc7, 0x00, 0xdb, 0x51, 0xca, 0x4c, 0x70, 0xdb, 0x43, 0x3e, 0xa2,
0x0d, 0x84, 0x6c, 0x42, 0x47, 0xd6, 0xba, 0xcb, 0x16, 0x35, 0x3b, 0x74, 0x1d, 0xbb, 0xf7, 0x08,
0x56, 0x0e, 0x0a, 0x9e, 0x2d, 0x29, 0x7f, 0x51, 0xf0, 0x5c, 0xc8, 0x47, 0x9a, 0x06, 0x86, 0xa4,
0xec, 0xce, 0x11, 0x38, 0xca, 0x6f, 0x7a, 0x45, 0x1b, 0xf8, 0x1f, 0x68, 0x1f, 0x46, 0xe1, 0x09,
0xcf, 0x5d, 0x4b, 0x35, 0x88, 0xb1, 0xa4, 0x8a, 0xe6, 0xb5, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b,
0xbb, 0xd0, 0x29, 0xdb, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4,
0xa5, 0xc6, 0x22, 0x6f, 0x10, 0xac, 0x9a, 0xe2, 0xf2, 0x45, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcb,
0xb2, 0x52, 0x83, 0x59, 0x96, 0xe1, 0x4d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb,
0x26, 0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xbb, 0x51, 0xa2, 0xa5, 0x22, 0xd6, 0xea,
0x08, 0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0x8e, 0x88, 0x2a, 0xa2,
0xbf, 0x35, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x11, 0x5a, 0x01, 0xb4, 0x67, 0x5a, 0x03, 0xed, 0x49,
0x39, 0xe4, 0x58, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xf9, 0x46, 0x93, 0x73, 0x96,
0x9c, 0xf1, 0x53, 0xf3, 0x7c, 0xa5, 0x49, 0x3e, 0x22, 0x58, 0x9d, 0xc7, 0x8b, 0x34, 0x13, 0xd7,
0x28, 0xa5, 0x76, 0x40, 0xa9, 0x94, 0x5e, 0x08, 0x23, 0x70, 0x94, 0x36, 0xaa, 0x13, 0x6d, 0xaa,
0x0d, 0xd5, 0x65, 0x66, 0xba, 0xa4, 0x50, 0x52, 0xc2, 0x1a, 0x90, 0x5d, 0x56, 0x8d, 0x57, 0xee,
0x3a, 0xca, 0xdd, 0x40, 0xa4, 0xbf, 0x1a, 0xb0, 0xdc, 0x6d, 0x7b, 0x96, 0x6f, 0xd1, 0x06, 0x42,
0x08, 0x0c, 0xca, 0x52, 0x7f, 0xa5, 0x1b, 0x39, 0x85, 0x61, 0x10, 0xa5, 0x27, 0x17, 0x53, 0x26,
0xd8, 0x9f, 0x60, 0x34, 0x02, 0x47, 0xe5, 0x33, 0x13, 0xa5, 0x0d, 0x72, 0x00, 0x6b, 0x8d, 0x5b,
0x4c, 0x31, 0x57, 0xc8, 0xa3, 0xeb, 0xc9, 0xb7, 0x7e, 0x24, 0x4f, 0xfe, 0x07, 0x67, 0xc2, 0x4e,
0xce, 0x7f, 0x93, 0x86, 0x7c, 0x40, 0xb0, 0xb6, 0xcb, 0x5e, 0xeb, 0xb9, 0xa8, 0xae, 0xde, 0x81,
0x5e, 0x05, 0x9a, 0xc1, 0xbc, 0x55, 0x77, 0xc2, 0x4f, 0xe7, 0x6b, 0x64, 0x96, 0x88, 0x6c, 0x49,
0xeb, 0xe0, 0xf5, 0xfb, 0x30, 0xb8, 0xea, 0x94, 0x6f, 0x7c, 0x51, 0xef, 0xa7, 0x0b, 0xbd, 0x70,
0x5f, 0xaa, 0x45, 0x61, 0x16, 0xae, 0x32, 0xee, 0xb5, 0xee, 0xa2, 0x60, 0xf8, 0xf9, 0x72, 0x8c,
0xbe, 0x5c, 0x8e, 0xd1, 0xd7, 0xcb, 0x31, 0x7a, 0xf7, 0x6d, 0xfc, 0xd7, 0x71, 0x5b, 0xfd, 0x9c,
0xee, 0x7c, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x22, 0xbf, 0xb0, 0xae, 0x06, 0x00, 0x00,
// 746 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xdd, 0x6a, 0x14, 0x4b,
0x10, 0x3e, 0xbd, 0x33, 0xb3, 0x3f, 0xb5, 0xc9, 0xb2, 0x69, 0x72, 0x0e, 0x43, 0x38, 0x2c, 0x43,
0x73, 0x0e, 0x0c, 0x82, 0x09, 0xc4, 0x1b, 0x11, 0x41, 0x9c, 0xdd, 0x0d, 0x59, 0x34, 0x21, 0xe9,
0xc4, 0xdc, 0x79, 0xd1, 0x89, 0x6d, 0x32, 0x64, 0x7e, 0xd6, 0x99, 0x1e, 0x93, 0xbd, 0xf4, 0xc2,
0x77, 0x10, 0x7d, 0x02, 0x7d, 0x12, 0x2f, 0x7d, 0x04, 0x89, 0x2f, 0x22, 0xfd, 0x33, 0x3f, 0x51,
0x8c, 0xf1, 0x6e, 0xea, 0xab, 0xae, 0xea, 0xfa, 0xea, 0xab, 0xea, 0x81, 0x41, 0x98, 0x08, 0x9e,
0x25, 0x2c, 0x5a, 0x9f, 0x67, 0xa9, 0x48, 0x71, 0xb7, 0xb4, 0xc9, 0x36, 0xb4, 0x26, 0x01, 0xf6,
0xa0, 0x7f, 0x18, 0xc6, 0x7c, 0xbf, 0x60, 0x89, 0x28, 0x62, 0x17, 0x79, 0xc8, 0xef, 0xd1, 0x26,
0x24, 0x4f, 0x8c, 0xd3, 0xa8, 0x88, 0x93, 0xa7, 0xec, 0x98, 0x47, 0x6e, 0x4b, 0x9f, 0x68, 0x40,
0x64, 0x0a, 0xce, 0x56, 0xc6, 0x62, 0x7e, 0x8b, 0x64, 0x6b, 0xd0, 0xa5, 0xe9, 0x45, 0x33, 0x53,
0x65, 0x93, 0x00, 0xda, 0x41, 0x28, 0x62, 0x36, 0xc7, 0x18, 0xec, 0x20, 0x14, 0xb9, 0x8b, 0x3c,
0xcb, 0xb7, 0xa9, 0xfa, 0xc6, 0xff, 0x81, 0xf3, 0x58, 0x88, 0x2c, 0x77, 0x5b, 0x9e, 0xe5, 0xf7,
0x37, 0x07, 0xeb, 0x15, 0x31, 0x09, 0x53, 0xed, 0x24, 0xeb, 0x60, 0xef, 0xb1, 0x30, 0xc3, 0x43,
0xb0, 0x9e, 0xf0, 0x85, 0xaa, 0xc0, 0xa6, 0xf2, 0x13, 0xaf, 0x82, 0x33, 0x4e, 0x8b, 0x44, 0xa8,
0x6b, 0x6d, 0xaa, 0x0d, 0xf2, 0x1c, 0xac, 0x20, 0x14, 0xb2, 0x2c, 0x7d, 0xf5, 0x6c, 0x62, 0x62,
0x2a, 0x1b, 0xff, 0x0b, 0xbd, 0xbd, 0x2c, 0x7d, 0x19, 0x46, 0x7c, 0x36, 0x31, 0xc1, 0x35, 0x20,
0xbd, 0x92, 0x5f, 0x2e, 0x58, 0x3c, 0x77, 0x2d, 0x0f, 0xf9, 0x16, 0xad, 0x01, 0xf2, 0x08, 0x3a,
0xe6, 0x28, 0x1e, 0x40, 0xab, 0x4a, 0xde, 0x9a, 0x4d, 0x6e, 0xc9, 0xe7, 0x13, 0x02, 0x5b, 0x7e,
0x35, 0x09, 0xf5, 0x34, 0x21, 0x0c, 0xf6, 0xe1, 0x62, 0xce, 0x4d, 0x49, 0xea, 0x5b, 0x0a, 0x70,
0x20, 0xb2, 0x30, 0x39, 0x3d, 0x62, 0x51, 0xc1, 0x55, 0x3d, 0x3d, 0xda, 0x84, 0x64, 0xbd, 0xcf,
0xc2, 0x44, 0x68, 0xbf, 0xad, 0xd9, 0x54, 0x80, 0xf4, 0x06, 0x69, 0x1a, 0x69, 0xaf, 0xe3, 0x21,
0xbf, 0x4b, 0x6b, 0x00, 0x8f, 0x00, 0xb6, 0xa2, 0x94, 0x99, 0xe0, 0xb6, 0x87, 0x7c, 0x44, 0x1b,
0x08, 0xd9, 0x80, 0x8e, 0xac, 0x75, 0x87, 0xcd, 0x6b, 0x76, 0xe8, 0x26, 0x76, 0xef, 0x11, 0x2c,
0xed, 0x17, 0x3c, 0x5b, 0x50, 0xfe, 0xaa, 0xe0, 0xb9, 0x90, 0x4d, 0x9a, 0x04, 0x86, 0xa4, 0x9c,
0xce, 0x55, 0x70, 0x94, 0xdf, 0xcc, 0x8a, 0x36, 0xf0, 0x3f, 0xd0, 0x3e, 0x88, 0xc2, 0x13, 0x9e,
0xbb, 0x96, 0x1a, 0x10, 0x63, 0x49, 0x15, 0x4d, 0xb7, 0x73, 0x45, 0xad, 0x4b, 0x2b, 0x1b, 0xbb,
0xd0, 0x29, 0xc7, 0xd2, 0x51, 0xb9, 0x4a, 0x53, 0x66, 0xa3, 0x3c, 0x4e, 0x85, 0x66, 0xd4, 0xa5,
0xc6, 0x22, 0x6f, 0x10, 0x2c, 0x9b, 0xe2, 0xf2, 0x79, 0x9a, 0xe4, 0x5c, 0x6a, 0x30, 0xcd, 0xb2,
0x52, 0x83, 0x69, 0x96, 0xe1, 0x0d, 0xe8, 0x50, 0x9e, 0x17, 0x91, 0x28, 0x65, 0xfc, 0xbb, 0x26,
0x5a, 0xc6, 0x16, 0x91, 0xa0, 0xe5, 0x29, 0x7c, 0xb7, 0x51, 0xa2, 0xa5, 0x22, 0x56, 0xea, 0x08,
0xe3, 0xa9, 0xab, 0x26, 0x6f, 0x11, 0xf4, 0x1b, 0x79, 0xb0, 0x5f, 0xae, 0x88, 0x2a, 0xa2, 0xbf,
0x39, 0xac, 0x83, 0x35, 0x4e, 0xcb, 0x15, 0x5a, 0x02, 0xb4, 0x6b, 0x46, 0x03, 0xed, 0x4a, 0x39,
0xe4, 0x5a, 0x94, 0x77, 0x36, 0xe4, 0x90, 0x30, 0xd5, 0x4e, 0xd9, 0xa3, 0xf1, 0x19, 0x4b, 0x4e,
0xf9, 0x0b, 0xd3, 0xbe, 0xd2, 0x24, 0x1f, 0x11, 0x2c, 0xcf, 0xe2, 0x79, 0x9a, 0x89, 0x1b, 0x94,
0x52, 0x6f, 0x40, 0xa9, 0x94, 0x7e, 0x10, 0x56, 0xc1, 0x51, 0xda, 0xa8, 0x49, 0xb4, 0xa9, 0x36,
0xd4, 0x94, 0x99, 0xed, 0x92, 0x42, 0x49, 0x09, 0x6b, 0x40, 0x4e, 0x59, 0xb5, 0x5e, 0xb9, 0xeb,
0x28, 0x77, 0x03, 0x91, 0xfe, 0x6a, 0xc1, 0x72, 0xb7, 0xed, 0x59, 0xbe, 0x45, 0x1b, 0x08, 0x21,
0x30, 0x28, 0x4b, 0xfd, 0x95, 0x6e, 0xe4, 0x12, 0x86, 0x41, 0x94, 0x9e, 0x9c, 0x4f, 0x98, 0x60,
0x7f, 0xcc, 0x48, 0x45, 0x96, 0x8c, 0x94, 0x51, 0xf3, 0xb4, 0x9b, 0x3c, 0x31, 0xd8, 0x47, 0x21,
0xbf, 0x30, 0x03, 0xa7, 0xbe, 0xc9, 0x3e, 0xac, 0x34, 0x6e, 0x36, 0x05, 0x5e, 0x6b, 0x08, 0xba,
0xb9, 0x21, 0xad, 0x1f, 0x1b, 0x42, 0xfe, 0x07, 0x67, 0xcc, 0x4e, 0xce, 0x7e, 0x93, 0x86, 0x7c,
0x40, 0xb0, 0xb2, 0xc3, 0x2e, 0xf5, 0xae, 0x54, 0x57, 0x6f, 0x43, 0xaf, 0x02, 0xcd, 0xb2, 0xde,
0xa9, 0xa7, 0xe3, 0xa7, 0xf3, 0x35, 0x32, 0x4d, 0x44, 0xb6, 0xa0, 0x75, 0xf0, 0xda, 0x43, 0x18,
0x5c, 0x77, 0xca, 0xbe, 0x9f, 0xd7, 0x6f, 0xd6, 0xb9, 0x7e, 0x84, 0x5f, 0xab, 0xc7, 0xc3, 0x3c,
0xc2, 0xca, 0x78, 0xd0, 0xba, 0x8f, 0x82, 0xe1, 0xe7, 0xab, 0x11, 0xfa, 0x72, 0x35, 0x42, 0x5f,
0xaf, 0x46, 0xe8, 0xdd, 0xb7, 0xd1, 0x5f, 0xc7, 0x6d, 0xf5, 0xc3, 0xba, 0xf7, 0x3d, 0x00, 0x00,
0xff, 0xff, 0x1c, 0xb1, 0xe1, 0x29, 0xc2, 0x06, 0x00, 0x00,
}

5
internal/internal.proto
View file

@ -84,8 +84,9 @@ message ImportResponse {
message BlockDataRequest {
string DB = 1;
string Frame = 2;
uint64 Slice = 3;
uint64 Block = 4;
string View = 5;
uint64 Slice = 4;
uint64 Block = 3;
}
message BlockDataResponse {

32
time.go
View file

@ -48,8 +48,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) {
return q, nil
}
// FrameByTimeUnit returns the frame name for time with a given quantum unit.
func FrameByTimeUnit(name string, t time.Time, unit rune) string {
// ViewByTimeUnit returns the view name for time with a given quantum unit.
func ViewByTimeUnit(name string, t time.Time, unit rune) string {
switch unit {
case 'Y':
return fmt.Sprintf("%s_%s", name, t.Format("2006"))
@ -64,21 +64,21 @@ func FrameByTimeUnit(name string, t time.Time, unit rune) string {
}
}
// FramesByTime returns a list of frames for a given timestamp.
func FramesByTime(name string, t time.Time, q TimeQuantum) []string {
// ViewsByTime returns a list of views for a given timestamp.
func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
a := make([]string, 0, len(q))
for _, unit := range q {
frame := FrameByTimeUnit(name, t, unit)
if frame == "" {
view := ViewByTimeUnit(name, t, unit)
if view == "" {
continue
}
a = append(a, frame)
a = append(a, view)
}
return a
}
// FramesByTimeRange returns a list of frames to traverse to query a time range.
func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
// ViewsByTimeRange returns a list of views to traverse to query a time range.
func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
t := start
// Save flags for performance.
@ -96,7 +96,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin
if !nextDayGTE(t, end) {
break
} else if t.Hour() != 0 {
results = append(results, FrameByTimeUnit(name, t, 'H'))
results = append(results, ViewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
continue
}
@ -107,7 +107,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin
if !nextMonthGTE(t, end) {
break
} else if t.Day() != 1 {
results = append(results, FrameByTimeUnit(name, t, 'D'))
results = append(results, ViewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
continue
}
@ -117,7 +117,7 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin
if !nextYearGTE(t, end) {
break
} else if t.Month() != 1 {
results = append(results, FrameByTimeUnit(name, t, 'M'))
results = append(results, ViewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
continue
}
@ -133,16 +133,16 @@ func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []strin
// Walk back down from largest units to smallest units.
for t.Before(end) {
if hasYear && nextYearGTE(t, end) {
results = append(results, FrameByTimeUnit(name, t, 'Y'))
results = append(results, ViewByTimeUnit(name, t, 'Y'))
t = t.AddDate(1, 0, 0)
} else if hasMonth && nextMonthGTE(t, end) {
results = append(results, FrameByTimeUnit(name, t, 'M'))
results = append(results, ViewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
} else if hasDay && nextDayGTE(t, end) {
results = append(results, FrameByTimeUnit(name, t, 'D'))
results = append(results, ViewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
} else if hasHour {
results = append(results, FrameByTimeUnit(name, t, 'H'))
results = append(results, ViewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
} else {
break

40
time_test.go
View file

@ -25,45 +25,45 @@ func TestParseTimeQuantum(t *testing.T) {
})
}
// Ensure generated frame name can be returned for a given time unit.
func TestFrameByTimeUnit(t *testing.T) {
// Ensure generated view name can be returned for a given time unit.
func TestViewByTimeUnit(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("Y", func(t *testing.T) {
if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" {
if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("M", func(t *testing.T) {
if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" {
if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("D", func(t *testing.T) {
if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" {
if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("H", func(t *testing.T) {
if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
t.Fatalf("unexpected name: %s", s)
}
})
}
// Ensure all applicable frame names can be generated when mutating a time bit.
func TestFramesByTime(t *testing.T) {
func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("YMDH", func(t *testing.T) {
a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH"))
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) {
t.Fatalf("unexpected names: %+v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D"))
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000102"}) {
t.Fatalf("unexpected names: %+v", a)
}
@ -71,63 +71,63 @@ func TestFramesByTime(t *testing.T) {
}
// Ensure sets of frames can be returned for a given time range.
func TestFramesByTimeRange(t *testing.T) {
func TestViewsByTimeRange(t *testing.T) {
t.Run("Y", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("YM", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("YMD", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("YMDH", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("M", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("MD", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("MDH", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("DH", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) {
t.Fatalf("unexpected frames: %#v", a)
}
})
t.Run("H", func(t *testing.T) {
a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) {
t.Fatalf("unexpected frames: %#v", a)
}

249
view.go Normal file
View file

@ -0,0 +1,249 @@
package pilosa
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"sync"
)
// View layout modes.
const (
ViewStandard = "standard"
ViewInverse = "inverse"
)
// View represents a container for frame data.
type View struct {
mu sync.Mutex
path string
db string
frame string
name string
// Fragments by slice.
fragments map[uint64]*Fragment
stats StatsClient
BitmapAttrStore *AttrStore
LogOutput io.Writer
}
// NewView returns a new instance of View.
func NewView(path, db, frame, name string) *View {
return &View{
path: path,
db: db,
frame: frame,
name: name,
fragments: make(map[uint64]*Fragment),
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}
}
// Name returns the name the view was initialized with.
func (v *View) Name() string { return v.name }
// DB returns the database name the view was initialized with.
func (v *View) DB() string { return v.db }
// Frame returns the frame name the view was initialized with.
func (v *View) Frame() string { return v.frame }
// Path returns the path the view was initialized with.
func (v *View) Path() string { return v.path }
// Open opens and initializes the view.
func (v *View) Open() error {
if err := func() error {
// Ensure the view's path exists.
if err := os.MkdirAll(v.path, 0777); err != nil {
return err
} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
return err
}
if err := v.openFragments(); err != nil {
return err
}
return nil
}(); err != nil {
v.Close()
return err
}
return nil
}
// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return err
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return err
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
continue
}
frag := v.newFragment(v.FragmentPath(slice), slice)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
}
frag.BitmapAttrStore = v.BitmapAttrStore
v.fragments[frag.Slice()] = frag
v.stats.Count("maxSlice", 1)
}
return nil
}
// Close closes the view and its fragments.
func (v *View) Close() error {
v.mu.Lock()
defer v.mu.Unlock()
// Close all fragments.
for _, frag := range v.fragments {
_ = frag.Close()
}
v.fragments = make(map[uint64]*Fragment)
return nil
}
// MaxSlice returns the max slice in the view.
func (v *View) MaxSlice() uint64 {
v.mu.Lock()
defer v.mu.Unlock()
var max uint64
for slice := range v.fragments {
if slice > max {
max = slice
}
}
return max
}
// FragmentPath returns the path to a fragment in the view.
func (v *View) FragmentPath(slice uint64) string {
return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10))
}
// Fragment returns a fragment in the view by slice.
func (v *View) Fragment(slice uint64) *Fragment {
v.mu.Lock()
defer v.mu.Unlock()
return v.fragment(slice)
}
func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] }
// Fragments returns a list of all fragments in the view.
func (v *View) Fragments() []*Fragment {
v.mu.Lock()
defer v.mu.Unlock()
other := make([]*Fragment, 0, len(v.fragments))
for _, fragment := range v.fragments {
other = append(other, fragment)
}
return other
}
// CreateFragmentIfNotExists returns a fragment in the view by slice.
func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
v.mu.Lock()
defer v.mu.Unlock()
return v.createFragmentIfNotExists(slice)
}
func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Find fragment in cache first.
if frag := v.fragments[slice]; frag != nil {
return frag, nil
}
// Initialize and open fragment.
frag := v.newFragment(v.FragmentPath(slice), slice)
if err := frag.Open(); err != nil {
return nil, err
}
frag.BitmapAttrStore = v.BitmapAttrStore
// Save to lookup.
v.fragments[slice] = frag
v.stats.Count("maxSlice", 1)
return frag, nil
}
func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.db, v.frame, v.name, slice)
frag.LogOutput = v.LogOutput
frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
return frag
}
// SetBit sets a bit within the view.
func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
slice := profileID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.SetBit(bitmapID, profileID)
}
// ClearBit clears a bit within the view.
func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) {
slice := profileID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.ClearBit(bitmapID, profileID)
}
type viewSlice []*View
func (p viewSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p viewSlice) Len() int { return len(p) }
func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`
}
type viewInfoSlice []*ViewInfo
func (p viewInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int { return len(p) }
func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }

80
view_test.go Normal file
View file

@ -0,0 +1,80 @@
package pilosa_test
import (
"io/ioutil"
"os"
"github.com/pilosa/pilosa"
)
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
BitmapAttrStore *AttrStore
}
// NewView returns a new instance of View with a temporary path.
func NewView(db, frame, name string) *View {
file, err := ioutil.TempFile("", "pilosa-view-")
if err != nil {
panic(err)
}
file.Close()
v := &View{
View: pilosa.NewView(file.Name(), db, frame, name),
BitmapAttrStore: MustOpenAttrStore(),
}
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
return v
}
// MustOpenView creates and opens an view at a temporary path. Panic on error.
func MustOpenView(db, frame, name string) *View {
v := NewView(db, frame, name)
if err := v.Open(); err != nil {
panic(err)
}
return v
}
// Close closes the view and removes all underlying data.
func (v *View) Close() error {
defer os.Remove(v.Path())
defer v.BitmapAttrStore.Close()
return v.View.Close()
}
// Reopen closes the view and reopens it as a new instance.
func (v *View) Reopen() error {
path := v.Path()
if err := v.View.Close(); err != nil {
return err
}
v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name())
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
if err := v.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets bits on a bitmap. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := v.SetBit(bitmapID, profileID); err != nil {
panic(err)
}
}
}
// MustClearBits clears bits on a bitmap. Panic on error.
func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := v.ClearBit(bitmapID, profileID); err != nil {
panic(err)
}
}
}