From 0b38a4d956995a0c8f9ec28e4c6bb69cae63ec68 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 29 Aug 2018 23:13:04 +0300 Subject: [PATCH 01/65] updated go client sample to match latest master --- docs/client-libraries.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index a277141ec..ac008def4 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -47,9 +47,9 @@ func main() { } // We need to refer to indexes and fields before we can use them in a query. - repository, _ := schema.Index("repository") - stargazer, _ := repository.Field("stargazer") - language, _ := repository.Field("language") + repository := schema.Index("repository") + stargazer := repository.Field("stargazer") + language := repository.Field("language") var response *pilosa.QueryResponse From 480c853a3ec36c0a1246f1f27a3640548c7d6766 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 16:23:31 +0300 Subject: [PATCH 02/65] Use passed stdin, stdout and stderr in the cmd package. Fixes #1538 --- cmd/check.go | 5 ++--- cmd/config.go | 3 +-- cmd/export.go | 2 +- cmd/generate_config.go | 5 ++--- cmd/inspect.go | 5 ++--- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index d7bd33afd..19e0519a8 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "os" "github.com/spf13/cobra" @@ -27,8 +26,8 @@ import ( var checker *ctl.CheckCommand -func newCheckCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - checker = ctl.NewCheckCommand(os.Stdin, os.Stdout, os.Stderr) +func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command { + checker = ctl.NewCheckCommand(stdin, stdout, stderr) checkCmd := &cobra.Command{ Use: "check [path2]...", Short: "Do a consistency check on a pilosa data file.", diff --git a/cmd/config.go b/cmd/config.go index a877d12ef..73464ba65 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -28,7 +27,7 @@ import ( var conf *ctl.ConfigCommand func newConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - conf = ctl.NewConfigCommand(os.Stdin, os.Stdout, os.Stderr) + conf = ctl.NewConfigCommand(stdin, stdout, stderr) Server := server.NewCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "config", diff --git a/cmd/export.go b/cmd/export.go index 217e9f09c..4e7c4d232 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -26,7 +26,7 @@ import ( var Exporter *ctl.ExportCommand -func newExportCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { +func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) exportCmd := &cobra.Command{ Use: "export", diff --git a/cmd/generate_config.go b/cmd/generate_config.go index 6421a1f4f..8c8553aa4 100644 --- a/cmd/generate_config.go +++ b/cmd/generate_config.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -26,8 +25,8 @@ import ( var generateConf *ctl.GenerateConfigCommand -func newGenerateConfigCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr) +func newGenerateConfigCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command { + generateConf = ctl.NewGenerateConfigCommand(stdin, stdout, stderr) confCmd := &cobra.Command{ Use: "generate-config", Short: "Print the default configuration.", diff --git a/cmd/inspect.go b/cmd/inspect.go index 4f5ff9301..570e7aae3 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "os" "github.com/spf13/cobra" @@ -27,8 +26,8 @@ import ( var inspector *ctl.InspectCommand -func newInspectCommand(_ io.Reader, _, _ io.Writer) *cobra.Command { - inspector = ctl.NewInspectCommand(os.Stdin, os.Stdout, os.Stderr) +func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + inspector = ctl.NewInspectCommand(stdin, stdout, stderr) inspectCmd := &cobra.Command{ Use: "inspect", From 7b872e00cb46128224e83df7a090a8c3e71b3d23 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 17:10:56 +0300 Subject: [PATCH 03/65] Implements #1570 --- client.go | 8 ++++++++ cmd/import.go | 2 ++ ctl/import.go | 19 +++++++++++++++++-- http/client.go | 29 ++++++++++++++++++++++++++--- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index a0b89f604..1b1b44e31 100644 --- a/client.go +++ b/client.go @@ -41,10 +41,12 @@ type InternalClient interface { ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error + EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error ImportValueK(ctx context.Context, index, field string, vals []FieldValue) error ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error + CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) @@ -112,6 +114,9 @@ func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { return nil } +func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue) error { return nil } @@ -122,6 +127,9 @@ func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, s return nil } func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil } +func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) { return nil, nil } diff --git a/cmd/import.go b/cmd/import.go index 7e82bfd46..5c05949f9 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -52,6 +52,8 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") + flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") + flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/ctl/import.go b/ctl/import.go index 2969b581b..83c95be84 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -40,9 +40,12 @@ type ImportCommand struct { // nolint: maligned Index string `json:"index"` Field string `json:"field"` - // Options for index & field to be created if they don't exist + // Options for the index to be created if it doesn't exist indexOptions pilosa.IndexOptions + // Options for the field to be created if it doesn't exist + fieldOptions pilosa.FieldOptions + // CreateSchema ensures the schema exists before import CreateSchema bool @@ -50,6 +53,12 @@ type ImportCommand struct { // nolint: maligned // TODO: remove this in a future release StringKeys bool `json:"StringKeys"` + // IndexKeys makes the import command use keys=true when creating an index + IndexKeys bool `json:"indexKeys"` + + // FieldKeys makes the import command use keys=true when creating a field + FieldKeys bool `json:"fieldKeys"` + // Filenames to import from. Paths []string `json:"paths"` @@ -102,6 +111,12 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { + cmd.indexOptions = pilosa.IndexOptions{ + Keys: cmd.IndexKeys, + } + cmd.fieldOptions = pilosa.FieldOptions{ + Keys: cmd.FieldKeys, + } err := cmd.ensureSchema(ctx) if err != nil { return errors.Wrap(err, "ensuring schema") @@ -146,7 +161,7 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.client.EnsureField(ctx, cmd.Index, cmd.Field) + err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } diff --git a/http/client.go b/http/client.go index d996b9e57..64a6675e0 100644 --- a/http/client.go +++ b/http/client.go @@ -372,7 +372,11 @@ func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options p } func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error { - err := c.CreateField(ctx, indexName, fieldName) + return c.EnsureFieldWithOptions(ctx, indexName, fieldName, pilosa.FieldOptions{}) +} + +func (c *InternalClient) EnsureFieldWithOptions(ctx context.Context, indexName string, fieldName string, opt pilosa.FieldOptions) error { + err := c.CreateFieldWithOptions(ctx, indexName, fieldName, opt) if err == nil || errors.Cause(err) == pilosa.ErrFieldExists { return nil } @@ -636,16 +640,35 @@ func (c *InternalClient) backupShardNode(ctx context.Context, index, field strin return resp.Body, nil } -// CreateField creates a new field on the server. func (c *InternalClient) CreateField(ctx context.Context, index, field string) error { + return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{}) +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { if index == "" { return pilosa.ErrIndexRequired } + // convert pilosa.FieldOptions to fieldOptions + fieldOpt := fieldOptions{ + Type: opt.Type, + Keys: &opt.Keys, + } + if fieldOpt.Type == "set" { + fieldOpt.CacheType = &opt.CacheType + fieldOpt.CacheSize = &opt.CacheSize + } else if fieldOpt.Type == "int" { + fieldOpt.Min = &opt.Min + fieldOpt.Max = &opt.Max + } else if fieldOpt.Type == "time" { + fieldOpt.TimeQuantum = &opt.TimeQuantum + } + // TODO: remove buf completely? (depends on whether importer needs to create specific field types) // Encode query request. buf, err := json.Marshal(&postFieldRequest{ - //Options: opt, + Options: fieldOpt, }) if err != nil { return errors.Wrap(err, "marshaling") From fe1208ac656d1ce4d21b6418a741280e31b7023a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 17:37:44 +0300 Subject: [PATCH 04/65] Fix std{in,out,err} in export cmd --- cmd/export.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index 4e7c4d232..6cfeb0d9c 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -17,7 +17,6 @@ package cmd import ( "context" "io" - "os" "github.com/spf13/cobra" @@ -27,7 +26,7 @@ import ( var Exporter *ctl.ExportCommand func newExportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Exporter = ctl.NewExportCommand(os.Stdin, os.Stdout, os.Stderr) + Exporter = ctl.NewExportCommand(stdin, stdout, stderr) exportCmd := &cobra.Command{ Use: "export", Short: "Export data from pilosa.", From 019ca63cf4f0bfaa2d3ed280540b7cb753731808 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 31 Jul 2018 13:18:19 -0500 Subject: [PATCH 05/65] add support for reading standard roaring bitmaps to pilosa/roaring. --- roaring/roaring.go | 164 +++++++++++++++++++++++++++++++ roaring/roaring_internal_test.go | 31 ++++++ 2 files changed, 195 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index e7333325c..db0633fac 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3339,3 +3339,167 @@ func popcountAndSlice(s, m []uint64) uint64 { } return cnt } + +// Stuff i needed from the roaring project +const ( //taken from roaring/util.go + serialCookieNoRunContainer = 12346 // only arrays and bitmaps + serialCookie = 12347 // runs, arrays, and bitmaps + noOffsetThreshold = 4 +) + +//end roaring stuff + +func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, err error, haveRuns bool) { + if len(buf) < 8 { + err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) + return + } + cf := func(index uint, card int) (newType byte) { + newType = containerBitmap + if card < ArrayMaxSize { + newType = containerArray + } + return + } + containerTyper = cf + cookie := binary.LittleEndian.Uint32(buf) + pos += 4 + + // cookie header + if cookie == serialCookieNoRunContainer { + size = binary.LittleEndian.Uint32(buf[pos:]) + pos += 4 + } else if cookie&0x0000FFFF == serialCookie { + haveRuns = true + size = uint32(uint16(cookie>>16) + 1) // number of containers + + // create is-run-container bitmap + isRunBitmapSize := (int(size) + 7) / 8 + if pos+isRunBitmapSize > len(buf) { + err = fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) + return + } + + isRunBitmap := buf[pos : pos+isRunBitmapSize] + pos += isRunBitmapSize + containerTyper = func(index uint, card int) byte { + if isRunBitmap[index/8]&(1<<(index%8)) != 0 { + return containerRun + } + return cf(index, card) + } + } else { + err = fmt.Errorf("did not find expected serialCookie in header") + return + } + + header = pos + if size > (1 << 16) { + err = fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + return + } + + // descriptive header + if pos+2*2*int(size) > len(buf) { + err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) + return + } + pos += 2 * 2 * int(size) //moving pos past keycount + return +} + +func unmarshalStandardRoaring(data []byte) (*Bitmap, error) { + + b := NewBitmap() + keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) + if err != nil { + return nil, err + } + + b.Containers.Reset() + // Descriptive header section: Read container keys and cardinalities. + for i, buf := uint(0), data[header:]; i < uint(keyN); i, buf = i+1, buf[4:] { + card := int(binary.LittleEndian.Uint16(buf[2:4])) + 1 + b.Containers.PutContainerValues( + uint64(binary.LittleEndian.Uint16(buf[0:2])), + containerTyper(i, card), /// container type voodo with isRunBitmap + card, + true) + } + + // Read container offsets and attach data. + if haveRuns { + readWithRuns(b, data, pos, keyN) + } else { + readOffsets(b, data, pos, keyN) + } + return b, nil +} + +func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { + + citer, _ := b.Containers.Iterator(0) + for i, buf := 0, data[pos:]; i < int(keyN); i, buf = i+1, buf[4:] { + offset := binary.LittleEndian.Uint32(buf[0:4]) + // Verify the offset is within the bounds of the input data. + if int(offset) >= len(data) { + err = fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) + return + } + + // Map byte slice directly to the container data. + citer.Next() + _, c := citer.Value() + switch c.containerType { + case containerArray: + c.runs = nil + c.bitmap = nil + c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n] + case containerBitmap: + c.array = nil + c.runs = nil + c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] + default: + err = fmt.Errorf("unsupported container type %d", c.containerType) + return + } + } + return + +} + +func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { + + citer, _ := b.Containers.Iterator(0) + for i := 0; i < int(keyN); i++ { + citer.Next() + _, c := citer.Value() + switch c.containerType { + case containerRun: + c.array = nil + c.bitmap = nil + runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize]) + c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount] + + for o := range c.runs { //need to convert to start:end vs start:length :( + r := c.runs[o] + r.last = r.start + r.last + c.runs[o] = r + + } + pos += int((runCount * interval16Size) + runCountHeaderSize) + case containerArray: + c.runs = nil + c.bitmap = nil + c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n] + pos += c.n * 2 + case containerBitmap: + c.array = nil + c.runs = nil + c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[pos]))[:bitmapN] + pos += bitmapN * 8 + } + } + return + +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 5bbcac86b..745aa917a 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -16,6 +16,7 @@ package roaring import ( "bytes" + "encoding/hex" "fmt" "reflect" "runtime" @@ -3225,3 +3226,33 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { } return nil } + +func TestUnmarshalStdRoaring(t *testing.T) { + //generated serialize image from java(clojure) with arrays + _2arrayContainer, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") + bm, er := unmarshalStandardRoaring(_2arrayContainer) + if er != nil { + t.Fatalf("unmarshalStandardRoaring %s", er) + } + if bm.Count() != 4 { + t.Fatalf("unexpected bitmap %v expected bits [1 2 3 65537]", bm.Slice()) + } + //generated serialize image from java(clojure) with a run and array + _rle_array_container, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + bm, er = unmarshalStandardRoaring(_rle_array_container) + if er != nil { + t.Fatalf("unmarshalStandardRoaring %s", er) + } + if bm.Count() != 11 { + t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) + } + _bitmap_array_container, _ := hex.DecodeString("3A3000000200000000000E27010000001800000018200000FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100") + bm, er = unmarshalStandardRoaring(_bitmap_array_container) + if er != nil { + t.Fatalf("unmarshalStandardRoaring %s", er) + } + if bm.Count() != 10000 { + t.Fatalf("expecting X got %d", bm.Count()) + } + +} From cc80e0b0e455bb98ebba5a75d870c9b9ca7d6305 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 3 Aug 2018 08:36:46 -0500 Subject: [PATCH 06/65] initial support for bulk importing standard roaring files per shard --- api.go | 38 ++++++++- client.go | 4 + field.go | 20 +++++ fragment.go | 107 +++++++++++++++----------- http/client.go | 46 +++++++++-- http/handler.go | 48 ++++++++++++ roaring/bitmapcontainer.roaringbitmap | Bin 0 -> 8218 bytes roaring/roaring.go | 7 +- roaring/roaring_internal_test.go | 4 +- 9 files changed, 215 insertions(+), 59 deletions(-) create mode 100644 roaring/bitmapcontainer.roaringbitmap diff --git a/api.go b/api.go index fb64b5bdc..2947bbae0 100644 --- a/api.go +++ b/api.go @@ -1,4 +1,3 @@ -// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +23,7 @@ import ( "io/ioutil" "strconv" "strings" + "sync" "time" "github.com/pilosa/pilosa/pql" @@ -294,6 +294,42 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e return field, nil } +// ImportRoaringBytes fast loading of standard roaring format +func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64, forward bool) (err error) { + if err = api.validate(apiField); err != nil { + err = errors.Wrap(err, "validating api method") + return + } + nodes := api.cluster.shardNodes(indexName, shard) + var wg sync.WaitGroup + + for _, node := range nodes { + if node.ID == api.server.nodeID { + field := api.holder.Field(indexName, fieldName) + if field == nil { + err = newNotFoundError(ErrFieldNotFound) + return + } + wg.Add(1) + go func(node *Node) { + err = field.importRoaringBytes(roaringBytes, shard) + wg.Done() + }(node) + } else if forward { + wg.Add(1) + //forward it on + go func(node *Node) { + //execute on node + err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes, forward) + wg.Done() + }(node) + } + + } + wg.Wait() + return +} + // DeleteField removes the named field from the named index. If the index is not // found, an error is returned. If the field is not found, it is ignored and no // action is taken. diff --git a/client.go b/client.go index a0b89f604..d0e36db6d 100644 --- a/client.go +++ b/client.go @@ -51,6 +51,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) + ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error } //=============== @@ -106,6 +107,9 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } +func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { + return nil +} func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { return nil } diff --git a/field.go b/field.go index 9fb352fc1..4223d3948 100644 --- a/field.go +++ b/field.go @@ -1055,6 +1055,26 @@ func (f *Field) importValue(columnIDs []uint64, values []int64) error { return nil } +func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error { + viewName := viewStandard + + view, err := f.createViewIfNotExists(viewName) + if err != nil { + return errors.Wrap(err, "creating view") + } + + frag, err := view.CreateFragmentIfNotExists(shard) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + + if err := frag.importRoaringBytes(stdRoaringBytes); err != nil { + return err + } + + return nil +} + type fieldSlice []*Field func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/fragment.go b/fragment.go index e4fbcbbda..63cde2d0e 100644 --- a/fragment.go +++ b/fragment.go @@ -1327,69 +1327,66 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() + // Verify that there are an equal number of row ids and column ids. if len(rowIDs) != len(columnIDs) { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } // Disconnect op writer so we don't append updates. - f.storage.OpWriter = nil - + localBitmap := roaring.NewBitmap() + localBitmap.OpWriter = nil // Process every bit. // If an error occurs then reopen the storage. lastID := uint64(0) - if err := func() error { - set := make(map[uint64]struct{}) - for i := range rowIDs { - rowID, columnID := rowIDs[i], columnIDs[i] + set := make(map[uint64]struct{}) + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] - // Determine the position of the bit in the storage. - pos, err := f.pos(rowID, columnID) - if err != nil { - return errors.Wrap(err, "getting bit pos") - } - - // Write to storage. - _, err = f.storage.Add(pos) - if err != nil { - return errors.Wrap(err, "writing") - } - // Reduce the StatsD rate for high volume stats - f.stats.Count("ImportBit", 1, 0.0001) - // import optimization to avoid linear foreach calls - // slight risk of concurrent cache counter being off but - // no real danger - if i == 0 || rowID != lastID { - lastID = rowID - set[rowID] = struct{}{} - } - - // Invalidate block checksum. - delete(f.checksums, int(rowID/HashBlockSize)) + // Determine the position of the bit in the storage. + pos, err := f.pos(rowID, columnID) + if err != nil { + return err } - // Update cache counts for all rows. - for rowID := range set { - // Import should ALWAYS have row() load a new row from fragment.storage - // because the row that's in rowCache hasn't been updated with - // this import's data. - f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count()) + // Write to storage. + _, err = localBitmap.Add(pos) + if err != nil { + return err + } + // Reduce the StatsD rate for high volume stats + f.stats.Count("ImportBit", 1, 0.0001) + // import optimization to avoid linear foreach calls + // slight risk of concurrent cache counter being off but + // no real danger + if i == 0 || rowID != lastID { + lastID = rowID + set[rowID] = struct{}{} } - f.cache.Invalidate() - return nil - }(); err != nil { - _ = f.closeStorage() - _ = f.openStorage() - return err + // Invalidate block checksum. + delete(f.checksums, int(rowID/HashBlockSize)) } - // Write the storage to disk and reload. - if err := f.snapshot(); err != nil { - return errors.Wrap(err, "snapshotting") + f.mu.Lock() + defer f.mu.Unlock() + + //f.storage.Unmmap() + // Update cache counts for all rows. + var results *roaring.Bitmap + if f.storage.Count() > 0 { + results = f.storage.Union(localBitmap) + } else { + results = localBitmap } - return nil + for rowID := range set { + n := results.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.BulkAdd(rowID, n) + } + + f.cache.Invalidate() + return snapshot(f, results) } // importValue bulk imports a set of range-encoded values. @@ -1425,6 +1422,19 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error return nil } +// importRoaringBytes imports from standard roaring data format defined at +// https://github.com/RoaringBitmap/RoaringFormatSpec +func (f *fragment) importRoaringBytes(roaringBytes []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + bm, err := roaring.UnmarshalStandardRoaring(roaringBytes) + if err != nil { + return err + } + err = snapshot(f, bm) + return err +} + // incrementOpN increase the operation count by one. // If the count exceeds the maximum allowed then a snapshot is performed. func (f *fragment) incrementOpN() error { @@ -1452,6 +1462,11 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *fragment) snapshot() error { + return snapshot(f, f.storage) +} + +func snapshot(f *fragment, bm *roaring.Bitmap) error { + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) start := time.Now() @@ -1467,7 +1482,7 @@ func (f *fragment) snapshot() error { // Write storage to snapshot. bw := bufio.NewWriter(file) - if _, err := f.storage.WriteTo(bw); err != nil { + if _, err := bm.WriteTo(bw); err != nil { return fmt.Errorf("snapshot write to: %s", err) } diff --git a/http/client.go b/http/client.go index d996b9e57..6b43c731f 100644 --- a/http/client.go +++ b/http/client.go @@ -475,12 +475,6 @@ func (c *InternalClient) ImportValue(ctx context.Context, index, field string, s // ImportValueK bulk imports keyed field values to a host. func (c *InternalClient) ImportValueK(ctx context.Context, index, field string, vals []pilosa.FieldValue) error { - if index == "" { - return pilosa.ErrIndexRequired - } else if field == "" { - return pilosa.ErrFieldRequired - } - buf, err := c.marshalImportValuePayload(index, field, 0, vals) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) @@ -527,6 +521,46 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui return buf, nil } +// ImportRoaringBytes fast import of raw bits in roaring standard format +func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + endpoint := fmt.Sprintf("/index/%s/field/%s/importroaring/%d", index, field, shard) + + // Create URL. + u := nodePathToURL(node, endpoint) + if forward { + v := url.Values{} + v.Set("noforward", "y") + u.RawQuery = v.Encode() + } + // Generate HTTP request. + req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Accept", "application/x-binary") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.httpClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Validate status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: %d", resp.StatusCode) + } + + return nil +} + // ExportCSV bulk exports data for a single shard from a host to CSV format. func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error { if index == "" { diff --git a/http/handler.go b/http/handler.go index 0624cfb99..f29128bd2 100644 --- a/http/handler.go +++ b/http/handler.go @@ -177,6 +177,7 @@ func (h *Handler) populateValidators() { h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") + h.validators["ImportRoaringBitmap"] = queryValidationSpecRequired("index", "field", "shard") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -217,6 +218,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}/importroaring/{shard}", handler.handlePostRoaringImport).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -1427,3 +1429,49 @@ func GetHTTPClient(t *tls.Config) *http.Client { } return &http.Client{Transport: transport} } + +// handlPostRoaringImport +func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-binary" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } /*else if validHeaderAcceptJSON(r.Header) { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + }*/ + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + shardName := mux.Vars(r)["shard"] + _, noForward := r.URL.Query()["noforward"] + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + shard, err := strconv.ParseUint(shardName, 10, 64) + if err != nil { + http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) + return + } + //TODO give meaningful stats for import + err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard, !noForward) + + // Marshal response object. + msg := string("") + if err != nil { + msg = err.Error() + } + buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: msg}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) + return + } + + // Write response. + w.Write(buf) +} diff --git a/roaring/bitmapcontainer.roaringbitmap b/roaring/bitmapcontainer.roaringbitmap new file mode 100644 index 0000000000000000000000000000000000000000..2282906cbb538435185203f0b3e4ec538e4f6bed GIT binary patch literal 8218 zcmeIyp%uU&5JbU+0-}*|*y!|MR|vEL^WWPmV8p#oQnR{xnYXRo$u~rR0D)Zv=A#n< i0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=C*1WJC=DBeu~ literal 0 HcmV?d00001 diff --git a/roaring/roaring.go b/roaring/roaring.go index db0633fac..834ff25c2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3408,7 +3408,7 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint return } -func unmarshalStandardRoaring(data []byte) (*Bitmap, error) { +func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { b := NewBitmap() keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) @@ -3482,10 +3482,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount] for o := range c.runs { //need to convert to start:end vs start:length :( - r := c.runs[o] - r.last = r.start + r.last - c.runs[o] = r - + c.runs[o].last = c.runs[o].start + c.runs[o].last } pos += int((runCount * interval16Size) + runCountHeaderSize) case containerArray: diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 745aa917a..e883a6684 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/hex" "fmt" + "io/ioutil" "reflect" "runtime" "strings" @@ -3246,7 +3247,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { if bm.Count() != 11 { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) } - _bitmap_array_container, _ := hex.DecodeString("3A3000000200000000000E27010000001800000018200000FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100") + //had to use an external file because emacs was barfing on the long line :() + _bitmap_array_container, _ := ioutil.ReadFile("./bitmapcontainer.roaringbitmap") bm, er = unmarshalStandardRoaring(_bitmap_array_container) if er != nil { t.Fatalf("unmarshalStandardRoaring %s", er) From 1d99da1f2871e66c69a122bd46dd30037fec7f3f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 9 Aug 2018 11:17:23 -0500 Subject: [PATCH 07/65] added support for both pilosa and standard roaring uploads --- http/client.go | 1 + roaring/roaring.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/http/client.go b/http/client.go index 6b43c731f..91fd7b571 100644 --- a/http/client.go +++ b/http/client.go @@ -557,6 +557,7 @@ func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.No if resp.StatusCode != http.StatusOK { return fmt.Errorf("invalid status: %d", resp.StatusCode) } + io.Copy(ioutil.Discard, resp.Body) return nil } diff --git a/roaring/roaring.go b/roaring/roaring.go index 834ff25c2..853fcade9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3409,8 +3409,14 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint } func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { - b := NewBitmap() + + fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) + if fileMagic == magicNumber { //if pilosa roaring + err := b.UnmarshalBinary(data) + return b, err + } + keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) if err != nil { return nil, err From de478870fb00d3649c90a5da2f9b7981b1adddc0 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 14 Aug 2018 13:58:54 -0500 Subject: [PATCH 08/65] found deadlock in new code --- fragment.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/fragment.go b/fragment.go index 63cde2d0e..e7014aeb0 100644 --- a/fragment.go +++ b/fragment.go @@ -1325,9 +1325,6 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e // bulkImport bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { - f.mu.Lock() - defer f.mu.Unlock() - // Verify that there are an equal number of row ids and column ids. if len(rowIDs) != len(columnIDs) { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) From 8a1f8659fa0d07202b6596fd15812b1019f4520f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 4 Sep 2018 10:35:47 -0500 Subject: [PATCH 09/65] change test to use exported UnmarshalStandardRoaring() --- api.go | 1 + roaring/roaring_internal_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api.go b/api.go index 2947bbae0..77ce4d0a0 100644 --- a/api.go +++ b/api.go @@ -1,3 +1,4 @@ +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index e883a6684..581a8040f 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3231,27 +3231,27 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { func TestUnmarshalStdRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays _2arrayContainer, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") - bm, er := unmarshalStandardRoaring(_2arrayContainer) + bm, er := UnmarshalStandardRoaring(_2arrayContainer) if er != nil { - t.Fatalf("unmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalStandardRoaring %s", er) } if bm.Count() != 4 { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 65537]", bm.Slice()) } //generated serialize image from java(clojure) with a run and array _rle_array_container, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - bm, er = unmarshalStandardRoaring(_rle_array_container) + bm, er = UnmarshalStandardRoaring(_rle_array_container) if er != nil { - t.Fatalf("unmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalStandardRoaring %s", er) } if bm.Count() != 11 { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) } //had to use an external file because emacs was barfing on the long line :() _bitmap_array_container, _ := ioutil.ReadFile("./bitmapcontainer.roaringbitmap") - bm, er = unmarshalStandardRoaring(_bitmap_array_container) + bm, er = UnmarshalStandardRoaring(_bitmap_array_container) if er != nil { - t.Fatalf("unmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalStandardRoaring %s", er) } if bm.Count() != 10000 { t.Fatalf("expecting X got %d", bm.Count()) From 48bd76b967e7893d4ccc81e18de54892177f0cfc Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 5 Sep 2018 22:58:23 +0300 Subject: [PATCH 10/65] Removed --string-keys option from pilosa import --- cmd/import.go | 1 - ctl/import.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 5c05949f9..3d63586ce 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -51,7 +51,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.StringKeys, "string-keys", false, "REMOVED (key type is now determined by index/field configuration): Treat payload as string keys.") flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") diff --git a/ctl/import.go b/ctl/import.go index 83c95be84..1c000dd73 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -49,10 +49,6 @@ type ImportCommand struct { // nolint: maligned // CreateSchema ensures the schema exists before import CreateSchema bool - // REMOVED: Indicates that the payload should be treated as string keys. - // TODO: remove this in a future release - StringKeys bool `json:"StringKeys"` - // IndexKeys makes the import command use keys=true when creating an index IndexKeys bool `json:"indexKeys"` @@ -89,11 +85,6 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // REMOVED: warning that --string-keys flag has been deprecated. - if cmd.StringKeys { - logger.Printf("REMOVED: The string-keys flag is no longer used.") - } - // Validate arguments. // Index and field are validated early before the files are parsed. if cmd.Index == "" { From 81f126dd34fa804650e5f7b1fbf26c52776d9a67 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 6 Sep 2018 17:04:07 +0300 Subject: [PATCH 11/65] Added field options to pilosa import --- cmd/import.go | 9 +++++++-- cmd/import_test.go | 16 ++++++++++++++++ ctl/import.go | 26 +++++++++++--------------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 3d63586ce..7cb6c34c3 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -51,8 +51,13 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.IndexKeys, "index-keys", false, "use keys=true when creating an index") - flags.BoolVar(&Importer.FieldKeys, "field-keys", false, "use keys=true when creating a field") + flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "specify keys=true when creating an index") + flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "specify keys=true when creating a field") + flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "specify the minimum for an int field on creation") + flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "specify the maximum for an int field on creation") + flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", "", "specify the cache type for a set field on creation") + flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 0, "specify the cache size for a set field on creation") + flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "specify the time quantum for a time field on creation") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/cmd/import_test.go b/cmd/import_test.go index e8e5d61f4..9a68a1218 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -18,6 +18,8 @@ import ( "strings" "testing" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/cmd" ) @@ -47,6 +49,20 @@ field = "f1" return v.Error() }, }, + { + args: []string{"import", "--index", "i1", "--field", "f1", "--field-keys", "--field-max", "100"}, + env: map[string]string{}, + validation: func() error { + v := validator{} + v.Check(cmd.Importer.Index, "i1") + v.Check(cmd.Importer.Field, "f1") + v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{ + Keys: true, + Max: 100, + }) + return v.Error() + }, + }, } executeDry(t, tests) } diff --git a/ctl/import.go b/ctl/import.go index 1c000dd73..d2694a64f 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -41,20 +41,14 @@ type ImportCommand struct { // nolint: maligned Field string `json:"field"` // Options for the index to be created if it doesn't exist - indexOptions pilosa.IndexOptions + IndexOptions pilosa.IndexOptions // Options for the field to be created if it doesn't exist - fieldOptions pilosa.FieldOptions + FieldOptions pilosa.FieldOptions // CreateSchema ensures the schema exists before import CreateSchema bool - // IndexKeys makes the import command use keys=true when creating an index - IndexKeys bool `json:"indexKeys"` - - // FieldKeys makes the import command use keys=true when creating a field - FieldKeys bool `json:"fieldKeys"` - // Filenames to import from. Paths []string `json:"paths"` @@ -102,11 +96,13 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { cmd.client = client if cmd.CreateSchema { - cmd.indexOptions = pilosa.IndexOptions{ - Keys: cmd.IndexKeys, - } - cmd.fieldOptions = pilosa.FieldOptions{ - Keys: cmd.FieldKeys, + // set the correct type for the field + if cmd.FieldOptions.TimeQuantum != "" { + cmd.FieldOptions.Type = "time" + } else if cmd.FieldOptions.Min != 0 || cmd.FieldOptions.Max != 0 { + cmd.FieldOptions.Type = "int" + } else { + cmd.FieldOptions.Type = "set" } err := cmd.ensureSchema(ctx) if err != nil { @@ -148,11 +144,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { - err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.indexOptions) + err := cmd.client.EnsureIndex(ctx, cmd.Index, cmd.IndexOptions) if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.fieldOptions) + err = cmd.client.EnsureFieldWithOptions(ctx, cmd.Index, cmd.Field, cmd.FieldOptions) if err != nil { return fmt.Errorf("Error Creating Field: %s", err) } From a6f22eb8b3b4980d287a3295c7f340f21be925a4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 6 Sep 2018 09:43:21 -0500 Subject: [PATCH 12/65] replace *testing.T with interface testing.TB --- test/pilosa.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index f6ed907ef..e7111ad92 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -129,40 +129,40 @@ func (m *Command) Reopen() error { // MustCreateIndex uses this command's API to create an index and fails the test // if there is an error. -func (m *Command) MustCreateIndex(t *testing.T, name string, opts pilosa.IndexOptions) *pilosa.Index { +func (m *Command) MustCreateIndex(tb testing.TB, name string, opts pilosa.IndexOptions) *pilosa.Index { idx, err := m.API.CreateIndex(context.Background(), name, opts) if err != nil { - t.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err) + tb.Fatalf("creating index: %v with options: %v, err: %v", name, opts, err) } return idx } // MustCreateField uses this command's API to create the field. The index must // already exist - it fails the test if there is an error. -func (m *Command) MustCreateField(t *testing.T, index, field string, opts ...pilosa.FieldOption) *pilosa.Field { +func (m *Command) MustCreateField(tb testing.TB, index, field string, opts ...pilosa.FieldOption) *pilosa.Field { f, err := m.API.CreateField(context.Background(), index, field, opts...) if err != nil { - t.Fatalf("creating field: %s in index: %s err: %v", field, index, err) + tb.Fatalf("creating field: %s in index: %s err: %v", field, index, err) } return f } // MustQuery uses this command's API to execute the given query request, failing // if Query returns a non-nil error, otherwise returning the QueryResponse. -func (m *Command) MustQuery(t *testing.T, req *pilosa.QueryRequest) pilosa.QueryResponse { +func (m *Command) MustQuery(tb testing.TB, req *pilosa.QueryRequest) pilosa.QueryResponse { resp, err := m.API.Query(context.Background(), req) if err != nil { - t.Fatalf("making query: %v, err: %v", req, err) + tb.Fatalf("making query: %v, err: %v", req, err) } return resp } // MustRecalculateCaches calls RecalculateCaches on the command's API, and fails // if there is an error. -func (m *Command) MustRecalculateCaches(t *testing.T) { +func (m *Command) MustRecalculateCaches(tb testing.TB) { err := m.API.RecalculateCaches(context.Background()) if err != nil { - t.Fatalf("recalcluating caches: %v", err) + tb.Fatalf("recalcluating caches: %v", err) } } @@ -224,10 +224,10 @@ func (c Cluster) Close() error { } // MustNewCluster creates a new cluster -func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { +func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { c, err := newCluster(size, opts...) if err != nil { - t.Fatalf("new cluster: %v", err) + tb.Fatalf("new cluster: %v", err) } return c } @@ -271,10 +271,10 @@ func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) { } // MustRunCluster creates and starts a new cluster -func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster { +func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { c, err := runCluster(size, opts...) if err != nil { - t.Fatalf("run cluster: %v", err) + tb.Fatalf("run cluster: %v", err) } return c } From 1ad5bf20d2f46f16b424cc2ceaf2262b2def206b Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 6 Sep 2018 23:19:10 +0300 Subject: [PATCH 13/65] Adds more tests; better help text; updated defaults for cache type, size --- cmd/import.go | 19 +++++++++---------- cmd/import_test.go | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 7cb6c34c3..7bfca1a84 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -18,6 +18,8 @@ import ( "context" "io" + "github.com/pilosa/pilosa" + "github.com/spf13/cobra" "github.com/pilosa/pilosa/ctl" @@ -51,19 +53,16 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") - flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "specify keys=true when creating an index") - flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "specify keys=true when creating a field") - flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "specify the minimum for an int field on creation") - flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "specify the maximum for an int field on creation") - flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", "", "specify the cache type for a set field on creation") - flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 0, "specify the cache size for a set field on creation") - flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "specify the time quantum for a time field on creation") + flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index") + flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field") + flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation") + flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation") + flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked") + flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation") + flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - //flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field") - //flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked") - //flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) return importCmd diff --git a/cmd/import_test.go b/cmd/import_test.go index 9a68a1218..ba015dc04 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -50,15 +50,47 @@ field = "f1" }, }, { - args: []string{"import", "--index", "i1", "--field", "f1", "--field-keys", "--field-max", "100"}, + args: []string{"import", "--index", "i1", "--field", "f1", "--field-keys", "--field-min", "-10", "--field-max", "100"}, env: map[string]string{}, validation: func() error { v := validator{} v.Check(cmd.Importer.Index, "i1") v.Check(cmd.Importer.Field, "f1") v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{ - Keys: true, - Max: 100, + Keys: true, + Max: 100, + Min: -10, + CacheType: pilosa.CacheTypeRanked, + CacheSize: 50000, + }) + return v.Error() + }, + }, + { + args: []string{"import", "--index", "i1", "--field", "f1", "--field-time-quantum", "YMD"}, + env: map[string]string{}, + validation: func() error { + v := validator{} + v.Check(cmd.Importer.Index, "i1") + v.Check(cmd.Importer.Field, "f1") + v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{ + TimeQuantum: "YMD", + CacheType: pilosa.CacheTypeRanked, + CacheSize: 50000, + }) + return v.Error() + }, + }, + { + args: []string{"import", "--index", "i1", "--field", "f1", "--field-cache-type", "lru", "--field-cache-size", "100"}, + env: map[string]string{}, + validation: func() error { + v := validator{} + v.Check(cmd.Importer.Index, "i1") + v.Check(cmd.Importer.Field, "f1") + v.Check(cmd.Importer.FieldOptions, pilosa.FieldOptions{ + CacheType: "lru", + CacheSize: 100, }) return v.Error() }, From 93e5767325ab83e14484082f4d71c8c7b3a19d7d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 6 Sep 2018 16:27:10 -0500 Subject: [PATCH 14/65] cleanup #1622 --- api.go | 6 +++--- client.go | 4 ++-- http/client.go | 7 +------ http/handler.go | 3 +-- roaring/roaring.go | 4 ++-- roaring/roaring_internal_test.go | 10 +++++----- .../{ => testdata}/bitmapcontainer.roaringbitmap | Bin 7 files changed, 14 insertions(+), 20 deletions(-) rename roaring/{ => testdata}/bitmapcontainer.roaringbitmap (100%) diff --git a/api.go b/api.go index 77ce4d0a0..ffe85dd37 100644 --- a/api.go +++ b/api.go @@ -296,7 +296,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e } // ImportRoaringBytes fast loading of standard roaring format -func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64, forward bool) (err error) { +func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64 ) (err error) { if err = api.validate(apiField); err != nil { err = errors.Wrap(err, "validating api method") return @@ -316,12 +316,12 @@ func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, ind err = field.importRoaringBytes(roaringBytes, shard) wg.Done() }(node) - } else if forward { + } else { wg.Add(1) //forward it on go func(node *Node) { //execute on node - err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes, forward) + err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes) wg.Done() }(node) } diff --git a/client.go b/client.go index d0e36db6d..d6f1f959f 100644 --- a/client.go +++ b/client.go @@ -51,7 +51,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error + ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error } //=============== @@ -107,7 +107,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { +func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/http/client.go b/http/client.go index 91fd7b571..7cc941c16 100644 --- a/http/client.go +++ b/http/client.go @@ -522,7 +522,7 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui } // ImportRoaringBytes fast import of raw bits in roaring standard format -func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { +func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -533,11 +533,6 @@ func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.No // Create URL. u := nodePathToURL(node, endpoint) - if forward { - v := url.Values{} - v.Set("noforward", "y") - u.RawQuery = v.Encode() - } // Generate HTTP request. req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes)) if err != nil { diff --git a/http/handler.go b/http/handler.go index f29128bd2..f86549c3c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1444,7 +1444,6 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] shardName := mux.Vars(r)["shard"] - _, noForward := r.URL.Query()["noforward"] // Read entire body. body, err := ioutil.ReadAll(r.Body) @@ -1459,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard, !noForward) + err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard ) // Marshal response object. msg := string("") diff --git a/roaring/roaring.go b/roaring/roaring.go index 853fcade9..abf36c40e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3340,7 +3340,7 @@ func popcountAndSlice(s, m []uint64) uint64 { return cnt } -// Stuff i needed from the roaring project +// constants from github.com/RoaringBitmap/roaring const ( //taken from roaring/util.go serialCookieNoRunContainer = 12346 // only arrays and bitmaps serialCookie = 12347 // runs, arrays, and bitmaps @@ -3404,7 +3404,7 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) return } - pos += 2 * 2 * int(size) //moving pos past keycount + pos += 2 * 2 * int(size) // moving pos past keycount return } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 581a8040f..2469657d1 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3230,8 +3230,8 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { func TestUnmarshalStdRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays - _2arrayContainer, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") - bm, er := UnmarshalStandardRoaring(_2arrayContainer) + rbContainerWithTwoArrays, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") + bm, er := UnmarshalStandardRoaring(rbContainerWithTwoArrays) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3239,8 +3239,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 65537]", bm.Slice()) } //generated serialize image from java(clojure) with a run and array - _rle_array_container, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - bm, er = UnmarshalStandardRoaring(_rle_array_container) + rbContainerWithRLEandArray, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + bm, er = UnmarshalStandardRoaring(rbContainerWithRLEandArray) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3248,7 +3248,7 @@ func TestUnmarshalStdRoaring(t *testing.T) { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) } //had to use an external file because emacs was barfing on the long line :() - _bitmap_array_container, _ := ioutil.ReadFile("./bitmapcontainer.roaringbitmap") + _bitmap_array_container, _ := ioutil.ReadFile("testdata/bitmapcontainer.roaringbitmap") bm, er = UnmarshalStandardRoaring(_bitmap_array_container) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) diff --git a/roaring/bitmapcontainer.roaringbitmap b/roaring/testdata/bitmapcontainer.roaringbitmap similarity index 100% rename from roaring/bitmapcontainer.roaringbitmap rename to roaring/testdata/bitmapcontainer.roaringbitmap From aa618c30d5797fcf2627352102eb34d143d35f1a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 10 Sep 2018 17:29:55 +0300 Subject: [PATCH 15/65] Implements Query call excludeRowAttrs, excludeColumns and shards args --- executor.go | 38 +++++++++++++++++++++ executor_test.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++ pql/pqlpeg_test.go | 17 ++++++++++ 3 files changed, 140 insertions(+) diff --git a/executor.go b/executor.go index 6f17cd6a7..5dfee2ea6 100644 --- a/executor.go +++ b/executor.go @@ -194,6 +194,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) + case "Query": + return e.executeQueryCall(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeBitmapCall(ctx, index, c, shards, opt) @@ -219,6 +221,42 @@ func (e *executor) validateCallArgs(c *pql.Call) error { return nil } +func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { + optCopy := &execOptions{} + *optCopy = *opt + if arg, ok := c.Args["excludeRowAttrs"]; ok { + if value, ok := arg.(bool); ok { + optCopy.ExcludeRowAttrs = value + } else { + return nil, errors.New("Query(): excludeRowAttrs must be a bool") + } + } + if arg, ok := c.Args["excludeColumns"]; ok { + if value, ok := arg.(bool); ok { + optCopy.ExcludeColumns = value + } else { + return nil, errors.New("Query(): excludeColumns must be a bool") + } + } + if arg, ok := c.Args["shards"]; ok { + if optShards, ok := arg.([]interface{}); ok { + shards = []uint64{} + for _, s := range optShards { + if shard, ok := s.(int64); ok { + shards = append(shards, uint64(shard)) + } else { + return nil, errors.New("Query(): shards must be a list of unsigned integers") + } + + } + } else { + return nil, errors.New("Query(): shards must be a list of unsigned integers") + } + + } + return e.executeCall(ctx, index, c.Children[0], shards, optCopy) +} + // executeSum executes a Sum() call. func (e *executor) executeSum(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (ValCount, error) { if field := c.Args["field"]; field == "" { diff --git a/executor_test.go b/executor_test.go index a04b5dba2..38fa9188b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1352,3 +1352,88 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { } } + +func TestExecutor_QueryCall(t *testing.T) { + t.Run("excludeRowAttrs", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar") + `}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), excludeRowAttrs=true)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + t.Fatalf("unexpected columns: %+v", bits) + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + + t.Run("excludeColumns", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar") + `}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), excludeColumns=true)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + + t.Run("shards", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` + Set(100, f=10) + Set(%d, f=10) + Set(%d, f=10) + `, ShardWidth, ShardWidth*2)}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), shards=[0, 2])`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) +} diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index fdcde382d..6f05fb402 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -595,6 +595,23 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Row"}, }, }}, + { + name: "QueryWrapper", + call: "Query(Row(f1=123), excludeRowAttrs=true)", + exp: &Call{ + Name: "Query", + Args: map[string]interface{}{ + "excludeRowAttrs": true, + }, + Children: []*Call{ + { + Name: "Row", + Args: map[string]interface{}{ + "f1": int64(123), + }, + }, + }, + }}, } for i, test := range tests { From 70ceb5809bd03a362d7714f885182e313b04b8a2 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 11:42:00 +0300 Subject: [PATCH 16/65] Added columnAttrs to Query call --- api.go | 8 ++++--- executor.go | 10 +++++++++ executor_test.go | 56 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/api.go b/api.go index fb64b5bdc..e137266c4 100644 --- a/api.go +++ b/api.go @@ -109,8 +109,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } execOpts := &execOptions{ Remote: req.Remote, - ExcludeRowAttrs: req.ExcludeRowAttrs, - ExcludeColumns: req.ExcludeColumns, + ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat. + ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. + ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. } results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { @@ -119,7 +120,8 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er resp.Results = results // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeColumns { + // execOpts.ColumnAttrs and execOpts.ExcludeColumns are out params from api.server.executor.Execute + if execOpts.ColumnAttrs && !execOpts.ExcludeColumns { // Consolidate all column ids across all calls. var columnIDs []uint64 for _, result := range results { diff --git a/executor.go b/executor.go index 5dfee2ea6..a777f7332 100644 --- a/executor.go +++ b/executor.go @@ -224,9 +224,17 @@ func (e *executor) validateCallArgs(c *pql.Call) error { func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { optCopy := &execOptions{} *optCopy = *opt + if arg, ok := c.Args["columnAttrs"]; ok { + if value, ok := arg.(bool); ok { + opt.ColumnAttrs = value + } else { + return nil, errors.New("Query(): columnAttrs must be a bool") + } + } if arg, ok := c.Args["excludeRowAttrs"]; ok { if value, ok := arg.(bool); ok { optCopy.ExcludeRowAttrs = value + opt.ExcludeRowAttrs = value } else { return nil, errors.New("Query(): excludeRowAttrs must be a bool") } @@ -234,6 +242,7 @@ func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Ca if arg, ok := c.Args["excludeColumns"]; ok { if value, ok := arg.(bool); ok { optCopy.ExcludeColumns = value + opt.ExcludeColumns = value } else { return nil, errors.New("Query(): excludeColumns must be a bool") } @@ -1715,6 +1724,7 @@ type execOptions struct { Remote bool ExcludeRowAttrs bool ExcludeColumns bool + ColumnAttrs bool } // hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls. diff --git a/executor_test.go b/executor_test.go index 38fa9188b..6372ecf74 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1364,12 +1364,10 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar") + `}); err != nil { t.Fatal(err) } @@ -1392,12 +1390,10 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar") + `}); err != nil { t.Fatal(err) } @@ -1410,6 +1406,36 @@ func TestExecutor_QueryCall(t *testing.T) { } }) + t.Run("columnAttrs", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(100, f=10) + SetColumnAttrs(100, foo="bar") + `}); err != nil { + t.Fatal(err) + } + + targetColAttrSets := []*pilosa.ColumnAttrSet{ + &pilosa.ColumnAttrSet{ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), columnAttrs=true)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + t.Fatalf("unexpected columns: %+v", bits) + } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + t.Run("shards", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -1420,13 +1446,11 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` - Set(100, f=10) - Set(%d, f=10) - Set(%d, f=10) - `, ShardWidth, ShardWidth*2)}); err != nil { + Set(100, f=10) + Set(%d, f=10) + Set(%d, f=10) + `, ShardWidth, ShardWidth*2)}); err != nil { t.Fatal(err) } From 45abad5147b2796b726760968dcb22ce51d1c469 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 11:50:54 +0300 Subject: [PATCH 17/65] trivial --- executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 6372ecf74..82d9c3da8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1424,7 +1424,7 @@ func TestExecutor_QueryCall(t *testing.T) { } targetColAttrSets := []*pilosa.ColumnAttrSet{ - &pilosa.ColumnAttrSet{ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, + {ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), columnAttrs=true)`}); err != nil { From 465028e9c3bad921f477591cb47f2ae778b1dee5 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 17:19:12 +0300 Subject: [PATCH 18/65] Renamed Query to Opt; added multiple Opt test --- api.go | 2 +- executor.go | 9 +++------ executor_test.go | 42 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index e137266c4..c3d64a63d 100644 --- a/api.go +++ b/api.go @@ -121,7 +121,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er // Fill column attributes if requested. // execOpts.ColumnAttrs and execOpts.ExcludeColumns are out params from api.server.executor.Execute - if execOpts.ColumnAttrs && !execOpts.ExcludeColumns { + if execOpts.ColumnAttrs { // Consolidate all column ids across all calls. var columnIDs []uint64 for _, result := range results { diff --git a/executor.go b/executor.go index a777f7332..f95ae101d 100644 --- a/executor.go +++ b/executor.go @@ -194,8 +194,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) - case "Query": - return e.executeQueryCall(ctx, index, c, shards, opt) + case "Opt": + return e.executeOptCall(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeBitmapCall(ctx, index, c, shards, opt) @@ -221,7 +221,7 @@ func (e *executor) validateCallArgs(c *pql.Call) error { return nil } -func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeOptCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { optCopy := &execOptions{} *optCopy = *opt if arg, ok := c.Args["columnAttrs"]; ok { @@ -234,7 +234,6 @@ func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Ca if arg, ok := c.Args["excludeRowAttrs"]; ok { if value, ok := arg.(bool); ok { optCopy.ExcludeRowAttrs = value - opt.ExcludeRowAttrs = value } else { return nil, errors.New("Query(): excludeRowAttrs must be a bool") } @@ -242,7 +241,6 @@ func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Ca if arg, ok := c.Args["excludeColumns"]; ok { if value, ok := arg.(bool); ok { optCopy.ExcludeColumns = value - opt.ExcludeColumns = value } else { return nil, errors.New("Query(): excludeColumns must be a bool") } @@ -261,7 +259,6 @@ func (e *executor) executeQueryCall(ctx context.Context, index string, c *pql.Ca } else { return nil, errors.New("Query(): shards must be a list of unsigned integers") } - } return e.executeCall(ctx, index, c.Children[0], shards, optCopy) } diff --git a/executor_test.go b/executor_test.go index 82d9c3da8..b6ed42520 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1371,7 +1371,7 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), excludeRowAttrs=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), excludeRowAttrs=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1397,7 +1397,7 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), excludeColumns=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), excludeColumns=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1427,7 +1427,7 @@ func TestExecutor_QueryCall(t *testing.T) { {ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), columnAttrs=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), columnAttrs=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1454,10 +1454,44 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Query(Row(f=10), shards=[0, 2])`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), shards=[0, 2])`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { t.Fatalf("unexpected columns: %+v", bits) } }) + + t.Run("multipleOpt", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar") + `}); err != nil { + t.Fatal(err) + } + + req := &pilosa.QueryRequest{ + Index: "i", + Query: `Opt(Row(f=10), excludeColumns=true)Opt(Row(f=10), excludeRowAttrs=true)`, + } + if res, err := c[0].API.Query(context.Background(), req); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } else if bits := res.Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + t.Fatalf("unexpected columns: %+v", bits) + } else if attrs := res.Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) } From c2d23b1f00d222710368da9f93bc3b0a2e843915 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 20:42:06 +0300 Subject: [PATCH 19/65] Renamed Opt call to Options --- executor.go | 6 +++--- executor_test.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/executor.go b/executor.go index f95ae101d..2888b9270 100644 --- a/executor.go +++ b/executor.go @@ -194,8 +194,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) - case "Opt": - return e.executeOptCall(ctx, index, c, shards, opt) + case "Options": + return e.executeOptionsCall(ctx, index, c, shards, opt) default: e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeBitmapCall(ctx, index, c, shards, opt) @@ -221,7 +221,7 @@ func (e *executor) validateCallArgs(c *pql.Call) error { return nil } -func (e *executor) executeOptCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { +func (e *executor) executeOptionsCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { optCopy := &execOptions{} *optCopy = *opt if arg, ok := c.Args["columnAttrs"]; ok { diff --git a/executor_test.go b/executor_test.go index b6ed42520..b6fa5ccfa 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1371,7 +1371,7 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), excludeRowAttrs=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeRowAttrs=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1397,7 +1397,7 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), excludeColumns=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeColumns=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1427,7 +1427,7 @@ func TestExecutor_QueryCall(t *testing.T) { {ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), columnAttrs=true)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), columnAttrs=true)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1454,7 +1454,7 @@ func TestExecutor_QueryCall(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Opt(Row(f=10), shards=[0, 2])`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), shards=[0, 2])`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { t.Fatalf("unexpected columns: %+v", bits) @@ -1480,7 +1480,7 @@ func TestExecutor_QueryCall(t *testing.T) { req := &pilosa.QueryRequest{ Index: "i", - Query: `Opt(Row(f=10), excludeColumns=true)Opt(Row(f=10), excludeRowAttrs=true)`, + Query: `Options(Row(f=10), excludeColumns=true)Options(Row(f=10), excludeRowAttrs=true)`, } if res, err := c[0].API.Query(context.Background(), req); err != nil { t.Fatal(err) From 70d386039f70302d0432f7625448829554a857de Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 20:47:15 +0300 Subject: [PATCH 20/65] trivial --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index c3d64a63d..831b9c32f 100644 --- a/api.go +++ b/api.go @@ -120,7 +120,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er resp.Results = results // Fill column attributes if requested. - // execOpts.ColumnAttrs and execOpts.ExcludeColumns are out params from api.server.executor.Execute + // execOpts.ColumnAttrs is an out param from api.server.executor.Execute if execOpts.ColumnAttrs { // Consolidate all column ids across all calls. var columnIDs []uint64 From 80c1a734ba251e6fd328a7a1dcc5c8342a149f57 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 20:51:32 +0300 Subject: [PATCH 21/65] trivial --- pql/pqlpeg_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 6f05fb402..2025120e9 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -596,10 +596,10 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { - name: "QueryWrapper", - call: "Query(Row(f1=123), excludeRowAttrs=true)", + name: "OptionsWrapper", + call: "Options(Row(f1=123), excludeRowAttrs=true)", exp: &Call{ - Name: "Query", + Name: "Options", Args: map[string]interface{}{ "excludeRowAttrs": true, }, From 408a3531ec547f05e4c1886e24915cfa2982937e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 11 Sep 2018 22:18:31 +0300 Subject: [PATCH 22/65] updated comment --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 831b9c32f..9e6ee1daa 100644 --- a/api.go +++ b/api.go @@ -120,7 +120,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er resp.Results = results // Fill column attributes if requested. - // execOpts.ColumnAttrs is an out param from api.server.executor.Execute + // execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true) if execOpts.ColumnAttrs { // Consolidate all column ids across all calls. var columnIDs []uint64 From 975e013deb1260cb319240d0d29462ce17a36b9b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 11 Sep 2018 14:38:04 -0500 Subject: [PATCH 23/65] Increase gometalinter deadline so CI stops failing --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 390a652fc..b403fde23 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ docker-test: # Run gometalinter with custom flags gometalinter: require-gometalinter gometalinter --vendor --disable-all \ - --deadline=120s \ + --deadline=300s \ --enable=deadcode \ --enable=gochecknoinits \ --enable=gofmt \ From d57dae3749973fc5c95a333f98cb2658e9c6e2d4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 6 Sep 2018 11:32:50 -0500 Subject: [PATCH 24/65] implement NotNull field with index option trackNotNull --- encoding/proto/proto.go | 4 +- executor.go | 7 ++ executor_test.go | 29 +++++++ holder.go | 4 + index.go | 37 +++++++- internal/private.pb.go | 182 ++++++++++++++++++++++++---------------- internal/private.proto | 1 + 7 files changed, 190 insertions(+), 74 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index a19f1e548..c00e71b40 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -509,7 +509,8 @@ func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateInde func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { return &internal.IndexMeta{ - Keys: m.Keys, + Keys: m.Keys, + TrackNotNull: m.TrackNotNull, } } @@ -741,6 +742,7 @@ func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateI func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { m.Keys = pb.Keys + m.TrackNotNull = pb.TrackNotNull } func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { diff --git a/executor.go b/executor.go index 2888b9270..1149f8c0e 100644 --- a/executor.go +++ b/executor.go @@ -1131,6 +1131,13 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op return false, ErrFieldNotFound } + // Set column on not-null field. + if nnf := idx.unprotectedNotNullField(); nnf != nil { + if _, err := nnf.SetBit(0, colID, nil); err != nil { + return false, errors.Wrap(err, "setting not-null column") + } + } + if f.Type() == FieldTypeInt { // Read remaining fields using labels. rowVal, ok, err := c.IntArg(fieldName) diff --git a/executor_test.go b/executor_test.go index b6fa5ccfa..03ac09102 100644 --- a/executor_test.go +++ b/executor_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "context" "fmt" + "math/rand" "reflect" "strconv" "strings" @@ -1495,3 +1496,31 @@ func TestExecutor_QueryCall(t *testing.T) { } }) } + +func benchmarkNotNull(nn bool, b *testing.B) { + c := test.MustRunCluster(b, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + indexName := "i" + fieldName := "f" + + index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackNotNull: nn}) + // Create field. + if _, err := index.CreateFieldIfNotExists(fieldName); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + colID := uint64(rand.Intn(1 << 20)) + rowID := uint64(rand.Intn(100000)) + qry := fmt.Sprintf(`Set(%d, %s=%d)`, colID, fieldName, rowID) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: qry}); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkExecutor_NotNull_True(b *testing.B) { benchmarkNotNull(true, b) } +func BenchmarkExecutor_NotNull_False(b *testing.B) { benchmarkNotNull(false, b) } diff --git a/holder.go b/holder.go index 957765511..252fca2db 100644 --- a/holder.go +++ b/holder.go @@ -38,6 +38,9 @@ const ( // fileLimit is the maximum open file limit (ulimit -n) to automatically set. fileLimit = 262144 // (512^2) + + // notNullFieldName is the name of the internal field used to store not-null values. + notNullFieldName = "notnull" // TODO: use a name less likely to collide? ) // Holder represents a container for indexes. @@ -354,6 +357,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } index.keys = opt.Keys + index.trackNotNull = opt.TrackNotNull if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") diff --git a/index.go b/index.go index 0547d07d2..adbc6810e 100644 --- a/index.go +++ b/index.go @@ -36,6 +36,10 @@ type Index struct { name string keys bool // use string keys + // Not-null tracking. + trackNotNull bool + notNullField *Field + // Fields by name. fields map[string]*Field @@ -91,7 +95,10 @@ func (i *Index) Options() IndexOptions { } func (i *Index) options() IndexOptions { - return IndexOptions{Keys: i.keys} + return IndexOptions{ + Keys: i.keys, + TrackNotNull: i.trackNotNull, + } } // Open opens and initializes the index. @@ -110,6 +117,12 @@ func (i *Index) Open() error { return errors.Wrap(err, "opening fields") } + if i.trackNotNull { + if err := i.openNotNullField(); err != nil { + return errors.Wrap(err, "opening not-null field") + } + } + if err := i.columnAttrs.Open(); err != nil { return errors.Wrap(err, "opening attrstore") } @@ -147,6 +160,16 @@ func (i *Index) openFields() error { return nil } +// openNotNullField gets or creates the not-null field and associates it to the index. +func (i *Index) openNotNullField() error { + f, err := i.createFieldIfNotExists(notNullFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0}) + if err != nil { + return errors.Wrap(err, "creating not-null field") + } + i.notNullField = f + return nil +} + // loadMeta reads meta data for the index, if any. func (i *Index) loadMeta() error { var pb internal.IndexMeta @@ -165,6 +188,7 @@ func (i *Index) loadMeta() error { // Copy metadata fields. i.keys = pb.Keys + i.trackNotNull = pb.TrackNotNull return nil } @@ -173,7 +197,8 @@ func (i *Index) loadMeta() error { func (i *Index) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.IndexMeta{ - Keys: i.keys, + Keys: i.keys, + TrackNotNull: i.trackNotNull, }) if err != nil { return errors.Wrap(err, "marshalling") @@ -250,6 +275,11 @@ func (i *Index) Fields() []*Field { return a } +// unprotectedNotNullField returns the internal field used to track not-null columns. +func (i *Index) unprotectedNotNullField() *Field { + return i.notNullField +} + // recalculateCaches recalculates caches on every field in the index. func (i *Index) recalculateCaches() { for _, field := range i.Fields() { @@ -408,7 +438,8 @@ func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // IndexOptions represents options to set when initializing an index. type IndexOptions struct { - Keys bool `json:"keys"` + Keys bool `json:"keys"` + TrackNotNull bool `json:"trackNotNull"` } // hasTime returns true if a contains a non-nil time. diff --git a/internal/private.pb.go b/internal/private.pb.go index 6247c58d4..2d1c0deef 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -62,7 +62,8 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` + TrackNotNull bool `protobuf:"varint,4,opt,name=TrackNotNull,proto3" json:"TrackNotNull,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -77,6 +78,13 @@ func (m *IndexMeta) GetKeys() bool { return false } +func (m *IndexMeta) GetTrackNotNull() bool { + if m != nil { + return m.TrackNotNull + } + return false +} + type FieldOptions struct { Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -1051,6 +1059,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.TrackNotNull { + dAtA[i] = 0x20 + i++ + if m.TrackNotNull { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -2300,6 +2318,9 @@ func (m *IndexMeta) Size() (n int) { if m.Keys { n += 2 } + if m.TrackNotNull { + n += 2 + } return n } @@ -2902,6 +2923,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } } m.Keys = bool(v != 0) + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TrackNotNull", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.TrackNotNull = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7143,73 +7184,74 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1077 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0x66, 0x7f, 0xec, 0xda, 0xc7, 0x75, 0x9a, 0x6c, 0x69, 0xd8, 0x22, 0x94, 0x9a, 0x51, 0xa5, - 0x9a, 0x4a, 0x84, 0xaa, 0xbd, 0xe1, 0xaf, 0x52, 0x49, 0x1c, 0x60, 0x29, 0x09, 0x65, 0x36, 0xc9, - 0x5d, 0x2f, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0xb3, 0xec, 0xce, 0x26, 0x71, 0x2f, 0xb8, 0x05, - 0x89, 0x17, 0x40, 0x3c, 0x09, 0x8f, 0xc0, 0x25, 0x8f, 0x80, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xfb, - 0x13, 0xc7, 0x21, 0x55, 0xe0, 0x6e, 0xce, 0x77, 0xce, 0x9c, 0xf3, 0xed, 0xf9, 0x9b, 0x85, 0x7e, - 0x9a, 0x45, 0xc7, 0x4c, 0xf2, 0xf5, 0x34, 0x13, 0x52, 0x78, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0x58, - 0x4c, 0xee, 0x41, 0x37, 0x48, 0x26, 0xfc, 0x74, 0x9b, 0x4b, 0xe6, 0x79, 0xe0, 0x3e, 0xe7, 0xb3, - 0xdc, 0x77, 0x06, 0xd6, 0xb0, 0x43, 0xf1, 0x4c, 0x7e, 0xb7, 0xe0, 0xe6, 0x97, 0x11, 0x8f, 0x27, - 0xdf, 0xa5, 0x32, 0x12, 0x49, 0xee, 0xbd, 0x07, 0xdd, 0x4d, 0x36, 0x3e, 0xe4, 0xbb, 0xb3, 0x94, - 0xa3, 0x65, 0x97, 0xd6, 0x40, 0xa5, 0x0d, 0xa3, 0xd7, 0xdc, 0x77, 0x07, 0xd6, 0xb0, 0x4f, 0x6b, - 0xc0, 0x1b, 0x40, 0x6f, 0x37, 0x9a, 0xf2, 0xef, 0x0b, 0x96, 0xc8, 0x62, 0xea, 0xb7, 0xf0, 0x76, - 0x13, 0x52, 0x14, 0xd0, 0x71, 0x07, 0x55, 0x78, 0xf6, 0x96, 0xc1, 0xd9, 0x8e, 0x12, 0xbf, 0x3b, - 0xb0, 0x86, 0x0e, 0x55, 0x47, 0x44, 0xd8, 0xa9, 0x0f, 0x06, 0x61, 0xa7, 0x15, 0xf5, 0x5e, 0x83, - 0x3a, 0x81, 0xa5, 0x60, 0x9a, 0x8a, 0x4c, 0x52, 0x9e, 0xa7, 0x22, 0xc9, 0xd1, 0xd3, 0x56, 0x96, - 0xf9, 0x16, 0x3a, 0x57, 0x47, 0xf2, 0x23, 0x2c, 0x6f, 0xc4, 0x62, 0x7c, 0x34, 0x62, 0x92, 0x51, - 0xfe, 0x43, 0xc1, 0x73, 0xe9, 0xbd, 0x0d, 0x2d, 0xcc, 0x89, 0xb1, 0xd3, 0x82, 0x42, 0x31, 0x0f, - 0xbe, 0xad, 0x51, 0x14, 0x14, 0x8a, 0xf7, 0x31, 0x13, 0x2e, 0xd5, 0x82, 0x42, 0xc3, 0x43, 0x96, - 0x4d, 0x30, 0x03, 0x2e, 0xd5, 0x82, 0xe2, 0xb8, 0x1f, 0xf1, 0x13, 0xf3, 0xd9, 0x78, 0x26, 0x01, - 0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x24, 0x18, 0xe5, 0xbe, 0x35, 0x70, 0x86, - 0x2e, 0x35, 0x12, 0x26, 0x57, 0xc4, 0xc5, 0x34, 0x51, 0x2a, 0x1b, 0x55, 0x35, 0x40, 0xee, 0x42, - 0x0b, 0x33, 0xad, 0xbe, 0xb2, 0xbe, 0xab, 0x8e, 0xe4, 0x27, 0x0b, 0xba, 0xdb, 0xec, 0x14, 0x69, - 0xe4, 0xde, 0x53, 0xe8, 0x84, 0x92, 0x25, 0x13, 0x45, 0x50, 0x19, 0xf5, 0x1e, 0xbf, 0xbf, 0x5e, - 0x36, 0xc4, 0x7a, 0x65, 0xb6, 0x5e, 0xda, 0x6c, 0x25, 0x32, 0x9b, 0xd1, 0xea, 0xca, 0xbb, 0x9f, - 0x41, 0xff, 0x9c, 0x4a, 0xc5, 0x3b, 0xe2, 0xb3, 0x32, 0xab, 0x47, 0x7c, 0xa6, 0xbe, 0xff, 0x98, - 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xda, 0x1f, 0x5b, 0x64, 0x1f, 0xbc, 0xcd, 0x8c, - 0x33, 0xc9, 0x31, 0xc8, 0x36, 0xcf, 0x73, 0xf6, 0x8a, 0x5f, 0x9e, 0x71, 0x9d, 0x45, 0xbb, 0x99, - 0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x21, 0x78, 0x23, 0x1e, 0x73, 0xc9, 0x4d, 0x37, 0xff, - 0x8b, 0x5f, 0x12, 0x96, 0x1c, 0xae, 0xb6, 0xf5, 0x1e, 0x80, 0xab, 0x46, 0x03, 0x29, 0xf4, 0x1e, - 0xdf, 0xae, 0xf3, 0x54, 0x4d, 0x0d, 0x45, 0x03, 0x12, 0x97, 0x4e, 0x91, 0xcf, 0x95, 0x1f, 0xb6, - 0xa0, 0x95, 0x1e, 0x9a, 0x50, 0x0e, 0x86, 0x5a, 0xad, 0x43, 0x35, 0xc7, 0xcf, 0x44, 0x7b, 0x56, - 0x7e, 0xee, 0x75, 0xa3, 0x91, 0x97, 0x06, 0x55, 0x5d, 0xb9, 0xc3, 0xa6, 0xdc, 0xdc, 0xc1, 0x73, - 0x45, 0xc5, 0xbe, 0x9a, 0x8a, 0x72, 0xaf, 0x3a, 0x59, 0x6d, 0x0d, 0x47, 0xb9, 0x47, 0x81, 0x3c, - 0x81, 0x76, 0x38, 0x3e, 0xe4, 0x53, 0xe6, 0x7d, 0x00, 0x37, 0x90, 0x07, 0xcf, 0x4d, 0xb3, 0xdd, - 0x9a, 0x4b, 0x22, 0x2d, 0xf5, 0x64, 0x64, 0xf8, 0x2f, 0xe4, 0xf4, 0x00, 0xda, 0x18, 0x3d, 0xf7, - 0xdd, 0x79, 0x37, 0x88, 0x53, 0xa3, 0x26, 0x5b, 0xe0, 0xec, 0xd1, 0x40, 0x0d, 0x11, 0x32, 0x28, - 0xbd, 0x18, 0x49, 0xf9, 0xfe, 0x5a, 0xe4, 0xd2, 0x64, 0x03, 0xcf, 0x0a, 0x7b, 0x21, 0x32, 0x89, - 0xa9, 0xef, 0x53, 0x3c, 0x93, 0x97, 0xe0, 0xee, 0x88, 0x09, 0xf7, 0x96, 0xc0, 0x0e, 0x46, 0xc6, - 0x87, 0x1d, 0x8c, 0xbc, 0x7b, 0xe8, 0xde, 0xa4, 0xa6, 0x5f, 0x93, 0xd8, 0xa3, 0x01, 0xc5, 0xc0, - 0xf7, 0xa1, 0x1f, 0xe4, 0x9b, 0x42, 0x64, 0x93, 0x28, 0x61, 0x52, 0x64, 0x66, 0x9d, 0x9e, 0x07, - 0xc9, 0x33, 0x58, 0x56, 0xee, 0x43, 0xc9, 0x24, 0x2f, 0xeb, 0xb7, 0x0a, 0x6d, 0x85, 0x55, 0xe1, - 0x8c, 0x84, 0x83, 0xa0, 0xec, 0xca, 0x0a, 0xa2, 0x40, 0xbe, 0xd5, 0x1e, 0xb6, 0x8e, 0x79, 0x22, - 0x1b, 0x1d, 0x80, 0x32, 0x3a, 0xe8, 0x53, 0x2d, 0x78, 0x44, 0x7f, 0x8a, 0xe1, 0xbc, 0x54, 0x73, - 0x56, 0x28, 0x45, 0x1d, 0xf9, 0xc5, 0x02, 0x28, 0x09, 0x15, 0x79, 0x75, 0xc5, 0xba, 0xfc, 0x8a, - 0x37, 0x2c, 0x6b, 0x6c, 0x5a, 0x76, 0xb9, 0xb6, 0xd2, 0x38, 0x2d, 0x7b, 0xe0, 0xa3, 0xba, 0x07, - 0x74, 0xf1, 0xee, 0xcc, 0xf5, 0x80, 0x8e, 0x5a, 0x77, 0xc2, 0x0b, 0xe8, 0x35, 0xf0, 0x85, 0xfd, - 0xf0, 0x61, 0xd5, 0x0f, 0xf6, 0xbc, 0x4b, 0xc4, 0x8d, 0xcb, 0xb2, 0x2b, 0x9e, 0x43, 0xaf, 0x01, - 0x2f, 0xf4, 0x38, 0x84, 0x5b, 0x5f, 0x1c, 0xb3, 0x28, 0x66, 0x07, 0xb1, 0x5e, 0x4f, 0xe5, 0x92, - 0x9d, 0x87, 0x49, 0x04, 0xfd, 0xcd, 0xb8, 0xc8, 0x25, 0xcf, 0x8c, 0x3b, 0xb5, 0x99, 0x35, 0x50, - 0x15, 0xaf, 0x06, 0x16, 0xd7, 0xcf, 0xbb, 0x0f, 0x2d, 0x95, 0x46, 0x3d, 0x38, 0x17, 0x73, 0xac, - 0x95, 0x64, 0x1f, 0x3a, 0x1b, 0x61, 0xf0, 0x55, 0x26, 0x8a, 0x74, 0x21, 0xe9, 0xf2, 0xc1, 0xb4, - 0x2f, 0x3e, 0x98, 0xce, 0x85, 0x07, 0xd3, 0xad, 0x1e, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a, - 0x5e, 0xaf, 0xb3, 0xae, 0xca, 0xd7, 0xcc, 0x69, 0xbc, 0x66, 0x21, 0xac, 0xe8, 0xb5, 0xf4, 0x7f, - 0x3a, 0xfd, 0xcd, 0x86, 0x15, 0xca, 0xf3, 0xe8, 0x35, 0x0f, 0x92, 0x5c, 0x66, 0xc5, 0x58, 0x6d, - 0x1f, 0x75, 0xff, 0x1b, 0x71, 0x60, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x49, 0xa7, 0x7b, 0x8f, 0xa0, - 0x37, 0x3f, 0x9d, 0x17, 0x4d, 0x9b, 0x26, 0xde, 0x23, 0xb8, 0x11, 0x8a, 0x22, 0x1b, 0x57, 0xed, - 0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x2b, 0x46, 0xe3, 0xe9, - 0x5c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x9c, 0x9a, 0x9e, 0xb7, 0x26, 0x3f, 0x5b, - 0x70, 0xb3, 0x49, 0xe1, 0x8d, 0x06, 0xb7, 0xaa, 0x88, 0xbd, 0xb0, 0x22, 0xce, 0xa2, 0x8a, 0xb8, - 0x75, 0x45, 0xea, 0x87, 0xb9, 0xd5, 0x78, 0x98, 0xc9, 0x11, 0xdc, 0xbd, 0x50, 0xa6, 0x4d, 0x31, - 0x4d, 0x55, 0x3f, 0xfc, 0x87, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2, - 0x09, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x01, 0x38, 0x3b, 0xfc, 0xe4, 0x92, 0xcf, - 0x57, 0x2a, 0xf2, 0x39, 0xf8, 0x7b, 0xe9, 0x84, 0x49, 0x7e, 0xad, 0xdb, 0x1b, 0xd0, 0xd9, 0x15, - 0xa9, 0x88, 0xc5, 0xab, 0xd9, 0x15, 0x53, 0xef, 0xc3, 0x0d, 0xbd, 0xbf, 0xf5, 0x1a, 0xe9, 0xd2, - 0x52, 0x24, 0xb7, 0x55, 0x43, 0x8f, 0x59, 0x3c, 0x2e, 0x62, 0x45, 0x43, 0xfd, 0xb4, 0xe5, 0x1b, - 0xcb, 0x7f, 0x9c, 0xad, 0x59, 0x7f, 0x9e, 0xad, 0x59, 0x7f, 0x9d, 0xad, 0x59, 0xbf, 0xfe, 0xbd, - 0xf6, 0xd6, 0x41, 0x1b, 0x7f, 0xd6, 0x9f, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x7c, 0x6d, - 0x22, 0xbd, 0x0b, 0x00, 0x00, + // 1093 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44, + 0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0x9b, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x95, + 0x1a, 0x2a, 0x11, 0xaa, 0xf6, 0x86, 0xbf, 0x4a, 0x25, 0xbb, 0x01, 0x4c, 0x49, 0x28, 0xe3, 0x24, + 0x77, 0xbd, 0x98, 0xec, 0x8e, 0x1a, 0x2b, 0x5e, 0x8f, 0xb1, 0xc7, 0x49, 0xb6, 0x17, 0xdc, 0x82, + 0xc4, 0x0b, 0x20, 0x9e, 0x84, 0x47, 0xe0, 0x92, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x9c, 0x19, 0xff, + 0x64, 0xb3, 0x21, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f, + 0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x66, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c, + 0x26, 0x03, 0xe8, 0x06, 0xc9, 0x98, 0x9f, 0xed, 0x70, 0xc9, 0x3c, 0x0f, 0xdc, 0x67, 0x7c, 0x9a, + 0xfb, 0xce, 0xba, 0xb5, 0xd1, 0xa1, 0x78, 0xf6, 0x08, 0x2c, 0xee, 0x65, 0x6c, 0x74, 0xbc, 0x2b, + 0xe4, 0x6e, 0x11, 0xc7, 0xbe, 0x8b, 0xba, 0x0b, 0x18, 0xf9, 0xc3, 0x82, 0xc5, 0xaf, 0x22, 0x1e, + 0x8f, 0xbf, 0x4f, 0x65, 0x24, 0x92, 0xdc, 0x7b, 0x0f, 0xba, 0x03, 0x36, 0x3a, 0xe2, 0x7b, 0xd3, + 0x94, 0xa3, 0xb7, 0x2e, 0xad, 0x81, 0x4a, 0x1b, 0x46, 0xaf, 0x38, 0xfa, 0xeb, 0xd3, 0x1a, 0xf0, + 0xd6, 0xa1, 0xb7, 0x17, 0x4d, 0xf8, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf1, 0x5b, 0x78, 0xbb, 0x09, + 0x29, 0x9a, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0xcb, 0xe0, 0xec, 0x44, 0x89, 0xdf, 0x5d, 0xb7, + 0x36, 0x1c, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0xe7, 0xf5, 0xea, 0xe7, + 0x11, 0x02, 0x4b, 0xc1, 0x24, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, 0xed, 0x2c, + 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xde, 0x8a, 0xc5, 0xe8, 0x78, 0xc8, 0x24, 0xa3, + 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x37, 0x63, 0xa7, 0x05, 0x85, 0x62, 0x1e, + 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, 0x47, 0x2c, + 0x1b, 0x63, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, 0x26, 0x01, + 0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe6, 0xbe, 0xb5, 0xee, 0x6c, + 0xb8, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0x93, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, 0xb9, 0x0b, + 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, 0x19, 0xd2, + 0xc8, 0xbd, 0x27, 0xd0, 0x09, 0x25, 0x4b, 0xc6, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfa, 0x60, 0xb3, + 0x6c, 0x9a, 0xcd, 0xca, 0x6c, 0xb3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa5, 0xd5, 0x95, 0x77, 0x3f, + 0x87, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0xa7, 0x65, 0x56, 0x8f, 0xf9, 0x54, 0xbd, 0xff, 0x84, + 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xd9, 0x9f, 0x58, 0xe4, 0x00, 0xbc, 0x41, 0xc6, + 0x99, 0xe4, 0x18, 0x64, 0x87, 0xe7, 0x39, 0x7b, 0xc9, 0xaf, 0xce, 0xb8, 0xce, 0xa2, 0xdd, 0xcc, + 0x62, 0x55, 0x07, 0xa7, 0x51, 0x07, 0xf2, 0x00, 0xbc, 0x21, 0x8f, 0xb9, 0xe4, 0xa6, 0xe3, 0xff, + 0xc3, 0x2f, 0x09, 0x4b, 0x0e, 0xd7, 0xdb, 0x7a, 0xf7, 0xc1, 0x55, 0xe3, 0x83, 0x14, 0x7a, 0x8f, + 0x6e, 0xd7, 0x79, 0xaa, 0x26, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0xe7, 0xda, 0x87, 0xcd, + 0x69, 0xa5, 0x07, 0x26, 0x94, 0x83, 0xa1, 0x56, 0xeb, 0x50, 0xcd, 0xf1, 0x33, 0xd1, 0x9e, 0x96, + 0xcf, 0xbd, 0x69, 0x34, 0xf2, 0xc2, 0xa0, 0xaa, 0x2b, 0x77, 0xd9, 0x84, 0x9b, 0x3b, 0x78, 0xae, + 0xa8, 0xd8, 0xd7, 0x53, 0x51, 0xee, 0x55, 0x27, 0xab, 0xcd, 0xe2, 0x28, 0xf7, 0x28, 0x90, 0xc7, + 0xd0, 0x0e, 0x47, 0x47, 0x7c, 0xc2, 0xbc, 0x0f, 0x61, 0x01, 0x79, 0xf0, 0xdc, 0x34, 0xdb, 0xad, + 0x99, 0x24, 0xd2, 0x52, 0x4f, 0x86, 0x86, 0xff, 0x5c, 0x4e, 0xf7, 0xa1, 0x8d, 0xd1, 0x73, 0xdf, + 0x9d, 0x75, 0x83, 0x38, 0x35, 0x6a, 0xb2, 0x0d, 0xce, 0x3e, 0x0d, 0xd4, 0x10, 0x21, 0x83, 0xd2, + 0x8b, 0x91, 0x94, 0xef, 0x6f, 0x44, 0x2e, 0x4d, 0x36, 0xf0, 0xac, 0xb0, 0xe7, 0x22, 0x93, 0x98, + 0xfa, 0x3e, 0xc5, 0x33, 0x79, 0x01, 0xee, 0xae, 0x18, 0x73, 0x6f, 0x09, 0xec, 0x60, 0x68, 0x7c, + 0xd8, 0xc1, 0xd0, 0x7b, 0x1f, 0xdd, 0x9b, 0xd4, 0xf4, 0x6b, 0x12, 0xfb, 0x34, 0xa0, 0x18, 0xf8, + 0x1e, 0xf4, 0x83, 0x7c, 0x20, 0x44, 0x36, 0x8e, 0x12, 0x26, 0x45, 0x66, 0x56, 0xee, 0x45, 0x90, + 0x3c, 0x85, 0x65, 0xe5, 0x3e, 0x94, 0x4c, 0xf2, 0xb2, 0x7e, 0xab, 0xd0, 0x56, 0x58, 0x15, 0xce, + 0x48, 0x38, 0x08, 0xca, 0xae, 0xac, 0x20, 0x0a, 0xe4, 0x3b, 0xed, 0x61, 0xfb, 0x84, 0x27, 0xb2, + 0xd1, 0x01, 0x28, 0xa3, 0x83, 0x3e, 0xd5, 0x82, 0x47, 0xf4, 0x53, 0x0c, 0xe7, 0xa5, 0x9a, 0xb3, + 0x42, 0x29, 0xea, 0xc8, 0xaf, 0x16, 0x40, 0x49, 0xa8, 0xc8, 0xab, 0x2b, 0xd6, 0xd5, 0x57, 0xbc, + 0x8d, 0xb2, 0xc6, 0xa6, 0x65, 0x97, 0x6b, 0x2b, 0x8d, 0xd3, 0xb2, 0x07, 0x3e, 0xae, 0x7b, 0x40, + 0x17, 0xef, 0xce, 0x4c, 0x0f, 0xe8, 0xa8, 0x75, 0x27, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0xdb, 0x0f, + 0x1f, 0x55, 0xfd, 0x60, 0xcf, 0xba, 0x44, 0xdc, 0xb8, 0x2c, 0xbb, 0xe2, 0x19, 0xf4, 0x1a, 0xf0, + 0x5c, 0x8f, 0x1b, 0x70, 0xeb, 0xcb, 0x13, 0x16, 0xc5, 0xec, 0x30, 0xd6, 0xeb, 0xa9, 0x5c, 0xb2, + 0xb3, 0x30, 0x89, 0xa0, 0x3f, 0x88, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, 0x03, 0x55, + 0xf1, 0x6a, 0x60, 0x7e, 0xfd, 0xbc, 0x7b, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, 0xc7, 0x5a, + 0x49, 0x0e, 0xa0, 0xb3, 0x15, 0x06, 0x5f, 0x67, 0xa2, 0x48, 0xe7, 0x92, 0x2e, 0x3f, 0x98, 0xf6, + 0xe5, 0x0f, 0xa6, 0x73, 0xe9, 0x83, 0xe9, 0x56, 0x1f, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a, + 0x5e, 0x6f, 0xb2, 0xae, 0xca, 0xaf, 0x99, 0xd3, 0xf8, 0x9a, 0x85, 0xb0, 0xa2, 0xd7, 0xd2, 0x9b, + 0x74, 0xfa, 0xbb, 0x0d, 0x2b, 0x94, 0xe7, 0xd1, 0x2b, 0x1e, 0x24, 0xb9, 0xcc, 0x8a, 0x91, 0xda, + 0x3e, 0xea, 0xfe, 0xb7, 0xe2, 0xd0, 0x64, 0xdb, 0xa1, 0x5a, 0x78, 0x9d, 0x4e, 0xf7, 0x1e, 0x42, + 0x6f, 0x76, 0x3a, 0x2f, 0x9b, 0x36, 0x4d, 0xbc, 0x87, 0xb0, 0x10, 0x8a, 0x22, 0x1b, 0x55, 0xed, + 0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x6b, 0x46, 0xe3, 0xc9, + 0x4c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x82, 0x9a, 0x5e, 0xb4, 0x26, 0xbf, 0x58, + 0xb0, 0xd8, 0xa4, 0xf0, 0x5a, 0x83, 0x5b, 0x55, 0xc4, 0x9e, 0x5b, 0x11, 0x67, 0x5e, 0x45, 0xdc, + 0xba, 0x22, 0xf5, 0x87, 0xb9, 0xd5, 0xf8, 0x30, 0x93, 0x63, 0xb8, 0x7b, 0xa9, 0x4c, 0x03, 0x31, + 0x49, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2, + 0x29, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x75, 0x70, 0x76, 0xf9, 0xe9, 0x15, 0xcf, + 0x57, 0x2a, 0xf2, 0x05, 0xf8, 0xfb, 0xe9, 0x98, 0x49, 0x7e, 0xa3, 0xdb, 0x5b, 0xd0, 0xd9, 0x13, + 0xa9, 0x88, 0xc5, 0xcb, 0xe9, 0x35, 0x53, 0xef, 0xc3, 0x82, 0xde, 0xdf, 0x7a, 0x8d, 0x74, 0x69, + 0x29, 0x92, 0xdb, 0xaa, 0xa1, 0x47, 0x2c, 0x1e, 0x15, 0xb1, 0xa2, 0xa1, 0x7e, 0xda, 0xf2, 0xad, + 0xe5, 0x3f, 0xcf, 0xd7, 0xac, 0xbf, 0xce, 0xd7, 0xac, 0xbf, 0xcf, 0xd7, 0xac, 0xdf, 0xfe, 0x59, + 0x7b, 0xeb, 0xb0, 0x8d, 0x3f, 0xf4, 0x8f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x4d, 0x5d, + 0xbe, 0xe1, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 95f211a81..1e97e29bc 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -4,6 +4,7 @@ package internal; message IndexMeta { bool Keys = 3; + bool TrackNotNull = 4; } message FieldOptions { From 60e83fc6dbdc25a21d8f0a483dc7f1d19d296635 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 6 Sep 2018 12:47:07 -0500 Subject: [PATCH 25/65] rename bitmap to row in tests --- executor_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/executor_test.go b/executor_test.go index 03ac09102..851c88713 100644 --- a/executor_test.go +++ b/executor_test.go @@ -33,8 +33,8 @@ import ( "github.com/pkg/errors" ) -// Ensure a bitmap query can be executed. -func TestExecutor_Execute_Bitmap(t *testing.T) { +// Ensure a row query can be executed. +func TestExecutor_Execute_Row(t *testing.T) { t.Run("Row", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -283,7 +283,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Run("OK", func(t *testing.T) { hldr.ClearBit("i", "f", 11, 1) if n := hldr.Row("i", "f", 11).Count(); n != 0 { - t.Fatalf("unexpected bitmap count: %d", n) + t.Fatalf("unexpected row count: %d", n) } if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil { @@ -295,7 +295,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } if n := hldr.Row("i", "f", 11).Count(); n != 1 { - t.Fatalf("unexpected bitmap count: %d", n) + t.Fatalf("unexpected row count: %d", n) } if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil { t.Fatal(err) @@ -328,7 +328,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Run("OK", func(t *testing.T) { hldr.SetBit("i", "f", 1, 0) if n := hldr.Row("i", "f", 11).Count(); n != 0 { - t.Fatalf("unexpected bitmap count: %d", n) + t.Fatalf("unexpected row count: %d", n) } if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil { @@ -340,7 +340,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } if n := hldr.Row("i", "f", 11).Count(); n != 1 { - t.Fatalf("unexpected bitmap count: %d", n) + t.Fatalf("unexpected row count: %d", n) } if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil { t.Fatal(err) @@ -477,7 +477,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { } t.Run("rowID", func(t *testing.T) { // Set two attrs on f/10. - // Also set attrs on other bitmaps and fields to test isolation. + // Also set attrs on other rows and fields to test isolation. if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } @@ -495,13 +495,13 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { - t.Fatalf("unexpected bitmap attr: %#v", m) + t.Fatalf("unexpected row attr: %#v", m) } }) t.Run("rowKey", func(t *testing.T) { // Set two attrs on f/10. - // Also set attrs on other bitmaps and fields to test isolation. + // Also set attrs on other rows and fields to test isolation. if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil { t.Fatal(err) } @@ -664,7 +664,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { } } -// Ensure a TopN() query with a source bitmap can be executed. +// Ensure a TopN() query with a source row can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -724,7 +724,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { } -//Ensure TopN handles Attribute filters with source bitmap +//Ensure TopN handles Attribute filters with source row func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() From 4c67cb18edd253df205ba6e8f9f3437012d1af72 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 6 Sep 2018 22:56:53 -0500 Subject: [PATCH 26/65] update notnull field on imports --- api.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/api.go b/api.go index 9e6ee1daa..8fdb4cfa0 100644 --- a/api.go +++ b/api.go @@ -686,6 +686,12 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error { timestamps[i] = &t } + // Import columnIDs into notnull field. + if err := importNotNullColumns(index, req.ColumnIDs); err != nil { + api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing notnull columns") + } + // Import into fragment. err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { @@ -720,6 +726,12 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error { } } + // Import columnIDs into notnull field. + if err := importNotNullColumns(index, req.ColumnIDs); err != nil { + api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing notnull columns") + } + // Import into fragment. err = field.importValue(req.ColumnIDs, req.Values) if err != nil { @@ -728,6 +740,17 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error { return errors.Wrap(err, "importing") } +func importNotNullColumns(index *Index, columnIDs []uint64) error { + nnf := index.unprotectedNotNullField() + if nnf == nil { + return nil + } + + notNullRowIDs := make([]uint64, len(columnIDs)) + notNullTimestamps := make([]*time.Time, len(columnIDs)) + return nnf.Import(notNullRowIDs, columnIDs, notNullTimestamps) +} + // MaxShards returns the maximum shard number for each index in a map. func (api *API) MaxShards(_ context.Context) map[string]uint64 { m := make(map[string]uint64) From a433862c989862699e7d9faf7af4924b617ea9fb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 7 Sep 2018 11:50:25 -0500 Subject: [PATCH 27/65] add tests for the notnull tracking --- executor_test.go | 46 ++++++++++++++++++++++++ http/client_test.go | 85 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 4 deletions(-) diff --git a/executor_test.go b/executor_test.go index 851c88713..602be93a5 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1497,6 +1497,52 @@ func TestExecutor_QueryCall(t *testing.T) { }) } +// Ensure a notnull field is maintained. +func TestExecutor_Execute_NotNull(t *testing.T) { + t.Run("Row", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackNotNull: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected notnull columns: %+v", bits) + } + + // Reopen cluster to ensure not-null is reloaded. + if err := c[0].Reopen(); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected notnull columns after reopen: %+v", bits) + } + }) +} + func benchmarkNotNull(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/http/client_test.go b/http/client_test.go index 131161d3c..33a452f7b 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -196,7 +196,7 @@ func TestClient_Export(t *testing.T) { {RowID: 2, ColumnID: 203, RowKey: "row2", ColumnKey: "col203"}, } - t.Run("Import unkeyed,unkeyedf", func(t *testing.T) { + t.Run("Export unkeyed,unkeyedf", func(t *testing.T) { // Populate data. for _, bit := range data { _, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{ @@ -230,7 +230,7 @@ func TestClient_Export(t *testing.T) { } }) - t.Run("Import unkeyed,keyedf", func(t *testing.T) { + t.Run("Export unkeyed,keyedf", func(t *testing.T) { // Populate data. for _, bit := range data { _, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{ @@ -264,7 +264,7 @@ func TestClient_Export(t *testing.T) { } }) - t.Run("Import keyed,unkeyedf", func(t *testing.T) { + t.Run("Export keyed,unkeyedf", func(t *testing.T) { // Populate data. for _, bit := range data { _, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{ @@ -298,7 +298,7 @@ func TestClient_Export(t *testing.T) { } }) - t.Run("Import keyed,keyedf", func(t *testing.T) { + t.Run("Export keyed,keyedf", func(t *testing.T) { // Populate data. for _, bit := range data { _, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{ @@ -655,6 +655,83 @@ func TestClient_ImportValue(t *testing.T) { } } +// Ensure client can bulk import data while tracking notnull. +func TestClient_ImportNotNull(t *testing.T) { + cmd := test.MustRunCluster(t, 1)[0] + host := cmd.URL() + holder := cmd.Server.Holder() + hldr := test.Holder{Holder: holder} + + t.Run("Set", func(t *testing.T) { + idxName := "iset" + fldName := "fset" + + index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true}) + _, err := index.CreateFieldIfNotExists(fldName) + if err != nil { + t.Fatal(err) + } + + // Send import request. + c := MustNewClient(host, http.GetHTTPClient(nil)) + if err := c.Import(context.Background(), idxName, fldName, 0, []pilosa.Bit{ + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, + }); err != nil { + t.Fatal(err) + } + + // Verify data. + if a := hldr.Row(idxName, fldName, 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + t.Fatalf("unexpected columns: %+v", a) + } + if a := hldr.Row(idxName, fldName, 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + t.Fatalf("unexpected columns: %+v", a) + } + + // Verify notnull. + if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { + t.Fatalf("unexpected notnull columns: %+v", a) + } + }) + + t.Run("Int", func(t *testing.T) { + idxName := "iint" + fldName := "fint" + + index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true}) + field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) + if err != nil { + t.Fatal(err) + } + + // Send import request. + c := MustNewClient(host, http.GetHTTPClient(nil)) + if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{ + {ColumnID: 1, Value: -10}, + {ColumnID: 2, Value: 20}, + {ColumnID: 3, Value: 40}, + }); err != nil { + t.Fatal(err) + } + + // Verify Sum. + sum, cnt, err := field.Sum(nil, fldName) + if err != nil { + t.Fatal(err) + } + if sum != 50 || cnt != 3 { + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify notnull. + if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + t.Fatalf("unexpected notnull columns: %+v", a) + } + }) +} + // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] From 3d7fc0c1c68cde863bf2c90c96f3b5c136285ff0 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 7 Sep 2018 11:54:44 -0500 Subject: [PATCH 28/65] remove unnecessary timestamp slice allocation --- api.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api.go b/api.go index 8fdb4cfa0..e5562602b 100644 --- a/api.go +++ b/api.go @@ -747,8 +747,7 @@ func importNotNullColumns(index *Index, columnIDs []uint64) error { } notNullRowIDs := make([]uint64, len(columnIDs)) - notNullTimestamps := make([]*time.Time, len(columnIDs)) - return nnf.Import(notNullRowIDs, columnIDs, notNullTimestamps) + return nnf.Import(notNullRowIDs, columnIDs, nil) } // MaxShards returns the maximum shard number for each index in a map. From 8b99414029bc9c9a22987b06c471e3507976fcb1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 10 Sep 2018 11:49:40 -0500 Subject: [PATCH 29/65] apply fieldName validation to index.CreateField --- field.go | 9 +++++++-- index.go | 12 +++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/field.go b/field.go index 9fb352fc1..7a60f3b3f 100644 --- a/field.go +++ b/field.go @@ -159,12 +159,17 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { func NewField(path, index, name string, opts FieldOption) (*Field, error) { err := validateName(name) if err != nil { - return nil, err + return nil, errors.Wrap(err, "validating name") } + return newField(path, index, name, opts) +} + +// newField returns a new instance of field (without name validation). +func newField(path, index, name string, opts FieldOption) (*Field, error) { // Apply functional option. fo := FieldOptions{} - err = opts(&fo) + err := opts(&fo) if err != nil { return nil, errors.Wrap(err, "applying option") } diff --git a/index.go b/index.go index adbc6810e..cccf2de69 100644 --- a/index.go +++ b/index.go @@ -289,6 +289,11 @@ func (i *Index) recalculateCaches() { // CreateField creates a field. func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { + err := validateName(name) + if err != nil { + return nil, errors.Wrap(err, "validating name") + } + i.mu.Lock() defer i.mu.Unlock() @@ -311,6 +316,11 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error) { + err := validateName(name) + if err != nil { + return nil, errors.Wrap(err, "validating name") + } + i.mu.Lock() defer i.mu.Unlock() @@ -379,7 +389,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { } func (i *Index) newField(path, name string) (*Field, error) { - f, err := NewField(path, i.name, name, OptFieldTypeDefault()) + f, err := newField(path, i.name, name, OptFieldTypeDefault()) if err != nil { return nil, err } From 9b4c67ee605c80d2bf3db3792ac196bc9fa08762 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 10 Sep 2018 13:13:11 -0500 Subject: [PATCH 30/65] rename notnull to exists --- api.go | 24 +++---- encoding/proto/proto.go | 6 +- executor.go | 6 +- executor_test.go | 24 +++---- holder.go | 6 +- http/client_test.go | 20 +++--- index.go | 40 +++++------ internal/private.pb.go | 156 ++++++++++++++++++++-------------------- internal/private.proto | 2 +- pilosa.go | 5 ++ pilosa_internal_test.go | 1 + test/holder.go | 14 ++++ 12 files changed, 162 insertions(+), 142 deletions(-) diff --git a/api.go b/api.go index e5562602b..beb2741d0 100644 --- a/api.go +++ b/api.go @@ -686,10 +686,10 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error { timestamps[i] = &t } - // Import columnIDs into notnull field. - if err := importNotNullColumns(index, req.ColumnIDs); err != nil { - api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) - return errors.Wrap(err, "importing notnull columns") + // Import columnIDs into existence field. + if err := importExistenceColumns(index, req.ColumnIDs); err != nil { + api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing existence columns") } // Import into fragment. @@ -726,10 +726,10 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error { } } - // Import columnIDs into notnull field. - if err := importNotNullColumns(index, req.ColumnIDs); err != nil { - api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) - return errors.Wrap(err, "importing notnull columns") + // Import columnIDs into existence field. + if err := importExistenceColumns(index, req.ColumnIDs); err != nil { + api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing existence columns") } // Import into fragment. @@ -740,14 +740,14 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error { return errors.Wrap(err, "importing") } -func importNotNullColumns(index *Index, columnIDs []uint64) error { - nnf := index.unprotectedNotNullField() +func importExistenceColumns(index *Index, columnIDs []uint64) error { + nnf := index.unprotectedExistenceField() if nnf == nil { return nil } - notNullRowIDs := make([]uint64, len(columnIDs)) - return nnf.Import(notNullRowIDs, columnIDs, nil) + existenceRowIDs := make([]uint64, len(columnIDs)) + return nnf.Import(existenceRowIDs, columnIDs, nil) } // MaxShards returns the maximum shard number for each index in a map. diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c00e71b40..0fcd9df3a 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -509,8 +509,8 @@ func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateInde func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta { return &internal.IndexMeta{ - Keys: m.Keys, - TrackNotNull: m.TrackNotNull, + Keys: m.Keys, + TrackExistence: m.TrackExistence, } } @@ -742,7 +742,7 @@ func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateI func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) { m.Keys = pb.Keys - m.TrackNotNull = pb.TrackNotNull + m.TrackExistence = pb.TrackExistence } func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) { diff --git a/executor.go b/executor.go index 1149f8c0e..5681644aa 100644 --- a/executor.go +++ b/executor.go @@ -1131,10 +1131,10 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op return false, ErrFieldNotFound } - // Set column on not-null field. - if nnf := idx.unprotectedNotNullField(); nnf != nil { + // Set column on existence field. + if nnf := idx.unprotectedExistenceField(); nnf != nil { if _, err := nnf.SetBit(0, colID, nil); err != nil { - return false, errors.Wrap(err, "setting not-null column") + return false, errors.Wrap(err, "setting existence column") } } diff --git a/executor_test.go b/executor_test.go index 602be93a5..25465411c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1497,13 +1497,13 @@ func TestExecutor_QueryCall(t *testing.T) { }) } -// Ensure a notnull field is maintained. -func TestExecutor_Execute_NotNull(t *testing.T) { +// Ensure an existence field is maintained. +func TestExecutor_Execute_Existence(t *testing.T) { t.Run("Row", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackNotNull: true}) + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) @@ -1524,26 +1524,26 @@ func TestExecutor_Execute_NotNull(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected notnull columns: %+v", bits) + t.Fatalf("unexpected existence columns: %+v", bits) } - // Reopen cluster to ensure not-null is reloaded. + // Reopen cluster to ensure existence field is reloaded. if err := c[0].Reopen(); err != nil { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected notnull columns after reopen: %+v", bits) + t.Fatalf("unexpected existence columns after reopen: %+v", bits) } }) } -func benchmarkNotNull(nn bool, b *testing.B) { +func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -1551,7 +1551,7 @@ func benchmarkNotNull(nn bool, b *testing.B) { indexName := "i" fieldName := "f" - index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackNotNull: nn}) + index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackExistence: nn}) // Create field. if _, err := index.CreateFieldIfNotExists(fieldName); err != nil { b.Fatal(err) @@ -1568,5 +1568,5 @@ func benchmarkNotNull(nn bool, b *testing.B) { } } -func BenchmarkExecutor_NotNull_True(b *testing.B) { benchmarkNotNull(true, b) } -func BenchmarkExecutor_NotNull_False(b *testing.B) { benchmarkNotNull(false, b) } +func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } +func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } diff --git a/holder.go b/holder.go index 252fca2db..794104e53 100644 --- a/holder.go +++ b/holder.go @@ -39,8 +39,8 @@ const ( // fileLimit is the maximum open file limit (ulimit -n) to automatically set. fileLimit = 262144 // (512^2) - // notNullFieldName is the name of the internal field used to store not-null values. - notNullFieldName = "notnull" // TODO: use a name less likely to collide? + // existenceFieldName is the name of the internal field used to store existence values. + existenceFieldName = "exists" ) // Holder represents a container for indexes. @@ -357,7 +357,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } index.keys = opt.Keys - index.trackNotNull = opt.TrackNotNull + index.trackExistence = opt.TrackExistence if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") diff --git a/http/client_test.go b/http/client_test.go index 33a452f7b..f0e8cf132 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -655,8 +655,8 @@ func TestClient_ImportValue(t *testing.T) { } } -// Ensure client can bulk import data while tracking notnull. -func TestClient_ImportNotNull(t *testing.T) { +// Ensure client can bulk import data while tracking existence. +func TestClient_ImportExistence(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] host := cmd.URL() holder := cmd.Server.Holder() @@ -666,7 +666,7 @@ func TestClient_ImportNotNull(t *testing.T) { idxName := "iset" fldName := "fset" - index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true}) + index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateFieldIfNotExists(fldName) if err != nil { t.Fatal(err) @@ -690,9 +690,9 @@ func TestClient_ImportNotNull(t *testing.T) { t.Fatalf("unexpected columns: %+v", a) } - // Verify notnull. - if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { - t.Fatalf("unexpected notnull columns: %+v", a) + // Verify existence. + if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { + t.Fatalf("unexpected existence columns: %+v", a) } }) @@ -700,7 +700,7 @@ func TestClient_ImportNotNull(t *testing.T) { idxName := "iint" fldName := "fint" - index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true}) + index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true}) field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100)) if err != nil { t.Fatal(err) @@ -725,9 +725,9 @@ func TestClient_ImportNotNull(t *testing.T) { t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) } - // Verify notnull. - if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { - t.Fatalf("unexpected notnull columns: %+v", a) + // Verify existence. + if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + t.Fatalf("unexpected existence columns: %+v", a) } }) } diff --git a/index.go b/index.go index cccf2de69..a78fd8365 100644 --- a/index.go +++ b/index.go @@ -37,8 +37,8 @@ type Index struct { keys bool // use string keys // Not-null tracking. - trackNotNull bool - notNullField *Field + trackExistence bool + existenceField *Field // Fields by name. fields map[string]*Field @@ -96,8 +96,8 @@ func (i *Index) Options() IndexOptions { func (i *Index) options() IndexOptions { return IndexOptions{ - Keys: i.keys, - TrackNotNull: i.trackNotNull, + Keys: i.keys, + TrackExistence: i.trackExistence, } } @@ -117,9 +117,9 @@ func (i *Index) Open() error { return errors.Wrap(err, "opening fields") } - if i.trackNotNull { - if err := i.openNotNullField(); err != nil { - return errors.Wrap(err, "opening not-null field") + if i.trackExistence { + if err := i.openExistenceField(); err != nil { + return errors.Wrap(err, "opening existence field") } } @@ -160,13 +160,13 @@ func (i *Index) openFields() error { return nil } -// openNotNullField gets or creates the not-null field and associates it to the index. -func (i *Index) openNotNullField() error { - f, err := i.createFieldIfNotExists(notNullFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0}) +// openExistenceField gets or creates the existence field and associates it to the index. +func (i *Index) openExistenceField() error { + f, err := i.createFieldIfNotExists(existenceFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0}) if err != nil { - return errors.Wrap(err, "creating not-null field") + return errors.Wrap(err, "creating existence field") } - i.notNullField = f + i.existenceField = f return nil } @@ -188,7 +188,7 @@ func (i *Index) loadMeta() error { // Copy metadata fields. i.keys = pb.Keys - i.trackNotNull = pb.TrackNotNull + i.trackExistence = pb.TrackExistence return nil } @@ -197,8 +197,8 @@ func (i *Index) loadMeta() error { func (i *Index) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.IndexMeta{ - Keys: i.keys, - TrackNotNull: i.trackNotNull, + Keys: i.keys, + TrackExistence: i.trackExistence, }) if err != nil { return errors.Wrap(err, "marshalling") @@ -275,9 +275,9 @@ func (i *Index) Fields() []*Field { return a } -// unprotectedNotNullField returns the internal field used to track not-null columns. -func (i *Index) unprotectedNotNullField() *Field { - return i.notNullField +// unprotectedExistenceField returns the internal field used to track column existence. +func (i *Index) unprotectedExistenceField() *Field { + return i.existenceField } // recalculateCaches recalculates caches on every field in the index. @@ -448,8 +448,8 @@ func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // IndexOptions represents options to set when initializing an index. type IndexOptions struct { - Keys bool `json:"keys"` - TrackNotNull bool `json:"trackNotNull"` + Keys bool `json:"keys"` + TrackExistence bool `json:"trackExistence"` } // hasTime returns true if a contains a non-nil time. diff --git a/internal/private.pb.go b/internal/private.pb.go index 2d1c0deef..c65049d51 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -62,8 +62,8 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` - TrackNotNull bool `protobuf:"varint,4,opt,name=TrackNotNull,proto3" json:"TrackNotNull,omitempty"` + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` + TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -78,9 +78,9 @@ func (m *IndexMeta) GetKeys() bool { return false } -func (m *IndexMeta) GetTrackNotNull() bool { +func (m *IndexMeta) GetTrackExistence() bool { if m != nil { - return m.TrackNotNull + return m.TrackExistence } return false } @@ -1059,10 +1059,10 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.TrackNotNull { + if m.TrackExistence { dAtA[i] = 0x20 i++ - if m.TrackNotNull { + if m.TrackExistence { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -2318,7 +2318,7 @@ func (m *IndexMeta) Size() (n int) { if m.Keys { n += 2 } - if m.TrackNotNull { + if m.TrackExistence { n += 2 } return n @@ -2925,7 +2925,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { m.Keys = bool(v != 0) case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TrackNotNull", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TrackExistence", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2942,7 +2942,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { break } } - m.TrackNotNull = bool(v != 0) + m.TrackExistence = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7184,74 +7184,74 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1093 bytes of a gzipped FileDescriptorProto + // 1095 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44, - 0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0x9b, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x95, - 0x1a, 0x2a, 0x11, 0xaa, 0xf6, 0x86, 0xbf, 0x4a, 0x25, 0xbb, 0x01, 0x4c, 0x49, 0x28, 0xe3, 0x24, - 0x77, 0xbd, 0x98, 0xec, 0x8e, 0x1a, 0x2b, 0x5e, 0x8f, 0xb1, 0xc7, 0x49, 0xb6, 0x17, 0xdc, 0x82, - 0xc4, 0x0b, 0x20, 0x9e, 0x84, 0x47, 0xe0, 0x92, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x9c, 0x19, 0xff, - 0x64, 0xb3, 0x21, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f, - 0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x66, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c, - 0x26, 0x03, 0xe8, 0x06, 0xc9, 0x98, 0x9f, 0xed, 0x70, 0xc9, 0x3c, 0x0f, 0xdc, 0x67, 0x7c, 0x9a, - 0xfb, 0xce, 0xba, 0xb5, 0xd1, 0xa1, 0x78, 0xf6, 0x08, 0x2c, 0xee, 0x65, 0x6c, 0x74, 0xbc, 0x2b, - 0xe4, 0x6e, 0x11, 0xc7, 0xbe, 0x8b, 0xba, 0x0b, 0x18, 0xf9, 0xc3, 0x82, 0xc5, 0xaf, 0x22, 0x1e, - 0x8f, 0xbf, 0x4f, 0x65, 0x24, 0x92, 0xdc, 0x7b, 0x0f, 0xba, 0x03, 0x36, 0x3a, 0xe2, 0x7b, 0xd3, - 0x94, 0xa3, 0xb7, 0x2e, 0xad, 0x81, 0x4a, 0x1b, 0x46, 0xaf, 0x38, 0xfa, 0xeb, 0xd3, 0x1a, 0xf0, - 0xd6, 0xa1, 0xb7, 0x17, 0x4d, 0xf8, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf1, 0x5b, 0x78, 0xbb, 0x09, - 0x29, 0x9a, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0xcb, 0xe0, 0xec, 0x44, 0x89, 0xdf, 0x5d, 0xb7, - 0x36, 0x1c, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0xe7, 0xf5, 0xea, 0xe7, - 0x11, 0x02, 0x4b, 0xc1, 0x24, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, 0xed, 0x2c, - 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xde, 0x8a, 0xc5, 0xe8, 0x78, 0xc8, 0x24, 0xa3, - 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x37, 0x63, 0xa7, 0x05, 0x85, 0x62, 0x1e, - 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, 0x47, 0x2c, - 0x1b, 0x63, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, 0x26, 0x01, - 0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe6, 0xbe, 0xb5, 0xee, 0x6c, - 0xb8, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0x93, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, 0xb9, 0x0b, - 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, 0x19, 0xd2, - 0xc8, 0xbd, 0x27, 0xd0, 0x09, 0x25, 0x4b, 0xc6, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfa, 0x60, 0xb3, - 0x6c, 0x9a, 0xcd, 0xca, 0x6c, 0xb3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa5, 0xd5, 0x95, 0x77, 0x3f, - 0x87, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0xa7, 0x65, 0x56, 0x8f, 0xf9, 0x54, 0xbd, 0xff, 0x84, - 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xd9, 0x9f, 0x58, 0xe4, 0x00, 0xbc, 0x41, 0xc6, - 0x99, 0xe4, 0x18, 0x64, 0x87, 0xe7, 0x39, 0x7b, 0xc9, 0xaf, 0xce, 0xb8, 0xce, 0xa2, 0xdd, 0xcc, - 0x62, 0x55, 0x07, 0xa7, 0x51, 0x07, 0xf2, 0x00, 0xbc, 0x21, 0x8f, 0xb9, 0xe4, 0xa6, 0xe3, 0xff, - 0xc3, 0x2f, 0x09, 0x4b, 0x0e, 0xd7, 0xdb, 0x7a, 0xf7, 0xc1, 0x55, 0xe3, 0x83, 0x14, 0x7a, 0x8f, - 0x6e, 0xd7, 0x79, 0xaa, 0x26, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0xe7, 0xda, 0x87, 0xcd, - 0x69, 0xa5, 0x07, 0x26, 0x94, 0x83, 0xa1, 0x56, 0xeb, 0x50, 0xcd, 0xf1, 0x33, 0xd1, 0x9e, 0x96, - 0xcf, 0xbd, 0x69, 0x34, 0xf2, 0xc2, 0xa0, 0xaa, 0x2b, 0x77, 0xd9, 0x84, 0x9b, 0x3b, 0x78, 0xae, - 0xa8, 0xd8, 0xd7, 0x53, 0x51, 0xee, 0x55, 0x27, 0xab, 0xcd, 0xe2, 0x28, 0xf7, 0x28, 0x90, 0xc7, - 0xd0, 0x0e, 0x47, 0x47, 0x7c, 0xc2, 0xbc, 0x0f, 0x61, 0x01, 0x79, 0xf0, 0xdc, 0x34, 0xdb, 0xad, - 0x99, 0x24, 0xd2, 0x52, 0x4f, 0x86, 0x86, 0xff, 0x5c, 0x4e, 0xf7, 0xa1, 0x8d, 0xd1, 0x73, 0xdf, - 0x9d, 0x75, 0x83, 0x38, 0x35, 0x6a, 0xb2, 0x0d, 0xce, 0x3e, 0x0d, 0xd4, 0x10, 0x21, 0x83, 0xd2, - 0x8b, 0x91, 0x94, 0xef, 0x6f, 0x44, 0x2e, 0x4d, 0x36, 0xf0, 0xac, 0xb0, 0xe7, 0x22, 0x93, 0x98, - 0xfa, 0x3e, 0xc5, 0x33, 0x79, 0x01, 0xee, 0xae, 0x18, 0x73, 0x6f, 0x09, 0xec, 0x60, 0x68, 0x7c, - 0xd8, 0xc1, 0xd0, 0x7b, 0x1f, 0xdd, 0x9b, 0xd4, 0xf4, 0x6b, 0x12, 0xfb, 0x34, 0xa0, 0x18, 0xf8, - 0x1e, 0xf4, 0x83, 0x7c, 0x20, 0x44, 0x36, 0x8e, 0x12, 0x26, 0x45, 0x66, 0x56, 0xee, 0x45, 0x90, - 0x3c, 0x85, 0x65, 0xe5, 0x3e, 0x94, 0x4c, 0xf2, 0xb2, 0x7e, 0xab, 0xd0, 0x56, 0x58, 0x15, 0xce, - 0x48, 0x38, 0x08, 0xca, 0xae, 0xac, 0x20, 0x0a, 0xe4, 0x3b, 0xed, 0x61, 0xfb, 0x84, 0x27, 0xb2, - 0xd1, 0x01, 0x28, 0xa3, 0x83, 0x3e, 0xd5, 0x82, 0x47, 0xf4, 0x53, 0x0c, 0xe7, 0xa5, 0x9a, 0xb3, - 0x42, 0x29, 0xea, 0xc8, 0xaf, 0x16, 0x40, 0x49, 0xa8, 0xc8, 0xab, 0x2b, 0xd6, 0xd5, 0x57, 0xbc, - 0x8d, 0xb2, 0xc6, 0xa6, 0x65, 0x97, 0x6b, 0x2b, 0x8d, 0xd3, 0xb2, 0x07, 0x3e, 0xae, 0x7b, 0x40, - 0x17, 0xef, 0xce, 0x4c, 0x0f, 0xe8, 0xa8, 0x75, 0x27, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0xdb, 0x0f, - 0x1f, 0x55, 0xfd, 0x60, 0xcf, 0xba, 0x44, 0xdc, 0xb8, 0x2c, 0xbb, 0xe2, 0x19, 0xf4, 0x1a, 0xf0, - 0x5c, 0x8f, 0x1b, 0x70, 0xeb, 0xcb, 0x13, 0x16, 0xc5, 0xec, 0x30, 0xd6, 0xeb, 0xa9, 0x5c, 0xb2, - 0xb3, 0x30, 0x89, 0xa0, 0x3f, 0x88, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, 0x03, 0x55, - 0xf1, 0x6a, 0x60, 0x7e, 0xfd, 0xbc, 0x7b, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, 0xc7, 0x5a, - 0x49, 0x0e, 0xa0, 0xb3, 0x15, 0x06, 0x5f, 0x67, 0xa2, 0x48, 0xe7, 0x92, 0x2e, 0x3f, 0x98, 0xf6, - 0xe5, 0x0f, 0xa6, 0x73, 0xe9, 0x83, 0xe9, 0x56, 0x1f, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a, - 0x5e, 0x6f, 0xb2, 0xae, 0xca, 0xaf, 0x99, 0xd3, 0xf8, 0x9a, 0x85, 0xb0, 0xa2, 0xd7, 0xd2, 0x9b, - 0x74, 0xfa, 0xbb, 0x0d, 0x2b, 0x94, 0xe7, 0xd1, 0x2b, 0x1e, 0x24, 0xb9, 0xcc, 0x8a, 0x91, 0xda, - 0x3e, 0xea, 0xfe, 0xb7, 0xe2, 0xd0, 0x64, 0xdb, 0xa1, 0x5a, 0x78, 0x9d, 0x4e, 0xf7, 0x1e, 0x42, - 0x6f, 0x76, 0x3a, 0x2f, 0x9b, 0x36, 0x4d, 0xbc, 0x87, 0xb0, 0x10, 0x8a, 0x22, 0x1b, 0x55, 0xed, - 0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x6b, 0x46, 0xe3, 0xc9, - 0x4c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x82, 0x9a, 0x5e, 0xb4, 0x26, 0xbf, 0x58, - 0xb0, 0xd8, 0xa4, 0xf0, 0x5a, 0x83, 0x5b, 0x55, 0xc4, 0x9e, 0x5b, 0x11, 0x67, 0x5e, 0x45, 0xdc, - 0xba, 0x22, 0xf5, 0x87, 0xb9, 0xd5, 0xf8, 0x30, 0x93, 0x63, 0xb8, 0x7b, 0xa9, 0x4c, 0x03, 0x31, - 0x49, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2, - 0x29, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x75, 0x70, 0x76, 0xf9, 0xe9, 0x15, 0xcf, - 0x57, 0x2a, 0xf2, 0x05, 0xf8, 0xfb, 0xe9, 0x98, 0x49, 0x7e, 0xa3, 0xdb, 0x5b, 0xd0, 0xd9, 0x13, - 0xa9, 0x88, 0xc5, 0xcb, 0xe9, 0x35, 0x53, 0xef, 0xc3, 0x82, 0xde, 0xdf, 0x7a, 0x8d, 0x74, 0x69, - 0x29, 0x92, 0xdb, 0xaa, 0xa1, 0x47, 0x2c, 0x1e, 0x15, 0xb1, 0xa2, 0xa1, 0x7e, 0xda, 0xf2, 0xad, - 0xe5, 0x3f, 0xcf, 0xd7, 0xac, 0xbf, 0xce, 0xd7, 0xac, 0xbf, 0xcf, 0xd7, 0xac, 0xdf, 0xfe, 0x59, - 0x7b, 0xeb, 0xb0, 0x8d, 0x3f, 0xf4, 0x8f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x4d, 0x5d, - 0xbe, 0xe1, 0x0b, 0x00, 0x00, + 0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0xdb, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x15, + 0x0d, 0x95, 0x08, 0x55, 0x7b, 0xc3, 0x5f, 0xa5, 0x92, 0x6c, 0x28, 0xa6, 0x24, 0x94, 0x71, 0x92, + 0xbb, 0x5e, 0x4c, 0x76, 0x47, 0x8d, 0x15, 0xaf, 0xc7, 0xd8, 0xe3, 0x24, 0xdb, 0x0b, 0x6e, 0x41, + 0xe2, 0x05, 0x10, 0x4f, 0xc2, 0x23, 0x70, 0xc9, 0x23, 0xa0, 0xf0, 0x22, 0x68, 0xce, 0x8c, 0x7f, + 0xb2, 0xd9, 0xb0, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f, + 0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x46, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c, + 0x26, 0x4f, 0xa1, 0x1b, 0x24, 0x23, 0x7e, 0xb6, 0xc3, 0x25, 0xf3, 0x3c, 0x70, 0x9f, 0xf1, 0x49, + 0xee, 0x3b, 0x6b, 0xd6, 0x7a, 0x87, 0xe2, 0xd9, 0xfb, 0x10, 0x16, 0xf7, 0x32, 0x36, 0x3c, 0xde, + 0x3e, 0x8b, 0x72, 0xc9, 0x93, 0x21, 0xf7, 0x5d, 0xd4, 0x4e, 0xa1, 0xe4, 0x0f, 0x0b, 0x6e, 0x7c, + 0x1d, 0xf1, 0x78, 0xf4, 0x7d, 0x2a, 0x23, 0x91, 0xe4, 0xde, 0x7b, 0xd0, 0xdd, 0x62, 0xc3, 0x23, + 0xbe, 0x37, 0x49, 0x39, 0x7a, 0xec, 0xd2, 0x1a, 0xa8, 0xb4, 0x61, 0xf4, 0x4a, 0x7b, 0xec, 0xd3, + 0x1a, 0xf0, 0xd6, 0xa0, 0xb7, 0x17, 0x8d, 0xf9, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf6, 0x5b, 0x78, + 0xbb, 0x09, 0x29, 0xaa, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0x4b, 0xe0, 0xec, 0x44, 0x89, 0xdf, + 0x5d, 0xb3, 0xd6, 0x1d, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0x27, 0xf6, + 0xea, 0x27, 0x12, 0x02, 0x8b, 0xc1, 0x38, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, + 0xed, 0x2c, 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xda, 0x8c, 0xc5, 0xf0, 0x78, 0xc0, + 0x24, 0xa3, 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x3b, 0x63, 0xa7, 0x05, 0x85, + 0x62, 0x1e, 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, + 0x47, 0x2c, 0x1b, 0x61, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, + 0x26, 0x01, 0x2c, 0x37, 0xe2, 0x1b, 0x9a, 0x2b, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe4, 0xbe, 0xb5, + 0xe6, 0xac, 0xbb, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0xe3, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, + 0xb9, 0x03, 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, + 0x19, 0xd2, 0xc8, 0xbd, 0xc7, 0xd0, 0x09, 0x25, 0x4b, 0x46, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfc, + 0x60, 0xa3, 0x6c, 0x9c, 0x8d, 0xca, 0x6c, 0xa3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa1, 0xd5, 0x95, + 0x77, 0xbf, 0x80, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0x27, 0x65, 0x56, 0x8f, 0xf9, 0x44, 0xbd, + 0xff, 0x84, 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xdb, 0x9f, 0x5a, 0xe4, 0x00, 0xbc, + 0xad, 0x8c, 0x33, 0xc9, 0x31, 0xc8, 0x0e, 0xcf, 0x73, 0xf6, 0x92, 0x5f, 0x9d, 0x71, 0x9d, 0x45, + 0xbb, 0x99, 0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x3e, 0x78, 0x03, 0x1e, 0x73, 0xc9, 0x4d, + 0xd7, 0xff, 0x87, 0x5f, 0x12, 0x96, 0x1c, 0xe6, 0xdb, 0x7a, 0xf7, 0xc0, 0x55, 0x23, 0x84, 0x14, + 0x7a, 0x0f, 0x6f, 0xd5, 0x79, 0xaa, 0xa6, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0x67, 0xee, + 0xc3, 0x66, 0xb4, 0xd2, 0x7d, 0x13, 0xca, 0xc1, 0x50, 0x2b, 0x75, 0xa8, 0xe6, 0xf8, 0x99, 0x68, + 0x4f, 0xca, 0xe7, 0x5e, 0x37, 0x1a, 0x79, 0x61, 0x50, 0xd5, 0x95, 0xbb, 0x6c, 0xcc, 0xcd, 0x1d, + 0x3c, 0x57, 0x54, 0xec, 0xf9, 0x54, 0x94, 0x7b, 0xd5, 0xc9, 0x6a, 0xbb, 0x38, 0xca, 0x3d, 0x0a, + 0xe4, 0x11, 0xb4, 0xc3, 0xe1, 0x11, 0x1f, 0x33, 0xef, 0x23, 0x58, 0x40, 0x1e, 0x3c, 0x37, 0xcd, + 0x76, 0x73, 0x2a, 0x89, 0xb4, 0xd4, 0x93, 0x81, 0xe1, 0x3f, 0x93, 0xd3, 0x3d, 0x68, 0x63, 0xf4, + 0xdc, 0x77, 0xa7, 0xdd, 0x20, 0x4e, 0x8d, 0x9a, 0x6c, 0x83, 0xb3, 0x4f, 0x03, 0x35, 0x44, 0xc8, + 0xa0, 0xf4, 0x62, 0x24, 0xe5, 0xfb, 0x1b, 0x91, 0x4b, 0x93, 0x0d, 0x3c, 0x2b, 0xec, 0xb9, 0xc8, + 0x24, 0xa6, 0xbe, 0x4f, 0xf1, 0x4c, 0x5e, 0x80, 0xbb, 0x2b, 0x46, 0xdc, 0x5b, 0x04, 0x3b, 0x18, + 0x18, 0x1f, 0x76, 0x30, 0xf0, 0xde, 0x47, 0xf7, 0x26, 0x35, 0xfd, 0x9a, 0xc4, 0x3e, 0x0d, 0x28, + 0x06, 0xbe, 0x0b, 0xfd, 0x20, 0xdf, 0x12, 0x22, 0x1b, 0x45, 0x09, 0x93, 0x22, 0x33, 0x6b, 0xf7, + 0x22, 0x48, 0x9e, 0xc0, 0x92, 0x72, 0x1f, 0x4a, 0x26, 0x79, 0x59, 0xbf, 0x15, 0x68, 0x2b, 0xac, + 0x0a, 0x67, 0x24, 0x1c, 0x04, 0x65, 0x57, 0x56, 0x10, 0x05, 0xf2, 0x9d, 0xf6, 0xb0, 0x7d, 0xc2, + 0x13, 0xd9, 0xe8, 0x00, 0x94, 0xd1, 0x41, 0x9f, 0x6a, 0xc1, 0x23, 0xfa, 0x29, 0x86, 0xf3, 0x62, + 0xcd, 0x59, 0xa1, 0x14, 0x75, 0xe4, 0x57, 0x0b, 0xa0, 0x24, 0x54, 0xe4, 0xd5, 0x15, 0xeb, 0xea, + 0x2b, 0xde, 0x7a, 0x59, 0x63, 0xd3, 0xb2, 0x4b, 0xb5, 0x95, 0xc6, 0x69, 0xd9, 0x03, 0x9f, 0xd4, + 0x3d, 0xa0, 0x8b, 0x77, 0x7b, 0xaa, 0x07, 0x74, 0xd4, 0xba, 0x13, 0x9e, 0x43, 0xaf, 0x81, 0xcf, + 0xec, 0x87, 0x8f, 0xab, 0x7e, 0xb0, 0xa7, 0x5d, 0x22, 0x6e, 0x5c, 0x96, 0x5d, 0xf1, 0x0c, 0x7a, + 0x0d, 0x78, 0xa6, 0xc7, 0x75, 0xb8, 0xf9, 0xd5, 0x09, 0x8b, 0x62, 0x76, 0x18, 0xeb, 0xf5, 0x54, + 0x2e, 0xd9, 0x69, 0x98, 0x44, 0xd0, 0xdf, 0x8a, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, + 0x03, 0x55, 0xf1, 0x6a, 0x60, 0x76, 0xfd, 0xbc, 0xbb, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, + 0xc7, 0x5a, 0x49, 0x0e, 0xa0, 0xb3, 0x19, 0x06, 0x4f, 0x33, 0x51, 0xa4, 0x33, 0x49, 0x97, 0x1f, + 0x4c, 0xfb, 0xf2, 0x07, 0xd3, 0xb9, 0xf4, 0xc1, 0x74, 0xab, 0x0f, 0x26, 0x09, 0x61, 0x59, 0xef, + 0x2b, 0x35, 0xaf, 0xd7, 0x59, 0x57, 0xe5, 0xd7, 0xcc, 0x69, 0x7c, 0xcd, 0x42, 0x58, 0xd6, 0x6b, + 0xe9, 0x4d, 0x3a, 0xfd, 0xdd, 0x86, 0x65, 0xca, 0xf3, 0xe8, 0x15, 0x0f, 0x92, 0x5c, 0x66, 0xc5, + 0x50, 0x6d, 0x1f, 0x75, 0xff, 0x5b, 0x71, 0x68, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x4e, 0xa7, 0x7b, + 0x0f, 0xa0, 0x37, 0x3d, 0x9d, 0x97, 0x4d, 0x9b, 0x26, 0xde, 0x03, 0x58, 0x08, 0x45, 0x91, 0x0d, + 0xab, 0xf6, 0x6d, 0x6c, 0x44, 0xcd, 0x4c, 0xab, 0x69, 0x69, 0xd6, 0x18, 0x8d, 0xd6, 0x9c, 0xd1, + 0x78, 0x3c, 0xd5, 0x4a, 0x7e, 0x1b, 0x2f, 0xbc, 0x53, 0x5f, 0xb8, 0xa0, 0xa6, 0x17, 0xad, 0xc9, + 0x2f, 0x16, 0xdc, 0x68, 0x52, 0x78, 0xad, 0xc1, 0xad, 0x2a, 0x62, 0xcf, 0xac, 0x88, 0x33, 0xab, + 0x22, 0x6e, 0x5d, 0x91, 0xfa, 0xc3, 0xdc, 0x6a, 0x7c, 0x98, 0xc9, 0x31, 0xdc, 0xb9, 0x54, 0xa6, + 0x2d, 0x31, 0x4e, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, + 0x05, 0xf2, 0x19, 0xdc, 0x0e, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x35, 0x70, 0x76, 0xf9, 0xe9, + 0x15, 0xcf, 0x57, 0x2a, 0xf2, 0x25, 0xf8, 0xfb, 0xe9, 0x88, 0x49, 0x7e, 0xad, 0xdb, 0x9b, 0xd0, + 0xd9, 0x13, 0xa9, 0x88, 0xc5, 0xcb, 0xc9, 0x9c, 0xa9, 0xf7, 0x61, 0x41, 0xef, 0x6f, 0xbd, 0x46, + 0xba, 0xb4, 0x14, 0xc9, 0x2d, 0xd5, 0xd0, 0x43, 0x16, 0x0f, 0x8b, 0x58, 0xd1, 0x50, 0x3f, 0x6d, + 0xf9, 0xe6, 0xd2, 0x9f, 0xe7, 0xab, 0xd6, 0x5f, 0xe7, 0xab, 0xd6, 0xdf, 0xe7, 0xab, 0xd6, 0x6f, + 0xff, 0xac, 0xbe, 0x75, 0xd8, 0xc6, 0x9f, 0xfa, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x31, + 0x07, 0xf3, 0xac, 0xe5, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1e97e29bc..88d16363f 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -4,7 +4,7 @@ package internal; message IndexMeta { bool Keys = 3; - bool TrackNotNull = 4; + bool TrackExistence = 4; } message FieldOptions { diff --git a/pilosa.go b/pilosa.go index 45d77a9f3..7378eb628 100644 --- a/pilosa.go +++ b/pilosa.go @@ -49,6 +49,8 @@ var ( ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") + ErrReservedName = errors.New("reserved index or field name") + // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") @@ -125,6 +127,9 @@ const TimeFormat = "2006-01-02T15:04" // validateName ensures that the name is a valid format. func validateName(name string) error { + if name == existenceFieldName { + return ErrReservedName + } if !nameRegexp.Match([]byte(name)) { return ErrName } diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index 139d5c2b7..96dae1433 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -34,6 +34,7 @@ func TestValidateNameInvalid(t *testing.T) { names := []string{ "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + "exists", } for _, name := range names { if validateName(name) == nil { diff --git a/test/holder.go b/test/holder.go index 80778fe2a..8d080e493 100644 --- a/test/holder.go +++ b/test/holder.go @@ -91,6 +91,20 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { return row } +// ReadRow returns a Row for a given field. If the field does not exist, +// it panics rather than creating the field. +func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row { + f := h.Holder.Field(index, field) + if f == nil { + panic(pilosa.ErrFieldNotFound) + } + row, err := f.Row(rowID) + if err != nil { + panic(err) + } + return row +} + func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault()) From f8c745340ff467983f840e682d7ff4601b39be3a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 10 Sep 2018 16:56:33 -0500 Subject: [PATCH 31/65] stop tracking existence if the existence field is deleted --- fragment_internal_test.go | 4 ++-- index.go | 12 +++++++++++ index_internal_test.go | 44 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index c85b7e88c..a0d1354ba 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -746,7 +746,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { cacheSize := uint32(3) // Create Index. - index := mustOpenIndex() + index := mustOpenIndex(IndexOptions{}) defer index.Close() // Create field. @@ -912,7 +912,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - index := mustOpenIndex() + index := mustOpenIndex(IndexOptions{}) defer index.Close() // Create field. diff --git a/index.go b/index.go index a78fd8365..34ab01b28 100644 --- a/index.go +++ b/index.go @@ -421,6 +421,18 @@ func (i *Index) DeleteField(name string) error { return errors.Wrap(err, "removing directory") } + // If the field being deleted is the existence field, + // turn off existence tracking on the index. + if name == existenceFieldName { + i.trackExistence = false + i.existenceField = nil + + // Update meta data on disk. + if err := i.saveMeta(); err != nil { + return errors.Wrap(err, "saving existence meta data") + } + } + // Remove reference. delete(i.fields, name) diff --git a/index_internal_test.go b/index_internal_test.go index 1e6d592ab..de4368d2a 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -16,10 +16,11 @@ package pilosa import ( "io/ioutil" + "testing" ) // mustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func mustOpenIndex() *Index { +func mustOpenIndex(opt IndexOptions) *Index { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { panic(err) @@ -28,6 +29,10 @@ func mustOpenIndex() *Index { if err != nil { panic(err) } + + index.keys = opt.Keys + index.trackExistence = opt.TrackExistence + if err := index.Open(); err != nil { panic(err) } @@ -44,3 +49,40 @@ func (i *Index) reopen() error { } return nil } + +// Ensure that deleting the existence field is handled properly. +func TestIndex_Existence_Delete(t *testing.T) { + // Create Index (with existence tracking). + index := mustOpenIndex(IndexOptions{TrackExistence: true}) + defer index.Close() + + // Ensure existence field has been created. + ef := index.Field(existenceFieldName) + if ef == nil { + t.Fatalf("expected field to have been created: %s", existenceFieldName) + } else if !index.trackExistence { + t.Fatalf("expected index.trackExistence to be true") + } else if index.existenceField == nil { + t.Fatalf("expected index.existenceField to be non-nil") + } + + // Delete existence field. + if err := index.DeleteField(existenceFieldName); err != nil { + t.Fatal(err) + } + + // Re-open index. + if err := index.reopen(); err != nil { + t.Fatal(err) + } + + // Ensure existence field no longer exists. + ef = index.Field(existenceFieldName) + if ef != nil { + t.Fatalf("expected field to have been deleted: %s", existenceFieldName) + } else if index.trackExistence { + t.Fatalf("expected index.trackExistence to be false") + } else if index.existenceField != nil { + t.Fatalf("expected index.existenceField to be nil") + } +} From f7abf6062741d8fb129171a21991064592bb2667 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 11 Sep 2018 10:24:49 -0500 Subject: [PATCH 32/65] add lock around existencFld --- api.go | 6 +++--- executor.go | 4 ++-- index.go | 17 ++++++++++------- index_internal_test.go | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/api.go b/api.go index beb2741d0..c8aa2b5cc 100644 --- a/api.go +++ b/api.go @@ -741,13 +741,13 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error { } func importExistenceColumns(index *Index, columnIDs []uint64) error { - nnf := index.unprotectedExistenceField() - if nnf == nil { + ef := index.existenceField() + if ef == nil { return nil } existenceRowIDs := make([]uint64, len(columnIDs)) - return nnf.Import(existenceRowIDs, columnIDs, nil) + return ef.Import(existenceRowIDs, columnIDs, nil) } // MaxShards returns the maximum shard number for each index in a map. diff --git a/executor.go b/executor.go index 5681644aa..389cb573b 100644 --- a/executor.go +++ b/executor.go @@ -1132,8 +1132,8 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } // Set column on existence field. - if nnf := idx.unprotectedExistenceField(); nnf != nil { - if _, err := nnf.SetBit(0, colID, nil); err != nil { + if ef := idx.existenceField(); ef != nil { + if _, err := ef.SetBit(0, colID, nil); err != nil { return false, errors.Wrap(err, "setting existence column") } } diff --git a/index.go b/index.go index 34ab01b28..0992ab213 100644 --- a/index.go +++ b/index.go @@ -36,9 +36,9 @@ type Index struct { name string keys bool // use string keys - // Not-null tracking. + // Existence tracking. trackExistence bool - existenceField *Field + existenceFld *Field // Fields by name. fields map[string]*Field @@ -166,7 +166,7 @@ func (i *Index) openExistenceField() error { if err != nil { return errors.Wrap(err, "creating existence field") } - i.existenceField = f + i.existenceFld = f return nil } @@ -275,9 +275,12 @@ func (i *Index) Fields() []*Field { return a } -// unprotectedExistenceField returns the internal field used to track column existence. -func (i *Index) unprotectedExistenceField() *Field { - return i.existenceField +// existenceField returns the internal field used to track column existence. +func (i *Index) existenceField() *Field { + i.mu.RLock() + defer i.mu.RUnlock() + + return i.existenceFld } // recalculateCaches recalculates caches on every field in the index. @@ -425,7 +428,7 @@ func (i *Index) DeleteField(name string) error { // turn off existence tracking on the index. if name == existenceFieldName { i.trackExistence = false - i.existenceField = nil + i.existenceFld = nil // Update meta data on disk. if err := i.saveMeta(); err != nil { diff --git a/index_internal_test.go b/index_internal_test.go index de4368d2a..fd83fd775 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -62,7 +62,7 @@ func TestIndex_Existence_Delete(t *testing.T) { t.Fatalf("expected field to have been created: %s", existenceFieldName) } else if !index.trackExistence { t.Fatalf("expected index.trackExistence to be true") - } else if index.existenceField == nil { + } else if index.existenceFld == nil { t.Fatalf("expected index.existenceField to be non-nil") } @@ -82,7 +82,7 @@ func TestIndex_Existence_Delete(t *testing.T) { t.Fatalf("expected field to have been deleted: %s", existenceFieldName) } else if index.trackExistence { t.Fatalf("expected index.trackExistence to be false") - } else if index.existenceField != nil { + } else if index.existenceFld != nil { t.Fatalf("expected index.existenceField to be nil") } } From a89e1c521becfe0a7291cc4fcb25e79925e18f83 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 11 Sep 2018 14:52:03 -0500 Subject: [PATCH 33/65] benchmark import instead of set on existence field --- executor_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/executor_test.go b/executor_test.go index 25465411c..5704e607b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1557,12 +1557,22 @@ func benchmarkExistence(nn bool, b *testing.B) { b.Fatal(err) } + bitCount := 10000 + req := &pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnIDs: make([]uint64, bitCount), + } + for i := 0; i < bitCount; i++ { + req.RowIDs[i] = uint64(rand.Intn(100000)) + req.ColumnIDs[i] = uint64(rand.Intn(1 << 20)) + } + b.ResetTimer() for i := 0; i < b.N; i++ { - colID := uint64(rand.Intn(1 << 20)) - rowID := uint64(rand.Intn(100000)) - qry := fmt.Sprintf(`Set(%d, %s=%d)`, colID, fieldName, rowID) - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: qry}); err != nil { + if err := c[0].API.Import(context.Background(), req); err != nil { b.Fatal(err) } } From 4ea48e1b40b9161e49c73aadc39a11d74ea82a12 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 21 Aug 2018 12:53:53 -0500 Subject: [PATCH 34/65] remove unused log buffers from test cluster, fixes race the buffers were unused internally and external users had no access to them. Those wishing to read the logs of the cluster in tests may replace stdout/stderr with buffers on the Command struct. The race occurred when a node was stopped and then started again. some memberlist goroutines might not be completely cleaned up by the time the node restarted, and then two loggers were using the same output buffer. --- test/pilosa.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index e7111ad92..9dbd78057 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "fmt" - "io" "io/ioutil" gohttp "net/http" "os" @@ -40,10 +39,6 @@ type Command struct { *server.Command commandOptions []server.CommandOption - - stdin bytes.Buffer - stdout bytes.Buffer - stderr bytes.Buffer } func OptAllowedOrigins(origins []string) server.CommandOption { @@ -65,17 +60,15 @@ func newCommand(opts ...server.CommandOption) *Command { // beginning of the option slice so that it can be overridden by user-passed // options. opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...) - m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts} + m := &Command{commandOptions: opts} + m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true - m.Command.Stdin = &m.stdin - m.Command.Stdout = &m.stdout - m.Command.Stderr = &m.stderr if testing.Verbose() { - m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) - m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) + m.Command.Stdout = os.Stdout + m.Command.Stderr = os.Stderr } return m @@ -120,7 +113,7 @@ func (m *Command) Reopen() error { // Create new main with the same config. config := m.Command.Config - m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...) + m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...) m.Command.Config = config // Run new program. From 319456bbd73493f3a12d24e224d50ae083bc0e3a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 11 Sep 2018 12:06:03 -0500 Subject: [PATCH 35/65] implements Not() query --- docs/administration.md | 2 ++ docs/query-language.md | 36 +++++++++++++++++++++--- executor.go | 34 ++++++++++++++++++++++ executor_test.go | 42 ++++++++++++++++++++++++++++ internal/test/querygenerator.go | 20 +++++++++++++ internal/test/querygenerator_test.go | 32 +++++++++++++++++++++ 6 files changed, 162 insertions(+), 4 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 71ab54672..cd4e30184 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -294,6 +294,8 @@ We currently track the following events - **Union:** Count of Union queries. - **Intersection:** Count of Intersection queries. - **Difference:** Count of Difference queries. +- **Xor:** Count of Xor queries. +- **Not:** Count of Not queries. - **Count:** Count of Count queries. - **Range:** Count of Range queries. - **Snapshot:** Event count when the snapshot process is triggered. diff --git a/docs/query-language.md b/docs/query-language.md index a61a9bf09..d289fb9a2 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -48,7 +48,7 @@ curl localhost:10101/index/repository/query \ * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`, `Not` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -94,7 +94,7 @@ Set(10, stargazer=1, 2016-01-01T00:00) Set multiple bits in a single request: ```request -Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30) +Set(10, stargazer=1) Set(20, stargazer=1) Set(10, stargazer=2) Set(30, stargazer=2) ``` ```response {"results":[false,true,true,true]} @@ -358,7 +358,7 @@ Difference(Row(stargazer=2), Row(stargazer=1)) {"attrs":{},"columns":[30]} ``` -* columnss are repositories that were starred by user 2 BUT NOT user 1 +* columns are repositories that were starred by user 2 BUT NOT user 1 #### Xor @@ -384,11 +384,39 @@ Query columns with a bit set in exactly one of two rows (repositories that are s Xor(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"results":[{"attrs":{},"columns":[10,20,30]}]} +{"results":[{"attrs":{},"columns":[20,30]}]} ``` * columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) +#### Not + +**Spec:** + +``` +Not() +``` + +**Description:** + +Not returns the inverse of all of the bits from the `ROW_CALL` argument. The Not query requires that `trackExistence` has been enabled on the Index. + +**Result Type:** object with attrs and columns + +attrs will always be empty + +**Examples:** + +Query columns with a bit set in one row and not another (repositories that are starred by one user and not another): +```request +Not(Row(stargazer=1)) +``` +```response +{"results":[{"attrs":{},"columns":[30]}]} +``` + +* columns are repositories that were not starred by user 1 + #### Count **Spec:** diff --git a/executor.go b/executor.go index 389cb573b..b0083a514 100644 --- a/executor.go +++ b/executor.go @@ -443,6 +443,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeUnionShard(ctx, index, c, shard) case "Xor": return e.executeXorShard(ctx, index, c, shard) + case "Not": + return e.executeNotShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -1007,6 +1009,38 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal return other, nil } +// executeNotShard executes a not() call for a local shard. +func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if len(c.Children) == 0 { + return nil, errors.New("Not() requires an input row") + } else if len(c.Children) > 1 { + return nil, errors.New("Not() only accepts a single row input") + } + + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + + return existenceRow.Difference(row), nil +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { if len(c.Children) == 0 { diff --git a/executor_test.go b/executor_test.go index 5704e607b..ee919f001 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1543,6 +1543,48 @@ func TestExecutor_Execute_Existence(t *testing.T) { }) } +// Ensure a not query can be executed. +func TestExecutor_Execute_Not(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), + }); err != nil { + t.Fatal(err) + } + + // Populated row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=20))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Populated row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=0))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // All existing. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Union(Row(f=10), Row(f=20)))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go index a7cf70c9b..ea0dc81c0 100644 --- a/internal/test/querygenerator.go +++ b/internal/test/querygenerator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -95,11 +109,17 @@ func Difference(args ...*pql.Call) *pql.Call { kvargs, children := magic(args) return &pql.Call{Name: "Difference", Args: kvargs, Children: children} } + func Xor(args ...*pql.Call) *pql.Call { kvargs, children := magic(args) return &pql.Call{Name: "Xor", Args: kvargs, Children: children} } +func Not(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Not", Args: kvargs, Children: children} +} + func Between(_ string, min, max int) *pql.Call { return &pql.Call{ Name: "Range", diff --git a/internal/test/querygenerator_test.go b/internal/test/querygenerator_test.go index 4ca50c265..9bb404ab4 100644 --- a/internal/test/querygenerator_test.go +++ b/internal/test/querygenerator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -208,6 +222,24 @@ func TestPQL_Generator(t *testing.T) { }, }, }, + { + pql: `Not(Row(aaa=10))`, + calc: PQL(Not(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Not", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, } { if !Compare(u.calc, u.exp) { From 2dcdd9614cc1c2a6fad03a2917d2bd9deb153b32 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 12 Sep 2018 00:01:22 +0300 Subject: [PATCH 36/65] Added docs for the Options call --- docs/query-language.md | 66 +++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index a61a9bf09..21722fd80 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -46,8 +46,10 @@ curl localhost:10101/index/repository/query \ * `field` The field specifies on which Pilosa [field](../glossary/#field) the query will operate. Valid field names are lower case strings; they start with an alphanumeric character, and contain only alphanumeric characters and `_-`. They must be 64 characters or less in length. * `TIMESTAMP` This is a timestamp in the following format `YYYY-MM-DDTHH:MM` (e.g. 2006-01-02T15:04) * `UINT` An unsigned integer (e.g. 42839) +* `BOOL` A boolean value, `true` or `false` * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. +* `CALL` Any query * `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) @@ -112,8 +114,8 @@ Set(10, pullrequests=2) **Spec:** ``` -SetRowAttrs(, , - , +SetRowAttrs(, , + , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -150,8 +152,8 @@ SetRowAttrs(stargazer, 10, username=null) **Spec:** ``` -SetColumnAttrs(, - , +SetColumnAttrs(, + , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -262,7 +264,7 @@ Row(stargazer=1) {"attrs":{"username":"mrpi","active":true},"columns":[10, 20]} ``` -* attrs are the attributes for user 1 +* attrs are the attributes for user 1 * columns are the repositories which user 1 has starred. #### Union @@ -497,7 +499,7 @@ Range(=, , ) **Description:** Similar to `Row`, but only returns bits which were set with timestamps -between the given `start` (first) and `end` (second) timestamps. +between the given `start` (first) and `end` (second) timestamps. **Result Type:** object with attrs and bits @@ -548,14 +550,14 @@ Range(commitactivity > 100) BSI range queries support the following operators: - Operator | Name | Value + Operator | Name | Value ----------|-------------------------------|-------------------- - `>` | greater-than, GT | integer - `<` | less-than, LT | integer - `<=` | less-than-or-equal-to, LTE | integer - `>=` | greater-than-or-equal-to, GTE | integer - `==` | equal-to, EQ | integer - `!=` | not-equal-to, NEQ | integer or `null` + `>` | greater-than, GT | integer + `<` | less-than, LT | integer + `<=` | less-than-or-equal-to, LTE | integer + `>=` | greater-than-or-equal-to, GTE | integer + `==` | equal-to, EQ | integer + `!=` | not-equal-to, NEQ | integer or `null` `<`, and `<=` can be chained together to represent a bounded interval. For example: @@ -645,3 +647,41 @@ Sum(field="diskusage") ``` * Result is the sum of all values (total size of all repositories in kilobytes, here), plus the count of columns. + +### Other Operations + +#### Options + +**Spec:** + +``` +Options(, columnAttrs=, excludeColumns=, excludeRowAttrs=, shards=[UINT ...]) +``` + +**Description:** + +Modifies the given query as follows: +* `columnAttrs`: The result includes column attributes (Default: `false`). +* `excludeColumns`: Column IDs are not included in the result (Default: `false`). +* `excludeRowAttrs`: Row attributes are not included in the result (Default: `false`). +* `shards`: Runs the query only for the given shards. By default, the query is executed for all shards. + +**Result Type:** Result type of the given query. + +**Examples:** + +Return column attributes: +```request +Query(Row(f1=10), columnAttrs=true) +``` +```response +{"attrs":{},"columns":[100]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}} +``` + +Run the query against shards 0 and 2 only: +```request +Query(Row(f1=10), shards=[0, 2]) +``` +```response +{"attrs":{},"columns":[100, 2097152]} +``` From ec9371aae4fc48e35576bffeb54c6be6160d3c6b Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 12 Sep 2018 00:53:40 +0300 Subject: [PATCH 37/65] Updated Options docs --- docs/query-language.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 21722fd80..bfb67789c 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -661,18 +661,18 @@ Options(, columnAttrs=, excludeColumns=, excludeRowAttrs=. **Examples:** Return column attributes: ```request -Query(Row(f1=10), columnAttrs=true) +Options(Row(f1=10), columnAttrs=true) ``` ```response {"attrs":{},"columns":[100]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}} @@ -680,7 +680,7 @@ Query(Row(f1=10), columnAttrs=true) Run the query against shards 0 and 2 only: ```request -Query(Row(f1=10), shards=[0, 2]) +Options(Row(f1=10), shards=[0, 2]) ``` ```response {"attrs":{},"columns":[100, 2097152]} From b1af66a427d96fd4da5aa372fdb4c653d629765f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 12 Sep 2018 00:57:45 +0300 Subject: [PATCH 38/65] updated Options docs --- docs/query-language.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index bfb67789c..8af1da587 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -662,8 +662,8 @@ Options(, columnAttrs=, excludeColumns=, excludeRowAttrs=. From 63f1b65c0aa473830bb66895e93648e3364ec14f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 12 Sep 2018 00:59:44 +0300 Subject: [PATCH 39/65] updated Options docs --- docs/query-language.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 8af1da587..3db96c987 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -662,8 +662,8 @@ Options(, columnAttrs=, excludeColumns=, excludeRowAttrs=. From 424ca9378a43df41b355747e14919ad517c71893 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 12 Sep 2018 09:20:31 -0500 Subject: [PATCH 40/65] fixing a copy/paste issue in the Not() docs --- docs/query-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/query-language.md b/docs/query-language.md index 34cc7a574..876cf69c6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -409,7 +409,7 @@ attrs will always be empty **Examples:** -Query columns with a bit set in one row and not another (repositories that are starred by one user and not another): +Query existing columns that do not have a bit set in the given row. ```request Not(Row(stargazer=1)) ``` From cf47950bd760f2de0a6fe4cec535d7a03bbc27c6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 10:06:03 -0500 Subject: [PATCH 41/65] gofmt all --- api.go | 2 +- client.go | 4 ++-- http/handler.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index ffe85dd37..3e8243668 100644 --- a/api.go +++ b/api.go @@ -296,7 +296,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e } // ImportRoaringBytes fast loading of standard roaring format -func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64 ) (err error) { +func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { if err = api.validate(apiField); err != nil { err = errors.Wrap(err, "validating api method") return diff --git a/client.go b/client.go index c13d44560..7eb24d121 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error + ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error { +func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/http/handler.go b/http/handler.go index f86549c3c..822f242f3 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1458,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard ) + err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard) // Marshal response object. msg := string("") From d3717e2afe0a369fcfdde77dc6399e72fd1e67ee Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 10:50:05 -0500 Subject: [PATCH 42/65] make Bitmap.UnmarshalBinary work for roaring or pilosa format --- ctl/check_test.go | 2 +- fragment.go | 3 ++- roaring/roaring.go | 23 ++++++++++++----------- roaring/roaring_internal_test.go | 9 ++++++--- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ctl/check_test.go b/ctl/check_test.go index 56d70f97a..1b2ebd6f6 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) { var buf bytes.Buffer io.Copy(&buf, r) - if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") { + if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: reading roaring header:") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) } // Todo: need correct roaring file for happy path diff --git a/fragment.go b/fragment.go index e7014aeb0..e05b42954 100644 --- a/fragment.go +++ b/fragment.go @@ -1424,7 +1424,8 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error func (f *fragment) importRoaringBytes(roaringBytes []byte) error { f.mu.Lock() defer f.mu.Unlock() - bm, err := roaring.UnmarshalStandardRoaring(roaringBytes) + bm := roaring.NewBitmap() + err := bm.UnmarshalBinary(roaringBytes) if err != nil { return err } diff --git a/roaring/roaring.go b/roaring/roaring.go index abf36c40e..c9f832126 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -17,13 +17,14 @@ package roaring import ( "encoding/binary" - "errors" "fmt" "hash/fnv" "io" "math/bits" "sort" "unsafe" + + "github.com/pkg/errors" ) const ( @@ -614,8 +615,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { return n, nil } -// UnmarshalBinary decodes b from a binary-encoded byte slice. -func (b *Bitmap) UnmarshalBinary(data []byte) error { +// unmarshalPilosaRoaring treats data as being encoded in Pilosa's 64 bit +// roaring format and decodes it into b. +func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { if len(data) < headerBaseSize { return errors.New("data too small") } @@ -3408,18 +3410,17 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint return } -func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { - b := NewBitmap() - +// UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in +// either standard roaring format or Pilosa's roaring format. +func (b *Bitmap) UnmarshalBinary(data []byte) error { fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - if fileMagic == magicNumber { //if pilosa roaring - err := b.UnmarshalBinary(data) - return b, err + if fileMagic == magicNumber { // if pilosa roaring + return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) if err != nil { - return nil, err + return errors.Wrap(err, "reading roaring header") } b.Containers.Reset() @@ -3439,7 +3440,7 @@ func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { } else { readOffsets(b, data, pos, keyN) } - return b, nil + return nil } func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 2469657d1..caea8e2c1 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3231,7 +3231,8 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { func TestUnmarshalStdRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays rbContainerWithTwoArrays, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") - bm, er := UnmarshalStandardRoaring(rbContainerWithTwoArrays) + bm := NewBitmap() + er := bm.UnmarshalBinary(rbContainerWithTwoArrays) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3240,7 +3241,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { } //generated serialize image from java(clojure) with a run and array rbContainerWithRLEandArray, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - bm, er = UnmarshalStandardRoaring(rbContainerWithRLEandArray) + bm = NewBitmap() + er = bm.UnmarshalBinary(rbContainerWithRLEandArray) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3249,7 +3251,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { } //had to use an external file because emacs was barfing on the long line :() _bitmap_array_container, _ := ioutil.ReadFile("testdata/bitmapcontainer.roaringbitmap") - bm, er = UnmarshalStandardRoaring(_bitmap_array_container) + bm = NewBitmap() + er = bm.UnmarshalBinary(_bitmap_array_container) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } From 8254d0fadd288a54999c57d1bc570fb8d7a47b77 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 12:08:57 -0500 Subject: [PATCH 43/65] improve import roaring docs, check errors --- api.go | 18 +++++++++++++++++- fragment.go | 3 ++- roaring/roaring.go | 17 ++++++++++------- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/api.go b/api.go index 3e8243668..012e4bf3e 100644 --- a/api.go +++ b/api.go @@ -295,7 +295,23 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e return field, nil } -// ImportRoaringBytes fast loading of standard roaring format +// ImportRoaringBytes is a low level interface for importing data to Pilosa when +// extremely high throughput is desired. The data must be encoded in a +// particular way which may be unintuitive (discussed below), and overwrites any +// existing data in the particular fragment into which it is being loaded. +// +// It takes as input a roaring bitmap which it uses as the data for the +// indicated index, field, and shard. The bitmap may be encoded according to the +// standard roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec), +// or to the pilosa roaring spec which supports 64 bit integers +// (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format). +// +// The data, roaringBytes, should be encoded the same way that Pilosa stores +// fragments internally. A bit "i" being set in the input bitmap indicates that +// the bit is set in Pilosa row "i/sliceWidth", and in column +// (shard*sliceWidth)+(i%sliceWidth). That is to say that roaringBytes +// represents all of the rows in this shard of this field concatenated together +// in one long bitmap. func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { if err = api.validate(apiField); err != nil { err = errors.Wrap(err, "validating api method") diff --git a/fragment.go b/fragment.go index e05b42954..904652a57 100644 --- a/fragment.go +++ b/fragment.go @@ -1420,7 +1420,8 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error } // importRoaringBytes imports from standard roaring data format defined at -// https://github.com/RoaringBitmap/RoaringFormatSpec +// https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version +// of the roaring format. func (f *fragment) importRoaringBytes(roaringBytes []byte) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/roaring/roaring.go b/roaring/roaring.go index c9f832126..6341453a4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3436,9 +3436,15 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Read container offsets and attach data. if haveRuns { - readWithRuns(b, data, pos, keyN) + err := readWithRuns(b, data, pos, keyN) + if err != nil { + return errors.Wrap(err, "reading runs from standard roaring format") + } } else { - readOffsets(b, data, pos, keyN) + err := readOffsets(b, data, pos, keyN) + if err != nil { + return errors.Wrap(err, "reading offsets from standard roaring format") + } } return nil } @@ -3450,8 +3456,7 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { offset := binary.LittleEndian.Uint32(buf[0:4]) // Verify the offset is within the bounds of the input data. if int(offset) >= len(data) { - err = fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) - return + return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) } // Map byte slice directly to the container data. @@ -3467,12 +3472,10 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { c.runs = nil c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] default: - err = fmt.Errorf("unsupported container type %d", c.containerType) - return + return fmt.Errorf("unsupported container type %d", c.containerType) } } return - } func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { From 156260e68e0836a149bd168c7a6814c6367d16fe Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 12 Sep 2018 13:20:07 -0500 Subject: [PATCH 44/65] roaring allowed import to merge with existing data --- fragment.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/fragment.go b/fragment.go index 904652a57..858a47e6c 100644 --- a/fragment.go +++ b/fragment.go @@ -1430,6 +1430,35 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { if err != nil { return err } + + // get a list of keys in order to update the cache + i, _ := bm.Containers.Iterator(0) + set := make([]uint64, 0) + var lastRow uint64 = math.MaxUint64 + + for i.Next() { + key, _ := i.Value() + + // virtual row for the current container + vRow := key >> containersPerRowSegment + + // skip dups + if vRow == lastRow { + continue + } + set = append(set, vRow) + lastRow = vRow + } + + if f.storage.Count() > 0 { + bm = f.storage.Union(bm) + } + + for _,rowID := range set { + n := bm.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) + f.cache.BulkAdd(rowID, n) + } + err = snapshot(f, bm) return err } From a3243f99e1e10f9e6fb12255c2e529fde7dfdb75 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 13:24:28 -0500 Subject: [PATCH 45/65] linter fixes - reorder return vals, remove unused const --- roaring/roaring.go | 10 ++++------ server/handler_test.go | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 6341453a4..9e62f399a 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3343,15 +3343,13 @@ func popcountAndSlice(s, m []uint64) uint64 { } // constants from github.com/RoaringBitmap/roaring -const ( //taken from roaring/util.go +// taken from roaring/util.go +const ( serialCookieNoRunContainer = 12346 // only arrays and bitmaps serialCookie = 12347 // runs, arrays, and bitmaps - noOffsetThreshold = 4 ) -//end roaring stuff - -func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, err error, haveRuns bool) { +func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { if len(buf) < 8 { err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) return @@ -3418,7 +3416,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } - keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) + keyN, containerTyper, header, pos, haveRuns, err := readStandardHeader(data) if err != nil { return errors.Wrap(err, "reading roaring header") } diff --git a/server/handler_test.go b/server/handler_test.go index 487756c45..0388dcec6 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -568,7 +568,8 @@ func TestHandler_Endpoints(t *testing.T) { if result.StatusCode != 200 { t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) } - if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { + + if result.Header["Access-Control-Allow-Origin"][0] != "http://test/" { t.Fatal("CORS header not present") } }) From b5a14de1fdfb93273ca509fd4bc2cd70308832b9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 16:11:51 -0500 Subject: [PATCH 46/65] roaring import: fix api doc, refactor/rename, test merge --- api.go | 4 +- fragment.go | 40 ++++++++++------- fragment_internal_test.go | 93 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 19 deletions(-) diff --git a/api.go b/api.go index 012e4bf3e..5bf8d0c21 100644 --- a/api.go +++ b/api.go @@ -297,8 +297,8 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // ImportRoaringBytes is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a -// particular way which may be unintuitive (discussed below), and overwrites any -// existing data in the particular fragment into which it is being loaded. +// particular way which may be unintuitive (discussed below). The data is merged +// with existing data. // // It takes as input a roaring bitmap which it uses as the data for the // indicated index, field, and shard. The bitmap may be encoded according to the diff --git a/fragment.go b/fragment.go index 858a47e6c..a1c5a38fe 100644 --- a/fragment.go +++ b/fragment.go @@ -44,15 +44,19 @@ import ( ) const ( - // ShardWidth is the number of column IDs in a shard. - ShardWidth = 1048576 + // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. + shardWidthExponent = 20 + ShardWidth = 1 << shardWidthExponent - // containersPerRowSegment is dependent upon ShardWidth, - // and it represents the number of containers per shard row - // (or rowSegment). Since containers are set in roaring - // to be 2^16, then this const should be ShardWidth / 2^16. - // It is represented as the exponent n of 2^n. - containersPerRowSegment = 4 + // shardVsContainerExponent is the power of 2 of ShardWith minus the power + // of two of roaring container width (which is 16). + // 2^shardVsContainerExponent is the number of containers in a shard row. + // + // It is represented in this rather awkward way because calculating the row + // which a given container is in means dividing by the number of rows per + // container which is performantly expressed as a right shift by this + // exponent. + shardVsContainerExponent = shardWidthExponent - 16 // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" @@ -1432,21 +1436,21 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { } // get a list of keys in order to update the cache - i, _ := bm.Containers.Iterator(0) - set := make([]uint64, 0) + iter, _ := bm.Containers.Iterator(0) + rowsInData := make([]uint64, 0) var lastRow uint64 = math.MaxUint64 - for i.Next() { - key, _ := i.Value() + for iter.Next() { + key, _ := iter.Value() // virtual row for the current container - vRow := key >> containersPerRowSegment + vRow := key >> shardVsContainerExponent // skip dups if vRow == lastRow { continue } - set = append(set, vRow) + rowsInData = append(rowsInData, vRow) lastRow = vRow } @@ -1454,7 +1458,7 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { bm = f.storage.Union(bm) } - for _,rowID := range set { + for _, rowID := range rowsInData { n := bm.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } @@ -1493,6 +1497,8 @@ func (f *fragment) snapshot() error { return snapshot(f, f.storage) } +// snapshot writes the fragment f with bm as the data. It is unprotected, and +// f.mu must be locked when calling it. func snapshot(f *fragment, bm *roaring.Bitmap) error { f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) @@ -1763,7 +1769,7 @@ func (f *fragment) rows() []uint64 { key, _ := i.Value() // virtual row for the current container - vRow := key >> containersPerRowSegment + vRow := key >> shardVsContainerExponent // skip dups if vRow == lastRow { @@ -1792,7 +1798,7 @@ func (f *fragment) rowsForColumn(columnID uint64) []uint64 { key, c := i.Value() // virtual row for the current container - vRow := key >> containersPerRowSegment + vRow := key >> shardVsContainerExponent // column container key for virtual row colKey = ((vRow * ShardWidth) + colID) >> 16 diff --git a/fragment_internal_test.go b/fragment_internal_test.go index c85b7e88c..db0125bbc 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -17,14 +17,17 @@ package pilosa import ( "bytes" "flag" + "fmt" "io/ioutil" "math" "reflect" + "sort" "testing" "testing/quick" "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/roaring" ) // Test flags @@ -1397,3 +1400,93 @@ func TestFragment_RowsIteration(t *testing.T) { } }) } + +// Test Importing roaring data. +func TestFragment_RoaringImport(t *testing.T) { + tests := [][][]uint64{ + { + []uint64{0}, + []uint64{1}, + }, + { + []uint64{0, 65535, 65536, 65537, 65538, 65539, 130000}, + []uint64{1000, 67000, 130000}, + }, + { + []uint64{0, 65535, 65536, 65537, 65538, 65539, 130000}, + []uint64{0, 65535, 65536, 65537, 65538, 65539, 130000}, + }, + { + []uint64{0, 65535, 65536, 1<<20 + 1, 1<<20*2 + 1}, + []uint64{1, 1<<20 + 2, 1<<20*2 + 2}, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + for num, input := range test { + buf := &bytes.Buffer{} + bm := roaring.NewBitmap(input...) + _, err := bm.WriteTo(buf) + if err != nil { + t.Fatalf("writing to buffer: %v", err) + } + f.importRoaringBytes(buf.Bytes()) + exp := calcExpected(test[:num+1]...) + for row, expCols := range exp { + cols := f.row(uint64(row)).Columns() + t.Logf("\nrow: %d\n exp:%v\n got:%v", row, expCols, cols) + if !reflect.DeepEqual(cols, expCols) { + t.Fatalf("input%d, row %d\n exp:%v\n got:%v", num, row, expCols, cols) + } + } + } + }) + } +} + +// calcExpected takes a number of slices of uint64 represented data to be added +// to a fragment. It calculates which rows and bits set in each row would be +// expected after importing that data, and returns a [][]uint64 where the index +// into the first slice is row id. +func calcExpected(inputs ...[]uint64) [][]uint64 { + // create map of row id to set of column values in that row. + rows := make(map[uint64]map[uint64]struct{}) + var maxrow uint64 = 0 + for _, input := range inputs { + for _, val := range input { + row := val / ShardWidth + if row > maxrow { + maxrow = row + } + m, ok := rows[row] + if !ok { + m = make(map[uint64]struct{}) + rows[row] = m + } + m[val%ShardWidth] = struct{}{} + } + } + + // initialize ret slice + ret := make([][]uint64, maxrow+1) + for i, _ := range ret { + ret[i] = make([]uint64, 0) + } + + // populate ret slices from rows map + for row, vals := range rows { + for val, _ := range vals { + ret[row] = append(ret[row], val) + } + } + + // sort ret slices + for _, slice := range ret { + sort.Slice(slice, func(i int, j int) bool { return slice[i] < slice[j] }) + } + + return ret +} From 14a83622617703b64ceaf62512eb73764fa68e49 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 16:48:07 -0500 Subject: [PATCH 47/65] roaring import - add cache invalidate and broken test --- fragment.go | 1 + fragment_internal_test.go | 107 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/fragment.go b/fragment.go index a1c5a38fe..827dda6e2 100644 --- a/fragment.go +++ b/fragment.go @@ -1462,6 +1462,7 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { n := bm.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } + f.cache.Invalidate() err = snapshot(f, bm) return err diff --git a/fragment_internal_test.go b/fragment_internal_test.go index db0125bbc..d4827f1d3 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1447,6 +1447,113 @@ func TestFragment_RoaringImport(t *testing.T) { } } +// Test Importing roaring data. +func TestFragment_RoaringImportTopN(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + rowIDs2 []uint64 + colIDs2 []uint64 + roaring []uint64 + }{ + { + rowIDs: []uint64{4, 4, 4, 4}, + colIDs: []uint64{0, 1, 2, 3}, + rowIDs2: []uint64{5, 5, 5, 5, 5}, + colIDs2: []uint64{0, 1, 2, 3, 4}, + roaring: []uint64{0, 65535, 65536, 1<<20 + 1, 1<<20 + 2, 1<<20*2 + 1}, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + expPairs := calcTop(test.rowIDs, test.colIDs) + pairs, err := f.top(topOptions{}) + if err != nil { + t.Fatalf("executing top after bulk import: %v", err) + } + if !reflect.DeepEqual(expPairs, pairs) { + t.Fatalf("post bulk import:\n exp: %v\n got: %v\n", expPairs, pairs) + } + + err = f.bulkImport(test.rowIDs2, test.colIDs2) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + test.rowIDs = append(test.rowIDs, test.rowIDs2...) + test.colIDs = append(test.colIDs, test.colIDs2...) + expPairs = calcTop(test.rowIDs, test.colIDs) + pairs, err = f.top(topOptions{}) + if err != nil { + t.Fatalf("executing top after bulk import: %v", err) + } + if !reflect.DeepEqual(expPairs, pairs) { + t.Fatalf("post bulk import2:\n exp: %v\n got: %v\n", expPairs, pairs) + } + + buf := &bytes.Buffer{} + bm := roaring.NewBitmap(test.roaring...) + _, err = bm.WriteTo(buf) + if err != nil { + t.Fatalf("writing to buffer: %v", err) + } + f.importRoaringBytes(buf.Bytes()) + rows, cols := toRowsCols(test.roaring) + expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) + pairs, err = f.top(topOptions{}) + if err != nil { + t.Fatalf("executing top after roaring import: %v", err) + } + if !reflect.DeepEqual(expPairs, pairs) { + t.Fatalf("post Roaring:\n exp: %v\n got: %v\n", expPairs, pairs) + } + }) + } +} + +func toRowsCols(roaring []uint64) (rowIDs, colIDs []uint64) { + rowIDs, colIDs = make([]uint64, len(roaring)), make([]uint64, len(roaring)) + for i, bit := range roaring { + rowIDs[i] = bit / ShardWidth + colIDs[i] = bit % ShardWidth + } + return rowIDs, colIDs +} + +func calcTop(rowIDs, colIDs []uint64) []Pair { + if len(rowIDs) != len(colIDs) { + panic("row and col ids must be of equal len") + } + // make map of rowID to colID set in order to dedup + counts := make(map[uint64]map[uint64]struct{}) + for i := 0; i < len(rowIDs); i++ { + row, col := rowIDs[i], colIDs[i] + m, ok := counts[row] + if !ok { + m = make(map[uint64]struct{}) + counts[row] = m + } + m[col] = struct{}{} + } + + // build slice of pairs from map + ret := make([]Pair, 0) + for row, cols := range counts { + ret = append(ret, Pair{ID: row, Count: uint64(len(cols))}) + } + + // reverse sort by count + sort.Slice(ret, func(i, j int) bool { return ret[i].Count > ret[j].Count }) + return ret +} + // calcExpected takes a number of slices of uint64 represented data to be added // to a fragment. It calculates which rows and bits set in each row would be // expected after importing that data, and returns a [][]uint64 where the index From 3ea07ae3a741435db2532846abd04036868cf843 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:07:46 -0500 Subject: [PATCH 48/65] use cache.Recalculate instead of Invalidate for imports Invalidate does not always rebuild the cache - if the last rebuild is < 10s ago, it does nothing. We always want to rebuild the cache after imports. Also updated the comments around recalculate/invalidate to clarify. --- cache.go | 4 ++-- fragment.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cache.go b/cache.go index c6821c393..8ae8caefc 100644 --- a/cache.go +++ b/cache.go @@ -40,10 +40,10 @@ type cache interface { // Returns a list of all IDs. IDs() []uint64 - // Updates the cache, if necessary. + // Soft ask for the cache to be rebuilt - may not if it has been done recently. Invalidate() - // Rebuilds the cache + // Rebuilds the cache. Recalculate() // Returns an ordered list of the top ranked bitmaps. diff --git a/fragment.go b/fragment.go index 827dda6e2..ef2a5bfab 100644 --- a/fragment.go +++ b/fragment.go @@ -1386,7 +1386,7 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.cache.BulkAdd(rowID, n) } - f.cache.Invalidate() + f.cache.Recalculate() return snapshot(f, results) } @@ -1462,7 +1462,7 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { n := bm.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } - f.cache.Invalidate() + f.cache.Recalculate() err = snapshot(f, bm) return err From f8569102f554ecaa5b67ba2d62d6017f834a9e4a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:10:28 -0500 Subject: [PATCH 49/65] update some fragment comments re: the cache --- fragment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index ef2a5bfab..c521d1187 100644 --- a/fragment.go +++ b/fragment.go @@ -1327,7 +1327,7 @@ func (f *fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, e } // bulkImport bulk imports a set of bits and then snapshots the storage. -// This does not affect the fragment's cache. +// The cache is updated to reflect the new data. func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { // Verify that there are an equal number of row ids and column ids. if len(rowIDs) != len(columnIDs) { @@ -1425,7 +1425,7 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error // importRoaringBytes imports from standard roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version -// of the roaring format. +// of the roaring format. The cache is updated to reflect the new data. func (f *fragment) importRoaringBytes(roaringBytes []byte) error { f.mu.Lock() defer f.mu.Unlock() From 70e9da3b753ffafb7bf17456deae1e02abd0e97c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:13:07 -0500 Subject: [PATCH 50/65] fmt -s fragment_internal_test --- fragment_internal_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index d4827f1d3..83966e657 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1579,13 +1579,13 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { // initialize ret slice ret := make([][]uint64, maxrow+1) - for i, _ := range ret { + for i := range ret { ret[i] = make([]uint64, 0) } // populate ret slices from rows map for row, vals := range rows { - for val, _ := range vals { + for val := range vals { ret[row] = append(ret[row], val) } } From 3bc9c66f5e7dc878acc0fb2410f53d9175198c0d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:21:17 -0500 Subject: [PATCH 51/65] remove some naked returns and simplify --- api.go | 8 +++----- roaring/roaring.go | 14 ++++---------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/api.go b/api.go index 5bf8d0c21..71fc039a7 100644 --- a/api.go +++ b/api.go @@ -314,8 +314,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // in one long bitmap. func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { if err = api.validate(apiField); err != nil { - err = errors.Wrap(err, "validating api method") - return + return errors.Wrap(err, "validating api method") } nodes := api.cluster.shardNodes(indexName, shard) var wg sync.WaitGroup @@ -324,8 +323,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, ind if node.ID == api.server.nodeID { field := api.holder.Field(indexName, fieldName) if field == nil { - err = newNotFoundError(ErrFieldNotFound) - return + return newNotFoundError(ErrFieldNotFound) } wg.Add(1) go func(node *Node) { @@ -344,7 +342,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, ind } wg.Wait() - return + return err } // DeleteField removes the named field from the named index. If the index is not diff --git a/roaring/roaring.go b/roaring/roaring.go index 9e62f399a..6f6103318 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3434,10 +3434,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Read container offsets and attach data. if haveRuns { - err := readWithRuns(b, data, pos, keyN) - if err != nil { - return errors.Wrap(err, "reading runs from standard roaring format") - } + readWithRuns(b, data, pos, keyN) } else { err := readOffsets(b, data, pos, keyN) if err != nil { @@ -3447,7 +3444,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return nil } -func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { +func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) error { citer, _ := b.Containers.Iterator(0) for i, buf := 0, data[pos:]; i < int(keyN); i, buf = i+1, buf[4:] { @@ -3473,11 +3470,10 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { return fmt.Errorf("unsupported container type %d", c.containerType) } } - return + return nil } -func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { - +func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { citer, _ := b.Containers.Iterator(0) for i := 0; i < int(keyN); i++ { citer.Next() @@ -3505,6 +3501,4 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { pos += bitmapN * 8 } } - return - } From d0d0bdd722b320c44f02c07e911e030d5bb0c953 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:25:12 -0500 Subject: [PATCH 52/65] simplify fragment.unprotectedRow (unparam) --- fragment.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/fragment.go b/fragment.go index c521d1187..6d92ce37f 100644 --- a/fragment.go +++ b/fragment.go @@ -338,15 +338,13 @@ func (f *fragment) closeStorage() error { func (f *fragment) row(rowID uint64) *Row { f.mu.Lock() defer f.mu.Unlock() - return f.unprotectedRow(rowID, true, true) + return f.unprotectedRow(rowID) } -func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { - if checkRowCache { - r, ok := f.rowCache.Fetch(rowID) - if ok && r != nil { - return r - } +func (f *fragment) unprotectedRow(rowID uint64) *Row { + r, ok := f.rowCache.Fetch(rowID) + if ok && r != nil { + return r } // Only use a subset of the containers. @@ -365,9 +363,7 @@ func (f *fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCac } row.invalidateCount() - if updateRowCache { - f.rowCache.Add(rowID, row) - } + f.rowCache.Add(rowID, row) return row } @@ -427,7 +423,7 @@ func (f *fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err } // Get the row from row cache or fragment.storage. - row := f.unprotectedRow(rowID, true, true) + row := f.unprotectedRow(rowID) row.SetBit(columnID) // Update the cache. @@ -479,7 +475,7 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er } // Get the row from cache or fragment.storage. - row := f.unprotectedRow(rowID, true, true) + row := f.unprotectedRow(rowID) row.clearBit(columnID) // Update the cache. From 4a5f12631c8bb89f4d89fd7adc4556ce61d0c695 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:29:35 -0500 Subject: [PATCH 53/65] more naked returns --- roaring/roaring.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 6f6103318..50936ceaa 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3352,14 +3352,14 @@ const ( func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { if len(buf) < 8 { err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) - return + return size, containerTyper, header, pos, haveRuns, err } cf := func(index uint, card int) (newType byte) { newType = containerBitmap if card < ArrayMaxSize { newType = containerArray } - return + return newType } containerTyper = cf cookie := binary.LittleEndian.Uint32(buf) @@ -3377,7 +3377,7 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint isRunBitmapSize := (int(size) + 7) / 8 if pos+isRunBitmapSize > len(buf) { err = fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) - return + return size, containerTyper, header, pos, haveRuns, err } isRunBitmap := buf[pos : pos+isRunBitmapSize] @@ -3390,22 +3390,22 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint } } else { err = fmt.Errorf("did not find expected serialCookie in header") - return + return size, containerTyper, header, pos, haveRuns, err } header = pos if size > (1 << 16) { err = fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") - return + return size, containerTyper, header, pos, haveRuns, err } // descriptive header if pos+2*2*int(size) > len(buf) { err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) - return + return size, containerTyper, header, pos, haveRuns, err } pos += 2 * 2 * int(size) // moving pos past keycount - return + return size, containerTyper, header, pos, haveRuns, err } // UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in From 79c783eaf80332633a178b8851b35dc4bbad77f2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 12 Sep 2018 17:31:57 -0500 Subject: [PATCH 54/65] add nolint directive there isn't much piont in making snapshot take a WriterTo, because the only thing that's going to be written to a fragment is a roaring bitmap. making it WriterTo just makes it slightly more annoying to jump into the Bitmap.WriterTo implementation. since it's a private function, it should be straightforward to change this in the future if the need arises. --- fragment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 6d92ce37f..0d2bc27f8 100644 --- a/fragment.go +++ b/fragment.go @@ -1496,7 +1496,7 @@ func (f *fragment) snapshot() error { // snapshot writes the fragment f with bm as the data. It is unprotected, and // f.mu must be locked when calling it. -func snapshot(f *fragment, bm *roaring.Bitmap) error { +func snapshot(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) From c6817340c765188724cbf9f188b096715db44836 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 10:31:58 -0500 Subject: [PATCH 55/65] rename sliceWidth->shardWidth --- api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 71fc039a7..205be064b 100644 --- a/api.go +++ b/api.go @@ -308,8 +308,8 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // // The data, roaringBytes, should be encoded the same way that Pilosa stores // fragments internally. A bit "i" being set in the input bitmap indicates that -// the bit is set in Pilosa row "i/sliceWidth", and in column -// (shard*sliceWidth)+(i%sliceWidth). That is to say that roaringBytes +// the bit is set in Pilosa row "i/ShardWidth", and in column +// (shard*ShardWidth)+(i%ShardWidth). That is to say that roaringBytes // represents all of the rows in this shard of this field concatenated together // in one long bitmap. func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { From f006df7758e3bbd4fda20cb4484f20ffad6e9dfe Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 10:39:00 -0500 Subject: [PATCH 56/65] re-order ImportRoaringBytes args to be more consistent --- api.go | 2 +- http/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 205be064b..d35bd2f3a 100644 --- a/api.go +++ b/api.go @@ -312,7 +312,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // (shard*ShardWidth)+(i%ShardWidth). That is to say that roaringBytes // represents all of the rows in this shard of this field concatenated together // in one long bitmap. -func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { +func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName string, shard uint64, roaringBytes []byte) (err error) { if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } diff --git a/http/handler.go b/http/handler.go index 822f242f3..e154b6db5 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1458,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard) + err = h.api.ImportRoaringBytes(r.Context(), indexName, fieldName, shard, body) // Marshal response object. msg := string("") From af7d405b11c631637c9cf785b22e9926566b416e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 10:41:36 -0500 Subject: [PATCH 57/65] add explanatory comment and move comment to correct spot --- fragment.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 0d2bc27f8..e52bffc42 100644 --- a/fragment.go +++ b/fragment.go @@ -1330,9 +1330,12 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } - // Disconnect op writer so we don't append updates. + // Create a temporary bitmap which will be populated by rowIDs and columnIDs + // and then merged into the existing fragment's bitmap. localBitmap := roaring.NewBitmap() + // Disconnect op writer so we don't append updates. localBitmap.OpWriter = nil + // Process every bit. // If an error occurs then reopen the storage. lastID := uint64(0) From 9e7cfeedc9ce137ebce2b08811423e87308066e5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 11:41:46 -0500 Subject: [PATCH 58/65] readability changes in fragment.go --- fragment.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/fragment.go b/fragment.go index e52bffc42..21ecbcd99 100644 --- a/fragment.go +++ b/fragment.go @@ -1339,7 +1339,7 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { // Process every bit. // If an error occurs then reopen the storage. lastID := uint64(0) - set := make(map[uint64]struct{}) + rowSet := make(map[uint64]struct{}) for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] @@ -1361,7 +1361,7 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { // no real danger if i == 0 || rowID != lastID { lastID = rowID - set[rowID] = struct{}{} + rowSet[rowID] = struct{}{} } // Invalidate block checksum. @@ -1372,7 +1372,7 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { defer f.mu.Unlock() //f.storage.Unmmap() - // Update cache counts for all rows. + // Merge localBitmap into fragment's existing data. var results *roaring.Bitmap if f.storage.Count() > 0 { results = f.storage.Union(localBitmap) @@ -1380,13 +1380,14 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { results = localBitmap } - for rowID := range set { + // Update cache counts for all affected rows. + for rowID := range rowSet { n := results.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } f.cache.Recalculate() - return snapshot(f, results) + return unprotectedWriteToFragment(f, results) } // importValue bulk imports a set of range-encoded values. @@ -1436,7 +1437,7 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { // get a list of keys in order to update the cache iter, _ := bm.Containers.Iterator(0) - rowsInData := make([]uint64, 0) + rowSet := make([]uint64, 0) var lastRow uint64 = math.MaxUint64 for iter.Next() { @@ -1449,7 +1450,7 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { if vRow == lastRow { continue } - rowsInData = append(rowsInData, vRow) + rowSet = append(rowSet, vRow) lastRow = vRow } @@ -1457,13 +1458,13 @@ func (f *fragment) importRoaringBytes(roaringBytes []byte) error { bm = f.storage.Union(bm) } - for _, rowID := range rowsInData { + for _, rowID := range rowSet { n := bm.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth) f.cache.BulkAdd(rowID, n) } f.cache.Recalculate() - err = snapshot(f, bm) + err = unprotectedWriteToFragment(f, bm) return err } @@ -1494,12 +1495,12 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *fragment) snapshot() error { - return snapshot(f, f.storage) + return unprotectedWriteToFragment(f, f.storage) } -// snapshot writes the fragment f with bm as the data. It is unprotected, and +// unprotectedWriteToFragment writes the fragment f with bm as the data. It is unprotected, and // f.mu must be locked when calling it. -func snapshot(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer +func unprotectedWriteToFragment(f *fragment, bm *roaring.Bitmap) error { // nolint: interfacer f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.shard) completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.shard) From fd655c998e3d7863bf1f601c70c7ff8bde825662 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 12:27:23 -0500 Subject: [PATCH 59/65] rename all instances of roaringbytes function names now importRoaring and byte slice args are just called data. Also renamed http endpoint from /importroaring to /import-roaring --- api.go | 20 ++++++++++---------- client.go | 4 ++-- field.go | 4 ++-- fragment.go | 6 +++--- fragment_internal_test.go | 4 ++-- http/client.go | 8 ++++---- http/handler.go | 4 ++-- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index d35bd2f3a..137fdfe1c 100644 --- a/api.go +++ b/api.go @@ -295,7 +295,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e return field, nil } -// ImportRoaringBytes is a low level interface for importing data to Pilosa when +// ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged // with existing data. @@ -306,13 +306,13 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // or to the pilosa roaring spec which supports 64 bit integers // (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format). // -// The data, roaringBytes, should be encoded the same way that Pilosa stores -// fragments internally. A bit "i" being set in the input bitmap indicates that -// the bit is set in Pilosa row "i/ShardWidth", and in column -// (shard*ShardWidth)+(i%ShardWidth). That is to say that roaringBytes -// represents all of the rows in this shard of this field concatenated together -// in one long bitmap. -func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName string, shard uint64, roaringBytes []byte) (err error) { +// The data should be encoded the same way that Pilosa stores fragments +// internally. A bit "i" being set in the input bitmap indicates that the bit is +// set in Pilosa row "i/ShardWidth", and in column +// (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all +// of the rows in this shard of this field concatenated together in one long +// bitmap. +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, data []byte) (err error) { if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } @@ -327,7 +327,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str } wg.Add(1) go func(node *Node) { - err = field.importRoaringBytes(roaringBytes, shard) + err = field.importRoaring(data, shard) wg.Done() }(node) } else { @@ -335,7 +335,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str //forward it on go func(node *Node) { //execute on node - err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes) + err = api.server.defaultClient.ImportRoaring(ctx, node, indexName, fieldName, shard, data) wg.Done() }(node) } diff --git a/client.go b/client.go index 7eb24d121..124b2c6c5 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error + ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/field.go b/field.go index 4223d3948..983ff7129 100644 --- a/field.go +++ b/field.go @@ -1055,7 +1055,7 @@ func (f *Field) importValue(columnIDs []uint64, values []int64) error { return nil } -func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error { +func (f *Field) importRoaring(data []byte, shard uint64) error { viewName := viewStandard view, err := f.createViewIfNotExists(viewName) @@ -1068,7 +1068,7 @@ func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error { return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaringBytes(stdRoaringBytes); err != nil { + if err := frag.importRoaring(data); err != nil { return err } diff --git a/fragment.go b/fragment.go index 21ecbcd99..19b0fdf5a 100644 --- a/fragment.go +++ b/fragment.go @@ -1423,14 +1423,14 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error return nil } -// importRoaringBytes imports from standard roaring data format defined at +// importRoaring imports from standard roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. -func (f *fragment) importRoaringBytes(roaringBytes []byte) error { +func (f *fragment) importRoaring(data []byte) error { f.mu.Lock() defer f.mu.Unlock() bm := roaring.NewBitmap() - err := bm.UnmarshalBinary(roaringBytes) + err := bm.UnmarshalBinary(data) if err != nil { return err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 83966e657..2fbb44124 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1433,7 +1433,7 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaringBytes(buf.Bytes()) + f.importRoaring(buf.Bytes()) exp := calcExpected(test[:num+1]...) for row, expCols := range exp { cols := f.row(uint64(row)).Columns() @@ -1504,7 +1504,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaringBytes(buf.Bytes()) + f.importRoaring(buf.Bytes()) rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) pairs, err = f.top(topOptions{}) diff --git a/http/client.go b/http/client.go index b3fd0ea83..e4e611c7b 100644 --- a/http/client.go +++ b/http/client.go @@ -525,20 +525,20 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui return buf, nil } -// ImportRoaringBytes fast import of raw bits in roaring standard format -func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte) error { +// ImportRoaring fast import of raw bits in roaring standard format +func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, data []byte) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - endpoint := fmt.Sprintf("/index/%s/field/%s/importroaring/%d", index, field, shard) + endpoint := fmt.Sprintf("/index/%s/field/%s/import-roaring/%d", index, field, shard) // Create URL. u := nodePathToURL(node, endpoint) // Generate HTTP request. - req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes)) + req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(data)) if err != nil { return errors.Wrap(err, "creating request") } diff --git a/http/handler.go b/http/handler.go index e154b6db5..f9a8c51c7 100644 --- a/http/handler.go +++ b/http/handler.go @@ -218,7 +218,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}/importroaring/{shard}", handler.handlePostRoaringImport).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostRoaringImport).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -1458,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), indexName, fieldName, shard, body) + err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, body) // Marshal response object. msg := string("") From 7b0d4d75b49ff489b72e6f369e2bbc2850834aaf Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 13:14:35 -0500 Subject: [PATCH 60/65] remove validators on import-roaring and test handler validators are for query args, not url vars. Also some misc cleanup and error handling in the handler. --- api.go | 4 +--- fragment.go | 1 - http/handler.go | 40 ++++++++++++++++++---------------------- server/handler_test.go | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/api.go b/api.go index 137fdfe1c..a1f1271ea 100644 --- a/api.go +++ b/api.go @@ -332,14 +332,12 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, }(node) } else { wg.Add(1) - //forward it on + // forward it on go func(node *Node) { - //execute on node err = api.server.defaultClient.ImportRoaring(ctx, node, indexName, fieldName, shard, data) wg.Done() }(node) } - } wg.Wait() return err diff --git a/fragment.go b/fragment.go index 19b0fdf5a..e551a2834 100644 --- a/fragment.go +++ b/fragment.go @@ -1371,7 +1371,6 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() - //f.storage.Unmmap() // Merge localBitmap into fragment's existing data. var results *roaring.Bitmap if f.storage.Count() > 0 { diff --git a/http/handler.go b/http/handler.go index f9a8c51c7..ac325de5c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -177,12 +177,12 @@ func (h *Handler) populateValidators() { h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") - h.validators["ImportRoaringBitmap"] = queryValidationSpecRequired("index", "field", "shard") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { if err := validator.validate(r.URL.Query()); err != nil { // TODO: Return the response depending on the Accept header @@ -218,7 +218,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostRoaringImport).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -1431,19 +1431,11 @@ func GetHTTPClient(t *tls.Config) *http.Client { } // handlPostRoaringImport -func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. +func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Content-Type") != "application/x-binary" { http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return - } /*else if validHeaderAcceptJSON(r.Header) { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - }*/ - - indexName := mux.Vars(r)["index"] - fieldName := mux.Vars(r)["field"] - shardName := mux.Vars(r)["shard"] + } // Read entire body. body, err := ioutil.ReadAll(r.Body) @@ -1452,25 +1444,29 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } - shard, err := strconv.ParseUint(shardName, 10, 64) + urlVars := mux.Vars(r) + shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) if err != nil { http.Error(w, "shard should be an unsigned integer", http.StatusBadRequest) return } - //TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, body) - // Marshal response object. - msg := string("") + // TODO give meaningful stats for import + err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, body) + resp := &pilosa.ImportResponse{} if err != nil { - msg = err.Error() + resp.Err = err.Error() } - buf, e := h.api.Serializer.Marshal(&pilosa.ImportResponse{Err: msg}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response"), http.StatusInternalServerError) + // Marshal response object. + buf, err := h.api.Serializer.Marshal(resp) + if err != nil { + http.Error(w, fmt.Sprintf("marshal import response: %v", err), http.StatusInternalServerError) return } // Write response. - w.Write(buf) + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing import-roaring response: %v", err) + } } diff --git a/server/handler_test.go b/server/handler_test.go index 0388dcec6..2c1aa9868 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -16,6 +16,8 @@ package server_test import ( "bytes" + "context" + "encoding/hex" "encoding/json" "fmt" "io" @@ -86,6 +88,22 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("ImportRoaring", func(t *testing.T) { + w := httptest.NewRecorder() + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + req := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(roaringData)) + req.Header.Set("Content-Type", "application/x-binary") + h.ServeHTTP(w, req) + resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) + if err != nil { + t.Fatalf("querying: %v", err) + } + if !reflect.DeepEqual(resp.Results[0], []pilosa.Pair{{Count: 12, ID: 0}}) { + t.Fatalf("Unexpected result %v", resp.Results[0]) + } + + }) + t.Run("Status", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) From b2ec5e373e87958eab75ba55cdfb7657a06b3ee1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 13:33:51 -0500 Subject: [PATCH 61/65] fixup explanatory comment --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 50936ceaa..2a2ed716f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3485,7 +3485,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { runCount := binary.LittleEndian.Uint16(data[pos : pos+runCountHeaderSize]) c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[pos+runCountHeaderSize]))[:runCount] - for o := range c.runs { //need to convert to start:end vs start:length :( + for o := range c.runs { // must convert from start:length to start:end :( c.runs[o].last = c.runs[o].start + c.runs[o].last } pos += int((runCount * interval16Size) + runCountHeaderSize) From b412309447644008bb6021d1e4540caafb4b120e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 13:38:18 -0500 Subject: [PATCH 62/65] rename standard roaring to "official" throughout --- api.go | 2 +- fragment.go | 2 +- http/client.go | 3 ++- roaring/roaring.go | 8 ++++---- roaring/roaring_internal_test.go | 8 ++++---- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/api.go b/api.go index a1f1271ea..ffddc0702 100644 --- a/api.go +++ b/api.go @@ -302,7 +302,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // // It takes as input a roaring bitmap which it uses as the data for the // indicated index, field, and shard. The bitmap may be encoded according to the -// standard roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec), +// official roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec), // or to the pilosa roaring spec which supports 64 bit integers // (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format). // diff --git a/fragment.go b/fragment.go index e551a2834..7f651b135 100644 --- a/fragment.go +++ b/fragment.go @@ -1422,7 +1422,7 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error return nil } -// importRoaring imports from standard roaring data format defined at +// importRoaring imports from the official roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. func (f *fragment) importRoaring(data []byte) error { diff --git a/http/client.go b/http/client.go index e4e611c7b..1ab8bd04b 100644 --- a/http/client.go +++ b/http/client.go @@ -525,7 +525,8 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui return buf, nil } -// ImportRoaring fast import of raw bits in roaring standard format +// ImportRoaring does fast import of raw bits in roaring format (pilosa or +// official format, see API.ImportRoaring). func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, data []byte) error { if index == "" { return pilosa.ErrIndexRequired diff --git a/roaring/roaring.go b/roaring/roaring.go index 2a2ed716f..41710de89 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3349,7 +3349,7 @@ const ( serialCookie = 12347 // runs, arrays, and bitmaps ) -func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { +func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) { if len(buf) < 8 { err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) return size, containerTyper, header, pos, haveRuns, err @@ -3409,14 +3409,14 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint } // UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in -// either standard roaring format or Pilosa's roaring format. +// either official roaring format or Pilosa's roaring format. func (b *Bitmap) UnmarshalBinary(data []byte) error { fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) if fileMagic == magicNumber { // if pilosa roaring return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } - keyN, containerTyper, header, pos, haveRuns, err := readStandardHeader(data) + keyN, containerTyper, header, pos, haveRuns, err := readOfficialHeader(data) if err != nil { return errors.Wrap(err, "reading roaring header") } @@ -3438,7 +3438,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { } else { err := readOffsets(b, data, pos, keyN) if err != nil { - return errors.Wrap(err, "reading offsets from standard roaring format") + return errors.Wrap(err, "reading offsets from official roaring format") } } return nil diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index caea8e2c1..b049d40cf 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3228,13 +3228,13 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { return nil } -func TestUnmarshalStdRoaring(t *testing.T) { +func TestUnmarshalOfficialRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays rbContainerWithTwoArrays, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") bm := NewBitmap() er := bm.UnmarshalBinary(rbContainerWithTwoArrays) if er != nil { - t.Fatalf("UnmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalOfficialRoaring %s", er) } if bm.Count() != 4 { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 65537]", bm.Slice()) @@ -3244,7 +3244,7 @@ func TestUnmarshalStdRoaring(t *testing.T) { bm = NewBitmap() er = bm.UnmarshalBinary(rbContainerWithRLEandArray) if er != nil { - t.Fatalf("UnmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalOfficialRoaring %s", er) } if bm.Count() != 11 { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) @@ -3254,7 +3254,7 @@ func TestUnmarshalStdRoaring(t *testing.T) { bm = NewBitmap() er = bm.UnmarshalBinary(_bitmap_array_container) if er != nil { - t.Fatalf("UnmarshalStandardRoaring %s", er) + t.Fatalf("UnmarshalOfficialRoaring %s", er) } if bm.Count() != 10000 { t.Fatalf("expecting X got %d", bm.Count()) From f943c17e0b0dfe8ca1ea4c163db8fdca2d937f31 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 15:03:31 -0500 Subject: [PATCH 63/65] ImportRoaring, add remote arg, fix data copy bug also, handle err properly in client method instead of discarding. --- api.go | 11 ++++++---- client.go | 4 ++-- http/client.go | 20 +++++++++++------ http/client_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++++ http/handler.go | 8 ++++++- 5 files changed, 82 insertions(+), 14 deletions(-) diff --git a/api.go b/api.go index ffddc0702..a369f3d4b 100644 --- a/api.go +++ b/api.go @@ -312,7 +312,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all // of the rows in this shard of this field concatenated together in one long // bitmap. -func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, data []byte) (err error) { +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, data []byte) (err error) { if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } @@ -326,15 +326,18 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return newNotFoundError(ErrFieldNotFound) } wg.Add(1) + // must make a copy of data to operate on locally. field.importRoaring changes data + d2 := make([]byte, len(data)) + copy(d2, data) go func(node *Node) { - err = field.importRoaring(data, shard) + err = field.importRoaring(d2, shard) wg.Done() }(node) - } else { + } else if !remote { // if remote == true we don't forward to other nodes wg.Add(1) // forward it on go func(node *Node) { - err = api.server.defaultClient.ImportRoaring(ctx, node, indexName, fieldName, shard, data) + err = api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data) wg.Done() }(node) } diff --git a/client.go b/client.go index 124b2c6c5..a2ca0bf96 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error + ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/http/client.go b/http/client.go index 1ab8bd04b..922ac6234 100644 --- a/http/client.go +++ b/http/client.go @@ -527,23 +527,24 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, data []byte) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, data []byte) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } + if uri == nil { + uri = c.defaultURI + } - endpoint := fmt.Sprintf("/index/%s/field/%s/import-roaring/%d", index, field, shard) + url := fmt.Sprintf("%s/index/%s/field/%s/import-roaring/%d?remote=%v", uri, index, field, shard, remote) - // Create URL. - u := nodePathToURL(node, endpoint) // Generate HTTP request. - req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) if err != nil { return errors.Wrap(err, "creating request") } - req.Header.Set("Accept", "application/x-binary") + req.Header.Set("Content-Type", "application/x-binary") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. @@ -557,8 +558,13 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, i if resp.StatusCode != http.StatusOK { return fmt.Errorf("invalid status: %d", resp.StatusCode) } - io.Copy(ioutil.Discard, resp.Body) + dec := json.NewDecoder(resp.Body) + rbody := &pilosa.ImportResponse{} + dec.Decode(rbody) + if rbody.Err != "" { + return errors.Errorf("importing roaring: %v", rbody.Err) + } return nil } diff --git a/http/client_test.go b/http/client_test.go index 131161d3c..fdb677265 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -18,6 +18,7 @@ import ( "bufio" "bytes" "context" + "encoding/hex" "fmt" gohttp "net/http" "reflect" @@ -363,6 +364,58 @@ func TestClient_Import(t *testing.T) { } } +// Ensure client can bulk import data. +func TestClient_ImportRoaring(t *testing.T) { + cluster := test.MustNewCluster(t, 2) + for _, c := range cluster { + c.Config.Cluster.ReplicaN = 2 + } + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + _, err = cluster[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = cluster[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + _, err = cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"}) + if err != nil { + t.Fatalf("querying: %v", err) + } + + // Send import request. + host := cluster[0].URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringData); err != nil { + t.Fatal(err) + } + + hldr := test.Holder{Holder: cluster[0].Server.Holder()} + // Verify data. + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) { + t.Fatalf("unexpected columns: %+v", a) + } + if a := hldr.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) { + t.Fatalf("unexpected columns: %+v", a) + } + + hldr2 := test.Holder{Holder: cluster[1].Server.Holder()} + // Verify data. + if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) { + t.Fatalf("unexpected columns: %+v", a) + } + if a := hldr2.Row("i", "f", 1).Columns(); !reflect.DeepEqual(a, []uint64{0}) { + t.Fatalf("unexpected columns: %+v", a) + } +} + // Ensure client can bulk import data. func TestClient_ImportKeys(t *testing.T) { t.Run("SingleNode", func(t *testing.T) { diff --git a/http/handler.go b/http/handler.go index ac325de5c..bc108568a 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1436,6 +1436,12 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return } + q := r.URL.Query() + remoteStr := q.Get("remote") + var remote bool + if remoteStr == "true" { + remote = true + } // Read entire body. body, err := ioutil.ReadAll(r.Body) @@ -1452,7 +1458,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } // TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, body) + err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body) resp := &pilosa.ImportResponse{} if err != nil { resp.Err = err.Error() From 870c5e131f3dd079fcaf0691ab2f77bce1026424 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 15:27:00 -0500 Subject: [PATCH 64/65] replace waitgroup with errgroup to avoid race on err --- api.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/api.go b/api.go index a369f3d4b..0f99f700f 100644 --- a/api.go +++ b/api.go @@ -24,12 +24,12 @@ import ( "io/ioutil" "strconv" "strings" - "sync" "time" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // API provides the top level programmatic interface to Pilosa. It is usually @@ -317,33 +317,31 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return errors.Wrap(err, "validating api method") } nodes := api.cluster.shardNodes(indexName, shard) - var wg sync.WaitGroup + var eg errgroup.Group for _, node := range nodes { + node := node if node.ID == api.server.nodeID { field := api.holder.Field(indexName, fieldName) if field == nil { return newNotFoundError(ErrFieldNotFound) } - wg.Add(1) // must make a copy of data to operate on locally. field.importRoaring changes data d2 := make([]byte, len(data)) copy(d2, data) + eg.Go(func() error { + return field.importRoaring(d2, shard) + }) go func(node *Node) { - err = field.importRoaring(d2, shard) - wg.Done() }(node) } else if !remote { // if remote == true we don't forward to other nodes - wg.Add(1) // forward it on - go func(node *Node) { - err = api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data) - wg.Done() - }(node) + eg.Go(func() error { + return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data) + }) } } - wg.Wait() - return err + return eg.Wait() } // DeleteField removes the named field from the named index. If the index is not From 2223b23b958f0ee734715f95742ebd1f347f5436 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 14 Sep 2018 09:40:48 -0500 Subject: [PATCH 65/65] re-generate the apimethod stringer --- apimethod_string.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apimethod_string.go b/apimethod_string.go index f3af365f0..01217092f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -2,7 +2,7 @@ package pilosa -import "fmt" +import "strconv" const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews" @@ -10,7 +10,7 @@ var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { - return fmt.Sprintf("apiMethod(%d)", i) + return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")" } return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] }