diff --git a/ctl/import_test.go b/ctl/import_test.go index cd75a5d0a..2913005fb 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { t.Fatalf("Import Run with values doesn't work: %s", err) } } + +// Ensure that import into bool field runs. +func TestImportCommand_RunBool(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + ctx := context.Background() + + cmd := test.MustRunCluster(t, 1)[0] + cm.Host = cmd.API.Node().URI.HostPort() + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + + cm.Index = "i" + cm.Field = "f" + + t.Run("Valid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run to bool field doesn't work: %s", err) + } + }) + + // Ensure that invalid bool values return an error. + t.Run("Invalid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-invalid-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3\n2,4")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") { + t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err) + } + }) +} diff --git a/docs/administration.md b/docs/administration.md index cd4e30184..e626cc0e5 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the pilosa import -i project -f stargazer-counts project-stargazer-counts.csv ``` +##### Importing Boolean Values + +If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true). + +For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row. +``` +0,3 +0,9 +1,1 +1,2 +1,4 +1,8 +``` +

Note that you must first create a field. View Create Field for more details. The `-e` flag can create the necessary schema when using a field of type "set".

diff --git a/docs/api-reference.md b/docs/api-reference.md index 890b37941..216a01bca 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option * `int` * `min` (int): Minimum integer value allowed for the field. * `max` (int): Maximum integer value allowed for the field. +* `bool` + * (boolean fields take no arguments) * `time` * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field. * `mutex` diff --git a/docs/data-model.md b/docs/data-model.md index ba7c12024..bababdd34 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust ### Field Type -Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`. +Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`. #### Set @@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00) #### Mutex Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field. + +#### Boolean + +A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values. diff --git a/docs/glossary.md b/docs/glossary.md index c39a0bf6f..fccbe0ae8 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -22,7 +22,7 @@ nav = [] Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). [Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. diff --git a/docs/tutorials.md b/docs/tutorials.md index 040d85a1c..47344db4f 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \ {"success":true} ``` -Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. +Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system: ``` request diff --git a/executor.go b/executor.go index b0083a514..4ae5ac81e 100644 --- a/executor.go +++ b/executor.go @@ -734,8 +734,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca rowID, rowOK, rowErr := c.UintArg(fieldName) if rowErr != nil { return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) - } - if !rowOK { + } else if !rowOK { return nil, fmt.Errorf("Row() must specify %v", rowLabel) } @@ -1140,7 +1139,6 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { - // Read colID. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { @@ -1172,8 +1170,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } } + // Int field. if f.Type() == FieldTypeInt { - // Read remaining fields using labels. + // Read row value. rowVal, ok, err := c.IntArg(fieldName) if err != nil { return false, fmt.Errorf("reading Set() row: %v", err) @@ -1182,27 +1181,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) - } else { - // Read remaining fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) - if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) - } - timestamp = &t - } - - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } + + // Read row ID. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } // executeSetBitField executes a Set() call for a specific field. @@ -1675,7 +1674,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { // will raise an error downstream when it's used. return nil } - if field.keys() { + + // Bool field keys do not use the translater because there + // are only two possible values. Instead, they are handled + // directly. + if field.Type() == FieldTypeBool { + boolVal, err := callArgBool(c, rowKey) + if err != nil { + return errors.Wrap(err, "getting bool key") + } + rowID := falseRowID + if boolVal { + rowID = trueRowID + } + c.Args[rowKey] = rowID + } else if field.keys() { if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { return errors.New("row value must be a string when field 'keys' option enabled") } @@ -1832,6 +1845,18 @@ func (vc *ValCount) larger(other ValCount) ValCount { } } +func callArgBool(call *pql.Call, key string) (bool, error) { + value, ok := call.Args[key] + if !ok { + return false, errors.New("missing bool argument") + } + b, ok := value.(bool) + if !ok { + return false, fmt.Errorf("invalid bool argument type: %T", value) + } + return b, nil +} + func callArgString(call *pql.Call, key string) string { value, ok := call.Args[key] if !ok { diff --git a/executor_test.go b/executor_test.go index ee919f001..828a40b09 100644 --- a/executor_test.go +++ b/executor_test.go @@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) } +// Ensure a set query can be executed on a bool field. +func TestExecutor_Execute_SetBool(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set a true bit. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Set the same bit to true again verify nothing changed. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if res.Results[0].(bool) { + t.Fatalf("expected column to be unchanged") + } + + // Set the same bit to false. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Ensure that the false row is set. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) { + t.Fatalf("unexpected colums: %+v", columns) + } + + // Ensure that the true row is empty. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected colums: %+v", columns) + } + }) + t.Run("Error", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set bool using a string value. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + // Set bool using an integer. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + }) +} + // Ensure old PQL syntax doesn't break anything too badly. func TestExecutor_Execute_OldPQL(t *testing.T) { c := test.MustRunCluster(t, 1) @@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - // Create felds. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { t.Fatal(err) diff --git a/field.go b/field.go index d038b606c..822522bc4 100644 --- a/field.go +++ b/field.go @@ -52,6 +52,7 @@ const ( FieldTypeInt = "int" FieldTypeTime = "time" FieldTypeMutex = "mutex" + FieldTypeBool = "bool" ) // Field represents a container for views. @@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { } } +func OptFieldTypeBool() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeBool + return nil + } +} + // NewField returns a new instance of field. func NewField(path, index, name string, opts FieldOption) (*Field, error) { err := validateName(name) @@ -441,6 +452,14 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Max = 0 f.options.TimeQuantum = "" f.options.Keys = opt.Keys + case FieldTypeBool: + f.options.Type = FieldTypeBool + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + f.options.Keys = false default: return errors.New("invalid field type") } @@ -681,6 +700,10 @@ func (f *Field) deleteView(name string) error { } // Row returns a row of the standard view. +// It seems this method is only being used by the test +// package, and the fact that it's only allowed on +// `set` fields is odd. This may be considered for +// deprecation in a future version. func (f *Field) Row(rowID uint64) (*Row, error) { if f.Type() != FieldTypeSet { return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) @@ -954,10 +977,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.New("time quantum not set in field") } + fieldType := f.Type() + // Split import data by fragment. dataByFragment := make(map[importKey]importData) for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] + + // Bool-specific data validation. + if fieldType == FieldTypeBool && rowID > 1 { + return errors.New("bool field imports only support values 0 and 1") + } + var timestamp *time.Time if len(timestamps) > i { timestamp = timestamps[i] @@ -1191,6 +1222,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.CacheSize, o.Keys, }) + case FieldTypeBool: + return json.Marshal(struct { + Type string `json:"type"` + }{ + o.Type, + }) } return nil, errors.New("invalid field type") } diff --git a/fragment.go b/fragment.go index 7e3cd26d9..312b4af10 100644 --- a/fragment.go +++ b/fragment.go @@ -72,6 +72,10 @@ const ( // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. defaultFragmentMaxOpN = 2000 + + // Row ids used for boolean fields. + falseRowID = uint64(0) + trueRowID = uint64(1) ) // fragment represents the intersection of a field and shard in an index. @@ -2206,3 +2210,33 @@ func (v *rowsVector) Get(colID uint64) (uint64, bool) { // Set is not used for rowsVector. func (v *rowsVector) Set(colID, rowID uint64) {} + +// boolVector implements the vector interface by looking +// at data in rows 0 and 1. +type boolVector struct { + f *fragment +} + +// newBoolVector returns a boolVector for a given fragment. +func newBoolVector(f *fragment) *boolVector { + return &boolVector{ + f: f, + } +} + +// Get returns the rowID associated to the given colID. +// Additionally, it returns true if a value was found, +// otherwise it returns false. +func (v *boolVector) Get(colID uint64) (uint64, bool) { + rows := v.f.rowsForColumn(colID) + if len(rows) == 1 { + switch rows[0] { + case falseRowID, trueRowID: + return rows[0], true + } + } + return 0, false +} + +// Set is not used for boolVector. +func (v *boolVector) Set(colID, rowID uint64) {} diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e0865c606..87f4e6137 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1248,6 +1248,76 @@ func TestFragment_ImportMutex(t *testing.T) { } } +// Ensure a fragment can import bool values. +func TestFragment_ImportBool(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + exp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1, 1, 1, 1}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 0: {}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 0: {0, 2, 3}, + 1: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 0: {8}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 1, 2}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 0: {}, + 1: {}, // This isn't {8} because fragment doesn't validate bool values. + 2: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + // Check for expected results. + for k, v := range test.exp { + cols := f.row(k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func BenchmarkFragment_Snapshot(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -1372,6 +1442,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st return frag } +// mustOpenBoolFragment returns a new instance of Fragment for a bool field. +func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment { + frag := mustOpenFragment(index, field, view, shard, cacheType) + frag.mutexVector = newBoolVector(frag) + return frag +} + // Reopen closes the fragment and reopens it as a new instance. func (f *fragment) reopen() error { if err := f.Close(); err != nil { diff --git a/http/handler.go b/http/handler.go index bc108568a..820bc6521 100644 --- a/http/handler.go +++ b/http/handler.go @@ -687,6 +687,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) case pilosa.FieldTypeMutex: fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize)) + case pilosa.FieldTypeBool: + fos = append(fos, pilosa.OptFieldTypeBool()) } if req.Options.Keys != nil { if *req.Options.Keys { @@ -778,6 +780,20 @@ func (o *fieldOptions) validate() error { } else if o.TimeQuantum != nil { return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } default: return errors.Errorf("invalid field type: %s", o.Type) } diff --git a/pql/ast.go b/pql/ast.go index cd483e030..0e2ff6aef 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) { return "", fmt.Errorf("No field argument specified") } +// BoolArg is for reading the value at key from call.Args as a bool. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a bool. An error is +// returned if the value is not a bool. +func (c *Call) BoolArg(key string) (bool, bool, error) { + val, ok := c.Args[key] + if !ok { + return false, false, nil + } + switch tval := val.(type) { + case bool: + return tval, true, nil + default: + return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval) + } +} + // UintArg is for reading the value at key from call.Args as a uint64. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a uint64 or an int64 and diff --git a/view.go b/view.go index 2feac8b44..5fc0f5eb8 100644 --- a/view.go +++ b/view.go @@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment { frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) + } else if v.fieldType == FieldTypeBool { + frag.mutexVector = newBoolVector(frag) } return frag }