scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add support for Bool fields

Browse source 
prevent import of non-boolean row values to bool fields
This commit is contained in:
Travis Turner 2018-09-18 10:34:31 -05:00
parent 6f304b1a98
commit 4f17dbdbf1
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
14 changed files with 375 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
ctl/import_test.go
View file

@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
}
// Ensure that import into bool field runs.
func TestImportCommand_RunBool(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
ctx := context.Background()
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.API.Node().URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`)))
cm.Index = "i"
cm.Field = "f"
t.Run("Valid", func(t *testing.T) {
file, err := ioutil.TempFile("", "import-bool.csv")
if err != nil {
t.Fatal(err)
}
file.Write([]byte("0,1\n1,2\n1,3"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run to bool field doesn't work: %s", err)
}
})
// Ensure that invalid bool values return an error.
t.Run("Invalid", func(t *testing.T) {
file, err := ioutil.TempFile("", "import-invalid-bool.csv")
if err != nil {
t.Fatal(err)
}
file.Write([]byte("0,1\n1,2\n1,3\n2,4"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") {
t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err)
}
})
}

14
docs/administration.md
View file

@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the
pilosa import -i project -f stargazer-counts project-stargazer-counts.csv
```
##### Importing Boolean Values
If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true).
For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row.
```
0,3
0,9
1,1
1,2
1,4
1,8
```
<div class="note">
<p>Note that you must first create a field. View <a href="../api-reference/#create-field">Create Field</a> for more details. The `-e` flag can create the necessary schema when using a field of type "set".</p>
</div>

2
docs/api-reference.md
View file

@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option
* `int`
* `min` (int): Minimum integer value allowed for the field.
* `max` (int): Maximum integer value allowed for the field.
* `bool`
* (boolean fields take no arguments)
* `time`
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field.
* `mutex`

6
docs/data-model.md
View file

@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust
### Field Type
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`.
Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`.
#### Set
@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00)
#### Mutex
Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field.
#### Boolean
A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values.

2
docs/glossary.md
View file

@ -22,7 +22,7 @@ nav = []
<strong id="fragment">Fragment:</strong> A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="field">[Field](../data-model/#field):</strong> Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field).
<strong id="frame">[Frame](../data-model/#field):</strong> Prior to Pilosa 1.0, fields were known as frames.

2
docs/tutorials.md
View file

@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \
{"success":true}
```
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system:
``` request

75
executor.go
View file

@ -734,8 +734,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca
rowID, rowOK, rowErr := c.UintArg(fieldName)
if rowErr != nil {
return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr)
}
if !rowOK {
} else if !rowOK {
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
@ -1140,7 +1139,6 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
// executeSet executes a Set() call.
func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
// Read colID.
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
@ -1172,8 +1170,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
}
}
// Int field.
if f.Type() == FieldTypeInt {
// Read remaining fields using labels.
// Read row value.
rowVal, ok, err := c.IntArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
@ -1182,27 +1181,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
}
return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt)
} else {
// Read remaining fields using labels.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// Read row ID.
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
return false, fmt.Errorf("invalid date: %s", sTimestamp)
}
timestamp = &t
}
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// executeSetBitField executes a Set() call for a specific field.
@ -1675,7 +1674,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error {
// will raise an error downstream when it's used.
return nil
}
if field.keys() {
// Bool field keys do not use the translater because there
// are only two possible values. Instead, they are handled
// directly.
if field.Type() == FieldTypeBool {
boolVal, err := callArgBool(c, rowKey)
if err != nil {
return errors.Wrap(err, "getting bool key")
}
rowID := falseRowID
if boolVal {
rowID = trueRowID
}
c.Args[rowKey] = rowID
} else if field.keys() {
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
return errors.New("row value must be a string when field 'keys' option enabled")
}
@ -1832,6 +1845,18 @@ func (vc *ValCount) larger(other ValCount) ValCount {
}
}
func callArgBool(call *pql.Call, key string) (bool, error) {
value, ok := call.Args[key]
if !ok {
return false, errors.New("missing bool argument")
}
b, ok := value.(bool)
if !ok {
return false, fmt.Errorf("invalid bool argument type: %T", value)
}
return b, nil
}
func callArgString(call *pql.Call, key string) string {
value, ok := call.Args[key]
if !ok {

74
executor_test.go
View file

@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
})
}
// Ensure a set query can be executed on a bool field.
func TestExecutor_Execute_SetBool(t *testing.T) {
t.Run("Basic", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
t.Fatal(err)
}
// Set a true bit.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
t.Fatal(err)
} else if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
// Set the same bit to true again verify nothing changed.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil {
t.Fatal(err)
} else if res.Results[0].(bool) {
t.Fatalf("expected column to be unchanged")
}
// Set the same bit to false.
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil {
t.Fatal(err)
} else if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
// Ensure that the false row is set.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil {
t.Fatal(err)
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) {
t.Fatalf("unexpected colums: %+v", columns)
}
// Ensure that the true row is empty.
if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil {
t.Fatal(err)
} else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected colums: %+v", columns)
}
})
t.Run("Error", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil {
t.Fatal(err)
}
// Set bool using a string value.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil {
t.Fatalf("expected invalid bool type error")
}
// Set bool using an integer.
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil {
t.Fatalf("expected invalid bool type error")
}
})
}
// Ensure old PQL syntax doesn't break anything too badly.
func TestExecutor_Execute_OldPQL(t *testing.T) {
c := test.MustRunCluster(t, 1)
@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
// Create felds.
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil {
t.Fatal(err)

37
field.go
View file

@ -52,6 +52,7 @@ const (
FieldTypeInt = "int"
FieldTypeTime = "time"
FieldTypeMutex = "mutex"
FieldTypeBool = "bool"
)
// Field represents a container for views.
@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption {
}
}
func OptFieldTypeBool() FieldOption {
return func(fo *FieldOptions) error {
if fo.Type != "" {
return errors.Errorf("field type is already set to: %s", fo.Type)
}
fo.Type = FieldTypeBool
return nil
}
}
// NewField returns a new instance of field.
func NewField(path, index, name string, opts FieldOption) (*Field, error) {
err := validateName(name)
@ -441,6 +452,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
f.options.Max = 0
f.options.TimeQuantum = ""
f.options.Keys = opt.Keys
case FieldTypeBool:
f.options.Type = FieldTypeBool
f.options.CacheType = CacheTypeNone
f.options.CacheSize = 0
f.options.Min = 0
f.options.Max = 0
f.options.TimeQuantum = ""
f.options.Keys = false
default:
return errors.New("invalid field type")
}
@ -681,6 +700,10 @@ func (f *Field) deleteView(name string) error {
}
// Row returns a row of the standard view.
// It seems this method is only being used by the test
// package, and the fact that it's only allowed on
// `set` fields is odd. This may be considered for
// deprecation in a future version.
func (f *Field) Row(rowID uint64) (*Row, error) {
if f.Type() != FieldTypeSet {
return nil, errors.Errorf("row method unsupported for field type: %s", f.Type())
@ -954,10 +977,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
return errors.New("time quantum not set in field")
}
fieldType := f.Type()
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
// Bool-specific data validation.
if fieldType == FieldTypeBool && rowID > 1 {
return errors.New("bool field imports only support values 0 and 1")
}
var timestamp *time.Time
if len(timestamps) > i {
timestamp = timestamps[i]
@ -1191,6 +1222,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
o.CacheSize,
o.Keys,
})
case FieldTypeBool:
return json.Marshal(struct {
Type string `json:"type"`
}{
o.Type,
})
}
return nil, errors.New("invalid field type")
}

34
fragment.go
View file

@ -72,6 +72,10 @@ const (
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 2000
// Row ids used for boolean fields.
falseRowID = uint64(0)
trueRowID = uint64(1)
)
// fragment represents the intersection of a field and shard in an index.
@ -2206,3 +2210,33 @@ func (v *rowsVector) Get(colID uint64) (uint64, bool) {
// Set is not used for rowsVector.
func (v *rowsVector) Set(colID, rowID uint64) {}
// boolVector implements the vector interface by looking
// at data in rows 0 and 1.
type boolVector struct {
f *fragment
}
// newBoolVector returns a boolVector for a given fragment.
func newBoolVector(f *fragment) *boolVector {
return &boolVector{
f: f,
}
}
// Get returns the rowID associated to the given colID.
// Additionally, it returns true if a value was found,
// otherwise it returns false.
func (v *boolVector) Get(colID uint64) (uint64, bool) {
rows := v.f.rowsForColumn(colID)
if len(rows) == 1 {
switch rows[0] {
case falseRowID, trueRowID:
return rows[0], true
}
}
return 0, false
}
// Set is not used for boolVector.
func (v *boolVector) Set(colID, rowID uint64) {}

77
fragment_internal_test.go
View file

@ -1248,6 +1248,76 @@ func TestFragment_ImportMutex(t *testing.T) {
}
}
// Ensure a fragment can import bool values.
func TestFragment_ImportBool(t *testing.T) {
tests := []struct {
rowIDs []uint64
colIDs []uint64
exp map[uint64][]uint64
}{
{
[]uint64{1, 1, 1, 1},
[]uint64{0, 1, 2, 3},
map[uint64][]uint64{
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 0, 0, 0, 1, 1, 1, 1},
[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
map[uint64][]uint64{
0: {},
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 0, 0, 0, 1},
[]uint64{0, 1, 2, 3, 1},
map[uint64][]uint64{
0: {0, 2, 3},
1: {1},
},
},
{
[]uint64{1, 1, 1, 1, 0, 0, 1},
[]uint64{0, 1, 2, 3, 1, 8, 1},
map[uint64][]uint64{
0: {8},
1: {0, 1, 2, 3},
},
},
{
[]uint64{0, 1, 2},
[]uint64{8, 8, 8},
map[uint64][]uint64{
0: {},
1: {}, // This isn't {8} because fragment doesn't validate bool values.
2: {8},
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
f := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
defer f.Close()
err := f.bulkImport(test.rowIDs, test.colIDs)
if err != nil {
t.Fatalf("bulk importing ids: %v", err)
}
// Check for expected results.
for k, v := range test.exp {
cols := f.row(k).Columns()
if !reflect.DeepEqual(cols, v) {
t.Fatalf("expected: %v, but got: %v", v, cols)
}
}
})
}
}
func BenchmarkFragment_Snapshot(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
@ -1372,6 +1442,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st
return frag
}
// mustOpenBoolFragment returns a new instance of Fragment for a bool field.
func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment {
frag := mustOpenFragment(index, field, view, shard, cacheType)
frag.mutexVector = newBoolVector(frag)
return frag
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *fragment) reopen() error {
if err := f.Close(); err != nil {

16
http/handler.go
View file

@ -687,6 +687,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum))
case pilosa.FieldTypeMutex:
fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize))
case pilosa.FieldTypeBool:
fos = append(fos, pilosa.OptFieldTypeBool())
}
if req.Options.Keys != nil {
if *req.Options.Keys {
@ -778,6 +780,20 @@ func (o *fieldOptions) validate() error {
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
}
case pilosa.FieldTypeBool:
if o.CacheType != nil {
return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool"))
} else if o.CacheSize != nil {
return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool"))
} else if o.Min != nil {
return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool"))
} else if o.Max != nil {
return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool"))
} else if o.TimeQuantum != nil {
return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
} else if o.Keys != nil {
return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
}
default:
return errors.Errorf("invalid field type: %s", o.Type)
}

17
pql/ast.go
View file

@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) {
return "", fmt.Errorf("No field argument specified")
}
// BoolArg is for reading the value at key from call.Args as a bool. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a bool. An error is
// returned if the value is not a bool.
func (c *Call) BoolArg(key string) (bool, bool, error) {
val, ok := c.Args[key]
if !ok {
return false, false, nil
}
switch tval := val.(type) {
case bool:
return tval, true, nil
default:
return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval)
}
}
// UintArg is for reading the value at key from call.Args as a uint64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a uint64 or an int64 and

2
view.go
View file

@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
if v.fieldType == FieldTypeMutex {
frag.mutexVector = newRowsVector(frag)
} else if v.fieldType == FieldTypeBool {
frag.mutexVector = newBoolVector(frag)
}
return frag
}