scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'input-definition' of github.com:pilosa/pilosa into input-definition

Browse source
This commit is contained in:
Linh Vo 2017-07-07 10:37:50 -05:00
commit 13aeaebe35
6 changed files with 132 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
docs/getting-started.md
View file

@ -140,7 +140,16 @@ curl localhost:10101/index/repository/input-definition/stargazer \
}
],
"name": "stargazer_id"
}
},
{
"actions": [
{
"frame": "stargazer",
"valueDestination": "set-timestamp"
}
],
"name": "time_value
}
]
}'
```
@ -151,7 +160,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel
- value-to-row: The value for this field is used as the `rowID`.
- single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type.
- mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type.
- set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame
#### Import Data Using an Input Definition
@ -167,6 +176,7 @@ curl localhost:10101/index/repository/input/stargazer \
"language_id": "Go",
"repo_id": 91720568,
"stargazer_id": 513114
"time_value": "2017-05-18T20:40"
},
{
"language_id": "Python",
@ -182,6 +192,7 @@ The data input above is equivalent to the following `SetBit()` operations:
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114)
'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40")
SetBit(frame="language", repo_id=91720568, language_id=5)
SetBit(frame="language", repo_id=95122322, language_id=17)
'

59
handler.go
View file

@ -1646,15 +1646,35 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
if err != nil {
return nil, err
}
// if field in input data is not in defined definition, return error
var columnLabel string
// If field in input data is not in defined definition, return error.
var colValue uint64
validFields := make(map[string]bool)
timestampFrame := make(map[string]int64)
for _, field := range inputDef.Fields() {
validFields[field.Name] = true
if field.PrimaryKey {
columnLabel = field.Name
columnLabel := field.Name
value, ok := req[columnLabel]
if !ok {
return nil, fmt.Errorf("columnLabel required")
}
rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float
if !ok {
return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value))
}
colValue = uint64(rawValue)
}
// Find frame that need to add timestamp.
for _, action := range field.Actions {
if action.ValueDestination == InputSetTimestamp {
timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name)
if err != nil {
return nil, err
}
}
}
}
for key := range req {
_, ok := validFields[key]
if !ok {
@ -1663,23 +1683,18 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
}
setBits := make(map[string][]*Bit)
for _, field := range inputDef.Fields() {
// skip field that defined in definition but not in input data
if _, ok := req[field.Name]; !ok {
continue
}
value, ok := req[columnLabel]
if !ok {
return nil, fmt.Errorf("columnLabel required")
}
colValue, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value))
}
// Looking into timestampFrame map and set timestamp to the whole frame
for _, action := range field.Actions {
frame := action.Frame
bit, err := HandleAction(action, req[field.Name], uint64(colValue))
timestamp := timestampFrame[action.Frame]
bit, err := HandleAction(action, req[field.Name], colValue, timestamp)
if err != nil {
return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err)
}
@ -1690,3 +1705,23 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index,
}
return setBits, nil
}
// GetTimeStamp retrieves unix timestamp from Input data.
func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) {
tmstamp, ok := data[timeField]
if !ok {
return 0, nil
}
timestamp, ok := tmstamp.(string)
if !ok {
return 0, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, has: %v", data[timeField])
}
v, err := time.Parse(TimeFormat, timestamp)
if err != nil {
return 0, err
}
return v.Unix(), nil
}

52
handler_test.go
View file

@ -1271,6 +1271,16 @@ var defaultBody = `
"frame":"foo",
"valueDestination":"value-to-row"
}
]
},
{
"name":"time_value",
"actions":[
{
"frame":"add-ons",
"valueDestination":"set-timestamp"
}
]
}
@ -1296,7 +1306,8 @@ func TestHandler_CreateInput(t *testing.T) {
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"withPet": true
"withPet": true,
"time_value": "2017-03-20T19:35"
}]`)
h := test.NewHandler()
h.Holder = hldr.Holder
@ -1318,6 +1329,7 @@ func TestHandler_CreateInput(t *testing.T) {
t.Fatalf("unexpected status code: %d", w.Code)
}
// Test successfully ingest data
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusOK {
@ -1327,7 +1339,6 @@ func TestHandler_CreateInput(t *testing.T) {
}
// Verify the bits set per frame.
// f := index.Frame("cab-type")
f0 := index.Frame("distance-miles")
v0 := f0.View(pilosa.ViewStandard)
fragment0 := v0.Fragment(0)
@ -1405,6 +1416,13 @@ func TestInput_JSON(t *testing.T) {
"noFrame": 1
}]`,
err: "Frame not found: foo"},
{json: `[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"time_value": 12345
}]`,
err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"},
}
h := test.NewHandler()
h.Holder = hldr.Holder
@ -1428,3 +1446,33 @@ func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error)
def.Name = name
return def, nil
}
func TestHandler_GetTimeStamp(t *testing.T) {
data := make(map[string]interface{})
timeField := "time"
data["time"] = "2017-03-20T19:35"
val, err := pilosa.GetTimeStamp(data, timeField)
if val != 1490038500 {
t.Fatalf("Timestamp is not set correctly for %s", data["time"])
}
// Verify that an integer is not a valid time format.
data["int"] = 1490000000
val, err = pilosa.GetTimeStamp(data, "int")
if !strings.Contains(err.Error(), "set-timestamp value must be in time format") {
t.Fatalf("Expected set-timestamp value must be in time format error, actual error: %s", err)
}
// Verify reversing month and year is not valid time format.
data["time"] = "03-2017-20T19:35"
val, err = pilosa.GetTimeStamp(data, timeField)
if !strings.Contains(err.Error(), "cannot parse") {
t.Fatalf("Expected Timestamp is not set correctly, actual error: %s", err)
}
// Handle time fields that do not exist.
val, err = pilosa.GetTimeStamp(data, "test")
if val != 0 {
t.Fatalf("Expected Ignore nonexistent fields")
}
}

2
index.go
View file

@ -772,7 +772,7 @@ func (i *Index) InputBits(frame string, bits []*Bit) error {
// Convert timestamps to time.Time.
if bit.Timestamp > 0 {
t := time.Unix(0, bit.Timestamp)
t := time.Unix(bit.Timestamp, 0)
timestamps[i] = &t
}
}

12
input_definition.go
View file

@ -29,9 +29,10 @@ const (
InputMapping = "mapping"
InputValueToRow = "value-to-row"
InputSingleRowBool = "single-row-boolean"
InputSetTimestamp = "set-timestamp"
)
var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool}
var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool, InputSetTimestamp}
// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
@ -210,7 +211,10 @@ func (a *Action) Validate() error {
if len(a.ValueMap) == 0 {
return ErrInputDefinitionValueMap
}
case InputSetTimestamp:
}
return nil
}
@ -336,11 +340,11 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error {
// HandleAction Process the input data with its action and return a bit to be imported later
// Note: if the Bit should not be set then nil is returned with no error
// From the JSON marshalling the possible types are: float64, boolean, string
// TODO handle Timestamps
func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) {
func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (*Bit, error) {
var err error
var bit Bit
bit.ColumnID = colID
bit.Timestamp = timestamp
switch a.ValueDestination {
case InputMapping:
@ -367,6 +371,8 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) {
return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value)
}
bit.RowID = uint64(v)
case InputSetTimestamp:
break
default:
return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination)
}

24
input_definition_test.go
View file

@ -196,9 +196,10 @@ func TestHandleAction(t *testing.T) {
colID := uint64(0)
rowID := uint64(100)
action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
timestamp := int64(0)
value = 1
b, err := pilosa.HandleAction(action, value, colID)
b, err := pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected integer type is not handled by single-row-boolean")
} else if !strings.Contains(err.Error(), "single-row-boolean value") {
@ -206,31 +207,31 @@ func TestHandleAction(t *testing.T) {
}
value = "1"
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected Ignore strings, only accept boolean")
}
value = "t"
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if !strings.Contains(err.Error(), "must equate to a Bool") {
t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err)
}
value = float64(1)
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if !strings.Contains(err.Error(), "must equate to a Bool") {
t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err)
}
value = false
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected Ignore values that do not equate to True")
}
value = true
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
if b.ColumnID != 0 {
t.Fatalf("Unexpected ColumnID %v", b.ColumnID)
@ -243,35 +244,34 @@ func TestHandleAction(t *testing.T) {
action.ValueDestination = pilosa.InputValueToRow
rowID = 101
value = float64(25.0)
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
if b.RowID != 25 {
t.Fatalf("Unexpected RowID %v", b.RowID)
}
}
value = "25"
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected Ignore values that are not type float64")
}
action.ValueDestination = pilosa.InputMapping
value = "test"
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected Ignore values that are not type string")
}
value = 25
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
t.Fatalf("Expected Ignore values that are not type string")
}
action.ValueDestination = "test"
b, err = pilosa.HandleAction(action, value, colID)
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if !strings.Contains(err.Error(), "Unrecognized Value Destination") {
t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err)
}
}