diff --git a/docs/getting-started.md b/docs/getting-started.md index 6fd5aafd0..8535a4ac2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -140,7 +140,16 @@ curl localhost:10101/index/repository/input-definition/stargazer \ } ], "name": "stargazer_id" - } + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "set-timestamp" + } + ], + "name": "time_value + } ] }' ``` @@ -151,7 +160,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel - value-to-row: The value for this field is used as the `rowID`. - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - + - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame #### Import Data Using an Input Definition @@ -167,6 +176,7 @@ curl localhost:10101/index/repository/input/stargazer \ "language_id": "Go", "repo_id": 91720568, "stargazer_id": 513114 + "time_value": "2017-05-18T20:40" }, { "language_id": "Python", @@ -182,6 +192,7 @@ The data input above is equivalent to the following `SetBit()` operations: curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) + 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") SetBit(frame="language", repo_id=91720568, language_id=5) SetBit(frame="language", repo_id=95122322, language_id=17) ' diff --git a/handler.go b/handler.go index 0f80c2a1c..87712feb0 100644 --- a/handler.go +++ b/handler.go @@ -1646,15 +1646,35 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if err != nil { return nil, err } - // if field in input data is not in defined definition, return error - var columnLabel string + // If field in input data is not in defined definition, return error. + var colValue uint64 validFields := make(map[string]bool) + timestampFrame := make(map[string]int64) for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - columnLabel = field.Name + columnLabel := field.Name + value, ok := req[columnLabel] + if !ok { + return nil, fmt.Errorf("columnLabel required") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) + } + // Find frame that need to add timestamp. + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } + } } } + for key := range req { _, ok := validFields[key] if !ok { @@ -1663,23 +1683,18 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } setBits := make(map[string][]*Bit) + for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data if _, ok := req[field.Name]; !ok { continue } - value, ok := req[columnLabel] - if !ok { - return nil, fmt.Errorf("columnLabel required") - } - colValue, ok := value.(float64) - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } + // Looking into timestampFrame map and set timestamp to the whole frame for _, action := range field.Actions { frame := action.Frame - bit, err := HandleAction(action, req[field.Name], uint64(colValue)) + timestamp := timestampFrame[action.Frame] + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } @@ -1690,3 +1705,23 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } return setBits, nil } + +// GetTimeStamp retrieves unix timestamp from Input data. +func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) { + tmstamp, ok := data[timeField] + if !ok { + return 0, nil + } + + timestamp, ok := tmstamp.(string) + if !ok { + return 0, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, has: %v", data[timeField]) + } + + v, err := time.Parse(TimeFormat, timestamp) + if err != nil { + return 0, err + } + + return v.Unix(), nil +} diff --git a/handler_test.go b/handler_test.go index d28221d7c..0609c22db 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1271,6 +1271,16 @@ var defaultBody = ` "frame":"foo", "valueDestination":"value-to-row" + } + ] + }, + { + "name":"time_value", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"set-timestamp" + } ] } @@ -1296,7 +1306,8 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "withPet": true + "withPet": true, + "time_value": "2017-03-20T19:35" }]`) h := test.NewHandler() h.Holder = hldr.Holder @@ -1318,6 +1329,7 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } + // Test successfully ingest data w = httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1327,7 +1339,6 @@ func TestHandler_CreateInput(t *testing.T) { } // Verify the bits set per frame. - // f := index.Frame("cab-type") f0 := index.Frame("distance-miles") v0 := f0.View(pilosa.ViewStandard) fragment0 := v0.Fragment(0) @@ -1405,6 +1416,13 @@ func TestInput_JSON(t *testing.T) { "noFrame": 1 }]`, err: "Frame not found: foo"}, + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "time_value": 12345 + }]`, + err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() h.Holder = hldr.Holder @@ -1428,3 +1446,33 @@ func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) def.Name = name return def, nil } + +func TestHandler_GetTimeStamp(t *testing.T) { + data := make(map[string]interface{}) + timeField := "time" + data["time"] = "2017-03-20T19:35" + val, err := pilosa.GetTimeStamp(data, timeField) + if val != 1490038500 { + t.Fatalf("Timestamp is not set correctly for %s", data["time"]) + } + + // Verify that an integer is not a valid time format. + data["int"] = 1490000000 + val, err = pilosa.GetTimeStamp(data, "int") + if !strings.Contains(err.Error(), "set-timestamp value must be in time format") { + t.Fatalf("Expected set-timestamp value must be in time format error, actual error: %s", err) + } + + // Verify reversing month and year is not valid time format. + data["time"] = "03-2017-20T19:35" + val, err = pilosa.GetTimeStamp(data, timeField) + if !strings.Contains(err.Error(), "cannot parse") { + t.Fatalf("Expected Timestamp is not set correctly, actual error: %s", err) + } + + // Handle time fields that do not exist. + val, err = pilosa.GetTimeStamp(data, "test") + if val != 0 { + t.Fatalf("Expected Ignore nonexistent fields") + } +} diff --git a/index.go b/index.go index f155bc991..16720630e 100644 --- a/index.go +++ b/index.go @@ -772,7 +772,7 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { // Convert timestamps to time.Time. if bit.Timestamp > 0 { - t := time.Unix(0, bit.Timestamp) + t := time.Unix(bit.Timestamp, 0) timestamps[i] = &t } } diff --git a/input_definition.go b/input_definition.go index d9cc543b5..ab5af2bc4 100644 --- a/input_definition.go +++ b/input_definition.go @@ -29,9 +29,10 @@ const ( InputMapping = "mapping" InputValueToRow = "value-to-row" InputSingleRowBool = "single-row-boolean" + InputSetTimestamp = "set-timestamp" ) -var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool, InputSetTimestamp} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -210,7 +211,10 @@ func (a *Action) Validate() error { if len(a.ValueMap) == 0 { return ErrInputDefinitionValueMap } + case InputSetTimestamp: + } + return nil } @@ -336,11 +340,11 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestamps -func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { +func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID + bit.Timestamp = timestamp switch a.ValueDestination { case InputMapping: @@ -367,6 +371,8 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) } bit.RowID = uint64(v) + case InputSetTimestamp: + break default: return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) } diff --git a/input_definition_test.go b/input_definition_test.go index 58bf9e0e5..da6e91231 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -196,9 +196,10 @@ func TestHandleAction(t *testing.T) { colID := uint64(0) rowID := uint64(100) action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + timestamp := int64(0) value = 1 - b, err := pilosa.HandleAction(action, value, colID) + b, err := pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected integer type is not handled by single-row-boolean") } else if !strings.Contains(err.Error(), "single-row-boolean value") { @@ -206,31 +207,31 @@ func TestHandleAction(t *testing.T) { } value = "1" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore strings, only accept boolean") } value = "t" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = float64(1) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that do not equate to True") } value = true - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) @@ -243,35 +244,34 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.RowID != 25 { t.Fatalf("Unexpected RowID %v", b.RowID) } } value = "25" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type float64") } action.ValueDestination = pilosa.InputMapping value = "test" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } value = 25 - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } action.ValueDestination = "test" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - }