From 0a22386a58ecf5b4a7d9ee506428e56e762461f3 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 5 Jul 2017 15:30:21 -0500 Subject: [PATCH 1/6] add action to handle timestamp --- index.go | 2 +- input_definition.go | 11 ++++++++++- input_definition_test.go | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index f155bc991..16720630e 100644 --- a/index.go +++ b/index.go @@ -772,7 +772,7 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { // Convert timestamps to time.Time. if bit.Timestamp > 0 { - t := time.Unix(0, bit.Timestamp) + t := time.Unix(bit.Timestamp, 0) timestamps[i] = &t } } diff --git a/input_definition.go b/input_definition.go index d9cc543b5..a79b9c53e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -22,6 +22,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "time" ) // Action types. @@ -29,9 +30,10 @@ const ( InputMapping = "mapping" InputValueToRow = "value-to-row" InputSingleRowBool = "single-row-boolean" + InputSetTimestamp = "set-timestamp" ) -var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool, InputSetTimestamp} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -367,6 +369,13 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) } bit.RowID = uint64(v) + case InputSetTimestamp: + v, err := time.Parse(TimeFormat, value.(string)) + if err != nil { + return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", value) + } + bit.Timestamp = v.Unix() + default: return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) } diff --git a/input_definition_test.go b/input_definition_test.go index 58bf9e0e5..e5138bf6b 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -23,6 +23,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" + "time" ) func TestInputDefinition_Open(t *testing.T) { @@ -268,6 +269,22 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type string") } + action.ValueDestination = pilosa.InputSetTimestamp + value = "2017-03-20T19:35" + parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) + b, err = pilosa.HandleAction(action, value, colID) + if b == nil { + t.Fatalf("Expected return bit") + } else if b.Timestamp != parsedTime.Unix() { + t.Fatalf("Timestamp is not set correctly") + } + + value = "12345677" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { + t.Fatal("Expect invalid timestamp format") + } + action.ValueDestination = "test" b, err = pilosa.HandleAction(action, value, colID) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { From ea3e71f78dabd3c3805fdbf86a60b9fd0804ae16 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 09:44:12 -0500 Subject: [PATCH 2/6] update adding timestamp --- handler.go | 17 ++++++++++++++++- handler_test.go | 10 ++++++++++ input_definition.go | 20 ++++++++++++-------- input_definition_test.go | 30 ++++++++++++++++-------------- 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/handler.go b/handler.go index 09f920920..ade37b0c3 100644 --- a/handler.go +++ b/handler.go @@ -1664,11 +1664,17 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, // if field in input data is not in defined definition, return error var columnLabel string validFields := make(map[string]bool) + timestampFrame := make(map[string]string) for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { columnLabel = field.Name } + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame] = field.Name + } + } } for key := range req { _, ok := validFields[key] @@ -1678,6 +1684,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } setBits := make(map[string][]*Bit) + for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data if _, ok := req[field.Name]; !ok { @@ -1692,9 +1699,17 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + var timestamp string for _, action := range field.Actions { frame := action.Frame - bit, err := HandleAction(action, req[field.Name], uint64(colValue)) + timeField, ok := timestampFrame[action.Frame] + if !ok { + timestamp = "" + } else { + timestamp = req[timeField].(string) + } + + bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } diff --git a/handler_test.go b/handler_test.go index d28221d7c..c1e752e32 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1271,6 +1271,16 @@ var defaultBody = ` "frame":"foo", "valueDestination":"value-to-row" + } + ] + }, + { + "name":"time_value", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"set_timestamp" + } ] } diff --git a/input_definition.go b/input_definition.go index a79b9c53e..ae8cc35c9 100644 --- a/input_definition.go +++ b/input_definition.go @@ -212,7 +212,10 @@ func (a *Action) Validate() error { if len(a.ValueMap) == 0 { return ErrInputDefinitionValueMap } + case InputSetTimestamp: + } + return nil } @@ -338,11 +341,17 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestamps -func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { +func HandleAction(a Action, value interface{}, colID uint64, timestamp string) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID + if timestamp != "" { + v, err := time.Parse(TimeFormat, timestamp) + if err != nil { + return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", timestamp) + } + bit.Timestamp = v.Unix() + } switch a.ValueDestination { case InputMapping: @@ -370,12 +379,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { } bit.RowID = uint64(v) case InputSetTimestamp: - v, err := time.Parse(TimeFormat, value.(string)) - if err != nil { - return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", value) - } - bit.Timestamp = v.Unix() - + break default: return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) } diff --git a/input_definition_test.go b/input_definition_test.go index e5138bf6b..d6bb7ba38 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -197,9 +197,10 @@ func TestHandleAction(t *testing.T) { colID := uint64(0) rowID := uint64(100) action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + timestamp := "" value = 1 - b, err := pilosa.HandleAction(action, value, colID) + b, err := pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected integer type is not handled by single-row-boolean") } else if !strings.Contains(err.Error(), "single-row-boolean value") { @@ -207,31 +208,31 @@ func TestHandleAction(t *testing.T) { } value = "1" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore strings, only accept boolean") } value = "t" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = float64(1) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that do not equate to True") } value = true - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) @@ -244,35 +245,35 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.RowID != 25 { t.Fatalf("Unexpected RowID %v", b.RowID) } } value = "25" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type float64") } action.ValueDestination = pilosa.InputMapping value = "test" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } value = 25 - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } action.ValueDestination = pilosa.InputSetTimestamp - value = "2017-03-20T19:35" + timestamp = "2017-03-20T19:35" parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b == nil { t.Fatalf("Expected return bit") } else if b.Timestamp != parsedTime.Unix() { @@ -280,13 +281,14 @@ func TestHandleAction(t *testing.T) { } value = "12345677" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { t.Fatal("Expect invalid timestamp format") } action.ValueDestination = "test" - b, err = pilosa.HandleAction(action, value, colID) + timestamp = "" + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } From 0f9f860624085e4d90cbb85bac70ec4dc6bb4e04 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 12:10:43 -0500 Subject: [PATCH 3/6] update timestamp for whole frame --- handler.go | 12 +++++++++++- handler_test.go | 15 ++++++++++++--- input_definition.go | 2 +- input_definition_test.go | 8 +------- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/handler.go b/handler.go index ade37b0c3..c9868a3a9 100644 --- a/handler.go +++ b/handler.go @@ -1670,6 +1670,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if field.PrimaryKey { columnLabel = field.Name } + // finding frame that need to add timestamp for _, action := range field.Actions { if action.ValueDestination == InputSetTimestamp { timestampFrame[action.Frame] = field.Name @@ -1699,6 +1700,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + // Looking into timestampFrame map and set timestamp to the whole frame var timestamp string for _, action := range field.Actions { frame := action.Frame @@ -1706,7 +1708,15 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if !ok { timestamp = "" } else { - timestamp = req[timeField].(string) + tmstamp, ok := req[timeField] + if !ok { + timestamp = "" + } else { + timestamp, ok = tmstamp.(string) + if !ok { + return nil, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, having: %v", req[timeField]) + } + } } bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) diff --git a/handler_test.go b/handler_test.go index c1e752e32..34eff2424 100644 --- a/handler_test.go +++ b/handler_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" @@ -1279,7 +1280,7 @@ var defaultBody = ` "actions":[ { "frame":"add-ons", - "valueDestination":"set_timestamp" + "valueDestination":"set-timestamp" } ] @@ -1306,7 +1307,8 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "withPet": true + "withPet": true, + "time_value": "2017-03-20T19:35" }]`) h := test.NewHandler() h.Holder = hldr.Holder @@ -1328,6 +1330,7 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } + // Test successfully ingest data w = httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1337,7 +1340,6 @@ func TestHandler_CreateInput(t *testing.T) { } // Verify the bits set per frame. - // f := index.Frame("cab-type") f0 := index.Frame("distance-miles") v0 := f0.View(pilosa.ViewStandard) fragment0 := v0.Fragment(0) @@ -1415,6 +1417,13 @@ func TestInput_JSON(t *testing.T) { "noFrame": 1 }]`, err: "Frame not found: foo"}, + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "time_value": 12345 + }]`, + err: "set-timestamp value must be in time format: YYYY-MM-DD, having: 12345"}, } h := test.NewHandler() h.Holder = hldr.Holder diff --git a/input_definition.go b/input_definition.go index ae8cc35c9..d8aa7a90a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -348,7 +348,7 @@ func HandleAction(a Action, value interface{}, colID uint64, timestamp string) ( if timestamp != "" { v, err := time.Parse(TimeFormat, timestamp) if err != nil { - return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", timestamp) + return nil, err } bit.Timestamp = v.Unix() } diff --git a/input_definition_test.go b/input_definition_test.go index d6bb7ba38..7558f6fac 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -272,7 +272,7 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputSetTimestamp timestamp = "2017-03-20T19:35" - parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) + parsedTime, _ := time.Parse(pilosa.TimeFormat, timestamp) b, err = pilosa.HandleAction(action, value, colID, timestamp) if b == nil { t.Fatalf("Expected return bit") @@ -280,12 +280,6 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Timestamp is not set correctly") } - value = "12345677" - b, err = pilosa.HandleAction(action, value, colID, timestamp) - if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { - t.Fatal("Expect invalid timestamp format") - } - action.ValueDestination = "test" timestamp = "" b, err = pilosa.HandleAction(action, value, colID, timestamp) From 541d839b4d7b9bb11112d2bcabd7538d357ea0c9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 12:33:09 -0500 Subject: [PATCH 4/6] remove unused import --- handler_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index 34eff2424..21cf22037 100644 --- a/handler_test.go +++ b/handler_test.go @@ -26,7 +26,6 @@ import ( "strings" "testing" - "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" From 2144de29d0701a6fe044d27e02916eea85e301bc Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 15:15:26 -0500 Subject: [PATCH 5/6] update doc with input definition timestamp --- docs/getting-started.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 6fd5aafd0..8535a4ac2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -140,7 +140,16 @@ curl localhost:10101/index/repository/input-definition/stargazer \ } ], "name": "stargazer_id" - } + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "set-timestamp" + } + ], + "name": "time_value + } ] }' ``` @@ -151,7 +160,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel - value-to-row: The value for this field is used as the `rowID`. - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - + - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame #### Import Data Using an Input Definition @@ -167,6 +176,7 @@ curl localhost:10101/index/repository/input/stargazer \ "language_id": "Go", "repo_id": 91720568, "stargazer_id": 513114 + "time_value": "2017-05-18T20:40" }, { "language_id": "Python", @@ -182,6 +192,7 @@ The data input above is equivalent to the following `SetBit()` operations: curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) + 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") SetBit(frame="language", repo_id=91720568, language_id=5) SetBit(frame="language", repo_id=95122322, language_id=17) ' From 7333e27392aa65a7a0cdb06c72bb9d4cf228700e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Jul 2017 09:16:02 -0500 Subject: [PATCH 6/6] refactor Input Data GetTimeStamp functionality into a method (#1) * refactor Input Data GetTimeStamp functionality into a method * Comment time format tests --- handler.go | 72 +++++++++++++++++++++++----------------- handler_test.go | 32 +++++++++++++++++- input_definition.go | 11 ++---- input_definition_test.go | 15 +-------- 4 files changed, 75 insertions(+), 55 deletions(-) diff --git a/handler.go b/handler.go index c9868a3a9..27ac356d7 100644 --- a/handler.go +++ b/handler.go @@ -1661,22 +1661,35 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if err != nil { return nil, err } - // if field in input data is not in defined definition, return error - var columnLabel string + // If field in input data is not in defined definition, return error. + var colValue uint64 validFields := make(map[string]bool) - timestampFrame := make(map[string]string) + timestampFrame := make(map[string]int64) for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - columnLabel = field.Name + columnLabel := field.Name + value, ok := req[columnLabel] + if !ok { + return nil, fmt.Errorf("columnLabel required") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) } - // finding frame that need to add timestamp + // Find frame that need to add timestamp. for _, action := range field.Actions { if action.ValueDestination == InputSetTimestamp { - timestampFrame[action.Frame] = field.Name + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } } } } + for key := range req { _, ok := validFields[key] if !ok { @@ -1691,35 +1704,12 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if _, ok := req[field.Name]; !ok { continue } - value, ok := req[columnLabel] - if !ok { - return nil, fmt.Errorf("columnLabel required") - } - colValue, ok := value.(float64) - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } // Looking into timestampFrame map and set timestamp to the whole frame - var timestamp string for _, action := range field.Actions { frame := action.Frame - timeField, ok := timestampFrame[action.Frame] - if !ok { - timestamp = "" - } else { - tmstamp, ok := req[timeField] - if !ok { - timestamp = "" - } else { - timestamp, ok = tmstamp.(string) - if !ok { - return nil, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, having: %v", req[timeField]) - } - } - } - - bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) + timestamp := timestampFrame[action.Frame] + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } @@ -1730,3 +1720,23 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } return setBits, nil } + +// GetTimeStamp retrieves unix timestamp from Input data. +func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) { + tmstamp, ok := data[timeField] + if !ok { + return 0, nil + } + + timestamp, ok := tmstamp.(string) + if !ok { + return 0, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, has: %v", data[timeField]) + } + + v, err := time.Parse(TimeFormat, timestamp) + if err != nil { + return 0, err + } + + return v.Unix(), nil +} diff --git a/handler_test.go b/handler_test.go index 21cf22037..0609c22db 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1422,7 +1422,7 @@ func TestInput_JSON(t *testing.T) { "distanceMiles": 8, "time_value": 12345 }]`, - err: "set-timestamp value must be in time format: YYYY-MM-DD, having: 12345"}, + err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() h.Holder = hldr.Holder @@ -1446,3 +1446,33 @@ func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) def.Name = name return def, nil } + +func TestHandler_GetTimeStamp(t *testing.T) { + data := make(map[string]interface{}) + timeField := "time" + data["time"] = "2017-03-20T19:35" + val, err := pilosa.GetTimeStamp(data, timeField) + if val != 1490038500 { + t.Fatalf("Timestamp is not set correctly for %s", data["time"]) + } + + // Verify that an integer is not a valid time format. + data["int"] = 1490000000 + val, err = pilosa.GetTimeStamp(data, "int") + if !strings.Contains(err.Error(), "set-timestamp value must be in time format") { + t.Fatalf("Expected set-timestamp value must be in time format error, actual error: %s", err) + } + + // Verify reversing month and year is not valid time format. + data["time"] = "03-2017-20T19:35" + val, err = pilosa.GetTimeStamp(data, timeField) + if !strings.Contains(err.Error(), "cannot parse") { + t.Fatalf("Expected Timestamp is not set correctly, actual error: %s", err) + } + + // Handle time fields that do not exist. + val, err = pilosa.GetTimeStamp(data, "test") + if val != 0 { + t.Fatalf("Expected Ignore nonexistent fields") + } +} diff --git a/input_definition.go b/input_definition.go index d8aa7a90a..ab5af2bc4 100644 --- a/input_definition.go +++ b/input_definition.go @@ -22,7 +22,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "time" ) // Action types. @@ -341,17 +340,11 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -func HandleAction(a Action, value interface{}, colID uint64, timestamp string) (*Bit, error) { +func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID - if timestamp != "" { - v, err := time.Parse(TimeFormat, timestamp) - if err != nil { - return nil, err - } - bit.Timestamp = v.Unix() - } + bit.Timestamp = timestamp switch a.ValueDestination { case InputMapping: diff --git a/input_definition_test.go b/input_definition_test.go index 7558f6fac..da6e91231 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -23,7 +23,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" - "time" ) func TestInputDefinition_Open(t *testing.T) { @@ -197,7 +196,7 @@ func TestHandleAction(t *testing.T) { colID := uint64(0) rowID := uint64(100) action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} - timestamp := "" + timestamp := int64(0) value = 1 b, err := pilosa.HandleAction(action, value, colID, timestamp) @@ -270,21 +269,9 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type string") } - action.ValueDestination = pilosa.InputSetTimestamp - timestamp = "2017-03-20T19:35" - parsedTime, _ := time.Parse(pilosa.TimeFormat, timestamp) - b, err = pilosa.HandleAction(action, value, colID, timestamp) - if b == nil { - t.Fatalf("Expected return bit") - } else if b.Timestamp != parsedTime.Unix() { - t.Fatalf("Timestamp is not set correctly") - } - action.ValueDestination = "test" - timestamp = "" b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - }