From 6711026d22be2192d7b1158d50961983bd5248d6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 11:05:34 -0500 Subject: [PATCH] JSONparser tests --- handler.go | 45 ++++++++----- handler_test.go | 157 ++++++++++++++++++++++++++++++++++++++++++-- input_definition.go | 4 +- 3 files changed, 184 insertions(+), 22 deletions(-) diff --git a/handler.go b/handler.go index 008ad89a8..56f86d39e 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,6 @@ import ( "net/http" _ "net/http/pprof" "os" - "sort" "strconv" "strings" "time" @@ -1639,6 +1638,10 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } // JSONParser validate input json file and execute SetBit @@ -1648,36 +1651,48 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri return ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error + var columnLabel string validFields := make(map[string]bool) for _, field := range inputDef.Fields() { validFields[field.Name] = true + if field.PrimaryKey { + columnLabel = field.Name + } } for key, _ := range req { _, ok := validFields[key] if !ok { - fmt.Errorf("field not found", key) + return fmt.Errorf("field not found: %s", key) } } + var bits []*Bit for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - var colValue uint64 + //var colValue uint64 if _, ok := req[field.Name]; !ok { continue - } else if field.PrimaryKey { - colValue, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) - } else { - val, ok := req[DefaultColumnLabel] - if !ok { - return errors.New("column ID not provided") - } - colValue = val.(float64) - } + } + value, ok := req[columnLabel] + if !ok { + return fmt.Errorf("columnLabel required") + } + colValue, ok := value.(float64) + if !ok { + return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + for _, action := range field.Actions { + bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + if err != nil { + return fmt.Errorf("error handling action: %s", action.ValueDestination) + } + bits = append(bits, bit) + } } - return nil } + +func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + return nil, nil +} diff --git a/handler_test.go b/handler_test.go index 35d57e912..0c3691f5b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,10 @@ import ( "context" "encoding/json" "errors" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -27,11 +31,6 @@ import ( "reflect" "strings" "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1174,3 +1173,151 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } } + +var defaultBody = ` + { + "frames":[ + { + "name":"event-time", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + } + ], + "fields":[ + { + "name":"id", + "primaryKey":true + }, + { + "name":"cabType", + "actions":[ + { + "frame":"cab-type", + "valueDestination":"mapping", + "valueMap":{ + "Green":1, + "Yellow":2 + } + } + ] + }, + { + "name":"withPet", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"single-row-boolean", + "rowID":100 + } + ] + }, + { + "name":"distanceMiles", + "actions":[ + { + "frame":"distance-miles", + "valueDestination":"value-to-row" + + } + ] + } + ] + }` + +func TestHandler_CreateInput(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + inputBody := []byte(` + [{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "with-pet": true + }]`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestInput_JSON(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + json string + err string + }{ + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "nofield": true + }]`, + err: "field not found: nofield"}, + {json: `[{ + "id": "abc", + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "float64 require, got value:abc, type: string"}, + {json: `[{ + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "columnLabel required"}, + } + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + for _, test := range tests { + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(test.json)))) + if body := w.Body.String(); body != test.err+"\n" { + t.Fatalf("Expect error: %s, actual: %s", test.err, body) + } + } + +} +func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { + var req pilosa.InputDefinitionInfo + err := json.Unmarshal(body, &req) + if err != nil { + return nil, err + } + def, err := req.Encode() + def.Name = name + return def, err +} diff --git a/input_definition.go b/input_definition.go index 1e8fd4680..c2d16c947 100644 --- a/input_definition.go +++ b/input_definition.go @@ -267,8 +267,8 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []InputDefinitionField `json:"fields"` + Frames []InputFrame `json:"frames"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation.