From d416d2c5d2f13c30e9089d3241e5a46e1c042ac5 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 12:41:52 -0500 Subject: [PATCH 01/17] JSON parser --- handler.go | 33 +++++++++++++++++++++++++++++++ index.go | 47 +++++++++++++++++++++++++++++++++++++++++++++ input_definition.go | 21 ++++++++++---------- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/handler.go b/handler.go index cefb60ec1..3eeb1ad12 100644 --- a/handler.go +++ b/handler.go @@ -115,6 +115,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") @@ -1510,6 +1511,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } + fmt.Println() // Find index. index := h.Holder.Index(indexName) @@ -1603,3 +1605,34 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req } type postInputDefinitionResponse struct{} + +func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + + + // Decode request. + var reqs []interface{} + err := json.NewDecoder(r.Body).Decode(&reqs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, req := range reqs { + definition := index.inputDefinition(inputDefName) + err = index.JSONParser(req.(map[string]interface{}), definition) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } +} + + diff --git a/index.go b/index.go index c0877f0d7..3c1cea61a 100644 --- a/index.go +++ b/index.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "reflect" ) // Default index settings. @@ -720,3 +721,49 @@ func (i *Index) openInputDefinition() error { } return nil } + + +func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { + fmt.Println(i.Frames) + for _, field := range inputDef.fields{ + if _, ok := req[field.Name]; !ok { + return fmt.Errorf("field not found") + } + for _, action := range field.Actions { + switch action.ValueDestination { + case "map": + err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "stringToBool": + fmt.Println("HERE") + err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "valueToRow": + err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + } + } + } + val := reflect.ValueOf(Action{}) + for i := 0; i < val.Type().NumField(); i++{ + fmt.Println(val.Type().Field(i).Type) + } + return nil +} +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ + return nil +} + +func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ + return nil +} + +func (i *Index) ValueToRow(frame, name string, value uint64) error{ + return nil +} diff --git a/input_definition.go b/input_definition.go index cb15732ca..c7d845903 100644 --- a/input_definition.go +++ b/input_definition.go @@ -94,19 +94,19 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } numPrimaryKey := 0 - countRowID := make(map[uint64]bool) + countRowID := make(map[string]uint64) + var actions []Action for _, field := range pb.Fields { - var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 { - _, ok := countRowID[action.RowID] - if !ok { - countRowID[action.RowID] = true - } else { + if action.RowID != 0 && action.Frame != ""{ + val, ok := countRowID[action.Frame] + if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + } else { + countRowID[action.Frame] = action.RowID } } actions = append(actions, Action{ @@ -116,7 +116,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: action.RowID, }) } - + fmt.Println(actions) if field.PrimaryKey { numPrimaryKey += 1 } @@ -280,18 +280,17 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - + fmt.Println("ACTION", action.ValueDestination) switch action.ValueDestination { case "map": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } case "stringToBool": + fmt.Println("HERR", action.RowID) if action.RowID == 0 { return errors.New("rowID required for stringToBool") } - default: - return nil } return nil From ed1136c466d58f796ccaba02455d42dad486c9e6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 15:10:05 -0500 Subject: [PATCH 02/17] update definition validation --- handler_test.go | 4 ++-- index.go | 4 +++- index_test.go | 8 ++++---- input_definition.go | 40 ++++++++++++++++++++++++---------------- input_definition_test.go | 28 +++++++++++++++++++--------- 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/handler_test.go b/handler_test.go index 45ca9f021..1b8c914ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,7 +1121,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -1150,7 +1150,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) diff --git a/index.go b/index.go index 3c1cea61a..83ca3762f 100644 --- a/index.go +++ b/index.go @@ -651,7 +651,9 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin return nil, err } - inputDef.LoadDefinition(pb) + if err = inputDef.LoadDefinition(pb); err != nil { + return nil, err + } if err = inputDef.saveMeta(); err != nil { return nil, err } diff --git a/index_test.go b/index_test.go index f948310ca..f4cb534af 100644 --- a/index_test.go +++ b/index_test.go @@ -247,7 +247,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -266,7 +266,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -297,7 +297,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -321,7 +321,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) diff --git a/input_definition.go b/input_definition.go index c7d845903..9d0304125 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"map", "valueToRow", "stringToBool"} +var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -95,16 +95,16 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { numPrimaryKey := 0 countRowID := make(map[string]uint64) - var actions []Action for _, field := range pb.Fields { + var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != ""{ + if action.RowID != 0 && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { - return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) } else { countRowID[action.Frame] = action.RowID } @@ -113,10 +113,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: &action.RowID, }) } - fmt.Println(actions) if field.PrimaryKey { numPrimaryKey += 1 } @@ -176,7 +175,7 @@ func (i *InputDefinition) saveMeta() error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: convert(action.RowID), } actions = append(actions, actionMeta) } @@ -226,7 +225,7 @@ type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` - RowID uint64 `json:"rowID,omitempty"` + RowID *uint64 `json:"rowID,omitempty"` } // Encode converts Action into its internal representation. @@ -235,10 +234,19 @@ func (o *Action) Encode() *internal.Action { Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, - RowID: o.RowID, + RowID: convert(o.RowID), } } +func convert(x *uint64) uint64 { + if x != nil { + return *x + } + var v int64 = -1 + var v2 uint64 = uint64(v) + return v2 +} + // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` @@ -273,6 +281,9 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } func (i *InputDefinition) ValidateAction(action *internal.Action) error { + if action.Frame == "" { + return ErrFrameRequired + } validValues := make(map[string]bool) for _, val := range ValidValueDestination { validValues[val] = true @@ -280,18 +291,15 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - fmt.Println("ACTION", action.ValueDestination) switch action.ValueDestination { - case "map": + case "mapping": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "stringToBool": - fmt.Println("HERR", action.RowID) - if action.RowID == 0 { - return errors.New("rowID required for stringToBool") + case "single-row-boolean": + if int64(action.RowID) == -1 { + return errors.New("rowID required for single-row-boolean") } } - return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 178b30ea7..9a1c4a5a4 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,7 +29,7 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -102,7 +102,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "ValueToRow", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) @@ -110,22 +110,25 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} + encodeAction := act.Encode() + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "rowID required for stringToBool") { - t.Fatalf("Expected rowID required for stringToBool error, actual error: %s", err) + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "map", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 101} + action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -133,11 +136,18 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } + + action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "frame required") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } } From d1d8c5b696ddb02e0565f6c12ba4038e24a0a9c9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 11:05:33 -0500 Subject: [PATCH 03/17] handle actions in JSONparser --- handler.go | 17 ++++++---- index.go | 71 ++++++++++++++++++++++++++-------------- input_definition.go | 46 ++++++++++++++++---------- input_definition_test.go | 22 ++++++------- pilosa.go | 14 ++++---- 5 files changed, 105 insertions(+), 65 deletions(-) diff --git a/handler.go b/handler.go index 3eeb1ad12..5dd2347e8 100644 --- a/handler.go +++ b/handler.go @@ -1520,7 +1520,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - def := req.Encode() + def, err := req.Encode() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } def.Name = inputDefName // Create InputDefinition. @@ -1617,7 +1621,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1626,13 +1629,13 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - definition := index.inputDefinition(inputDefName) - err = index.JSONParser(req.(map[string]interface{}), definition) - if err != nil { + err = index.JSONParser(req.(map[string]interface{}), inputDefName) + if err == ErrInputDefinitionNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } } - - diff --git a/index.go b/index.go index 83ca3762f..1114be5d0 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "reflect" ) // Default index settings. @@ -724,48 +723,72 @@ func (i *Index) openInputDefinition() error { return nil } - -func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { - fmt.Println(i.Frames) - for _, field := range inputDef.fields{ - if _, ok := req[field.Name]; !ok { - return fmt.Errorf("field not found") +func (i *Index) JSONParser(req map[string]interface{}, name string) error { + inputDef := i.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound + } + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + } + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } + } + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + for _, action := range field.Actions { switch action.ValueDestination { - case "map": + case "mapping": + value, ok := req[field.Name].(string) + if !ok { + return fmt.Errorf("String type required, got %s:%s", field.Name, value) + } err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action mapping : %s", err) } - case "stringToBool": - fmt.Println("HERE") - err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) - if err != nil { - return fmt.Errorf("Error map value: %s", err) + case "single-row-boolean": + value, ok := req[field.Name].(bool) + if !ok { + return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) } - case "valueToRow": - err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + err := i.StringRowBoolean(action.Frame, action.RowID, value) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action single-row-boolean : %s", err) + } + case "value-to-row": + value, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) + } + err := i.ValueToRow(action.Frame, field.Name, uint64(value)) + if err != nil { + return fmt.Errorf("Error action value-to-row: %s", err) } } } } - val := reflect.ValueOf(Action{}) - for i := 0; i < val.Type().NumField(); i++{ - fmt.Println(val.Type().Field(i).Type) - } + return nil } -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { return nil } -func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ +func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { return nil } -func (i *Index) ValueToRow(frame, name string, value uint64) error{ +func (i *Index) ValueToRow(frame, name string, value uint64) error { return nil } diff --git a/input_definition.go b/input_definition.go index 9d0304125..a24163ca6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,13 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} +const ( + Mapping = "mapping" + ValueToRow = "value-to-row" + SingleRowBool = "single-row-boolean" +) + +var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -212,12 +218,17 @@ type Field struct { } // Encode converts Field into its internal representation. -func (o *Field) Encode() *internal.InputDefinitionField { +func (o *Field) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { - field.Actions = append(field.Actions, action.Encode()) + actionEncode, err := action.Encode() + if err != nil { + return nil, err + } + field.Actions = append(field.Actions, actionEncode) } - return &field + return &field, nil } // Action descripes the mapping method for the field in the InputDefinition. @@ -229,22 +240,23 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() *internal.Action { +func (o *Action) Encode() (*internal.Action, error) { + if o.RowID == nil && o.ValueDestination == "single-row-boolean" { + return nil, errors.New("rowID required for single-row-boolean") + } return &internal.Action{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, RowID: convert(o.RowID), - } + }, nil } func convert(x *uint64) uint64 { if x != nil { return *x } - var v int64 = -1 - var v2 uint64 = uint64(v) - return v2 + return 0 } // InputFrame defines the frame used in the input definition. @@ -260,16 +272,20 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { +func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { var def internal.InputDefinition for _, f := range i.Frames { def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) } for _, f := range i.Fields { - def.Fields = append(def.Fields, f.Encode()) + fEncode, err := f.Encode() + if err != nil { + return nil, err + } + def.Fields = append(def.Fields, fEncode) } - return &def + return &def, nil } func (i *InputDefinition) AddFrame(frame InputFrame) error { @@ -292,14 +308,10 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } switch action.ValueDestination { - case "mapping": + case Mapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "single-row-boolean": - if int64(action.RowID) == -1 { - return errors.New("rowID required for single-row-boolean") - } } return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 9a1c4a5a4..160064e90 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -80,7 +80,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatal(err) } - internalDef := def.Encode() + internalDef, err := def.Encode() + if err != nil { + t.Fatal(err) + } if internalDef.Frames[0].Name != "event-time" { t.Fatalf("unexpected frame: %v", internalDef) @@ -110,16 +113,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} - encodeAction := act.Encode() - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -127,8 +127,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} + action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -136,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -144,7 +144,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { diff --git a/pilosa.go b/pilosa.go index 504df3b65..ff1ef148b 100644 --- a/pilosa.go +++ b/pilosa.go @@ -30,12 +30,14 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") + ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From 735a7287b79ee3bd7e5841b05aed13b2fd8b7beb Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 21:57:00 -0500 Subject: [PATCH 04/17] merge action's processing --- handler.go | 63 +++++++++++++++++++++++++++++++++++++++++++ server/server_test.go | 32 ++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/handler.go b/handler.go index 5dd2347e8..def9f8436 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "sort" "strconv" "strings" "time" @@ -1639,3 +1640,65 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } + +// MapAction Process the input data and set a bit +func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var ok bool + bit.ColumnID = colID + bit.RowID, ok = a.ValueMap[value] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", value) + } + + // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, nil +} + +// ValueToRow Sets a bitmap with rowID from the input value +func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// SingleRowBoolean Sets a bitmap with rowID from the Action defintion +func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// InputBits Process and Sort the Input Bits and route to appropriate nodes. +func (h *Handler) InputBits(index, frame string, bits []Bit) error { + client, err := NewClient(h.Host) + if err != nil { + return err + } + + bitsBySlice := Bits(bits).GroupBySlice() + + // Parse path into bits. + for slice, bits := range bitsBySlice { + sort.Sort(BitsByPos(bits)) + + h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) + if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { + return err + } + } + + return nil +} diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..a92da70a8 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -494,6 +494,29 @@ func TestMain_SendReceiveMessage(t *testing.T) { if maxSlices1["i"] != 2 { t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "id", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } } // availablePorts returns a slice of ports that can be used for testing. @@ -615,6 +638,15 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 From cfe146aa971b4605c277a021c74b1aff2b7dddd2 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 23 Jun 2017 11:17:36 -0500 Subject: [PATCH 05/17] update JSONparser --- handler.go | 89 +++++++++++++++++++++--------------------------------- index.go | 70 ------------------------------------------ 2 files changed, 34 insertions(+), 125 deletions(-) diff --git a/handler.go b/handler.go index def9f8436..008ad89a8 100644 --- a/handler.go +++ b/handler.go @@ -1630,7 +1630,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = index.JSONParser(req.(map[string]interface{}), inputDefName) + err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1641,64 +1641,43 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// MapAction Process the input data and set a bit -func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var ok bool - bit.ColumnID = colID - bit.RowID, ok = a.ValueMap[value] - if !ok { - return nil, fmt.Errorf("Value %s does not exist in definition map", value) +// JSONParser validate input json file and execute SetBit +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { + inputDef := index.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound } - - // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, nil -} - -// ValueToRow Sets a bitmap with rowID from the input value -func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// SingleRowBoolean Sets a bitmap with rowID from the Action defintion -func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err - } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// InputBits Process and Sort the Input Bits and route to appropriate nodes. -func (h *Handler) InputBits(index, frame string, bits []Bit) error { - client, err := NewClient(h.Host) - if err != nil { - return err - } - - bitsBySlice := Bits(bits).GroupBySlice() - - // Parse path into bits. - for slice, bits := range bitsBySlice { - sort.Sort(BitsByPos(bits)) - - h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) - if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { - return err + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } } + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + var colValue uint64 + if _, ok := req[field.Name]; !ok { + continue + } else if field.PrimaryKey { + colValue, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) + } else { + val, ok := req[DefaultColumnLabel] + if !ok { + return errors.New("column ID not provided") + } + colValue = val.(float64) + } + } + + } + return nil } diff --git a/index.go b/index.go index 1114be5d0..c70c9eec4 100644 --- a/index.go +++ b/index.go @@ -722,73 +722,3 @@ func (i *Index) openInputDefinition() error { } return nil } - -func (i *Index) JSONParser(req map[string]interface{}, name string) error { - inputDef := i.inputDefinition(name) - if inputDef == nil { - return ErrInputDefinitionNotFound - } - // if field in input data is not in defined definition, return error - validFields := make(map[string]bool) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - } - for key, _ := range req { - _, ok := validFields[key] - if !ok { - fmt.Errorf("field not found", key) - } - } - - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue - } - - for _, action := range field.Actions { - switch action.ValueDestination { - case "mapping": - value, ok := req[field.Name].(string) - if !ok { - return fmt.Errorf("String type required, got %s:%s", field.Name, value) - } - err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) - if err != nil { - return fmt.Errorf("Error action mapping : %s", err) - } - case "single-row-boolean": - value, ok := req[field.Name].(bool) - if !ok { - return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.StringRowBoolean(action.Frame, action.RowID, value) - if err != nil { - return fmt.Errorf("Error action single-row-boolean : %s", err) - } - case "value-to-row": - value, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.ValueToRow(action.Frame, field.Name, uint64(value)) - if err != nil { - return fmt.Errorf("Error action value-to-row: %s", err) - } - } - } - } - - return nil -} -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { - return nil -} - -func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { - return nil -} - -func (i *Index) ValueToRow(frame, name string, value uint64) error { - return nil -} From 6711026d22be2192d7b1158d50961983bd5248d6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 11:05:34 -0500 Subject: [PATCH 06/17] JSONparser tests --- handler.go | 45 ++++++++----- handler_test.go | 157 ++++++++++++++++++++++++++++++++++++++++++-- input_definition.go | 4 +- 3 files changed, 184 insertions(+), 22 deletions(-) diff --git a/handler.go b/handler.go index 008ad89a8..56f86d39e 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,6 @@ import ( "net/http" _ "net/http/pprof" "os" - "sort" "strconv" "strings" "time" @@ -1639,6 +1638,10 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } // JSONParser validate input json file and execute SetBit @@ -1648,36 +1651,48 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri return ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error + var columnLabel string validFields := make(map[string]bool) for _, field := range inputDef.Fields() { validFields[field.Name] = true + if field.PrimaryKey { + columnLabel = field.Name + } } for key, _ := range req { _, ok := validFields[key] if !ok { - fmt.Errorf("field not found", key) + return fmt.Errorf("field not found: %s", key) } } + var bits []*Bit for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - var colValue uint64 + //var colValue uint64 if _, ok := req[field.Name]; !ok { continue - } else if field.PrimaryKey { - colValue, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) - } else { - val, ok := req[DefaultColumnLabel] - if !ok { - return errors.New("column ID not provided") - } - colValue = val.(float64) - } + } + value, ok := req[columnLabel] + if !ok { + return fmt.Errorf("columnLabel required") + } + colValue, ok := value.(float64) + if !ok { + return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + for _, action := range field.Actions { + bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + if err != nil { + return fmt.Errorf("error handling action: %s", action.ValueDestination) + } + bits = append(bits, bit) + } } - return nil } + +func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + return nil, nil +} diff --git a/handler_test.go b/handler_test.go index 35d57e912..0c3691f5b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,10 @@ import ( "context" "encoding/json" "errors" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -27,11 +31,6 @@ import ( "reflect" "strings" "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1174,3 +1173,151 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } } + +var defaultBody = ` + { + "frames":[ + { + "name":"event-time", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + } + ], + "fields":[ + { + "name":"id", + "primaryKey":true + }, + { + "name":"cabType", + "actions":[ + { + "frame":"cab-type", + "valueDestination":"mapping", + "valueMap":{ + "Green":1, + "Yellow":2 + } + } + ] + }, + { + "name":"withPet", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"single-row-boolean", + "rowID":100 + } + ] + }, + { + "name":"distanceMiles", + "actions":[ + { + "frame":"distance-miles", + "valueDestination":"value-to-row" + + } + ] + } + ] + }` + +func TestHandler_CreateInput(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + inputBody := []byte(` + [{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "with-pet": true + }]`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestInput_JSON(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + json string + err string + }{ + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "nofield": true + }]`, + err: "field not found: nofield"}, + {json: `[{ + "id": "abc", + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "float64 require, got value:abc, type: string"}, + {json: `[{ + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "columnLabel required"}, + } + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + for _, test := range tests { + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(test.json)))) + if body := w.Body.String(); body != test.err+"\n" { + t.Fatalf("Expect error: %s, actual: %s", test.err, body) + } + } + +} +func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { + var req pilosa.InputDefinitionInfo + err := json.Unmarshal(body, &req) + if err != nil { + return nil, err + } + def, err := req.Encode() + def.Name = name + return def, err +} diff --git a/input_definition.go b/input_definition.go index 1e8fd4680..c2d16c947 100644 --- a/input_definition.go +++ b/input_definition.go @@ -267,8 +267,8 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []InputDefinitionField `json:"fields"` + Frames []InputFrame `json:"frames"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation. From 03e5b36af76d6bd2e8d765984a8100d5c6c93819 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 12:19:55 -0500 Subject: [PATCH 07/17] merge handleAction --- handler.go | 35 ++++++++------- handler_test.go | 25 +++++++++-- index.go | 23 ++++++++++ index_test.go | 34 +++++++++++++++ input_definition.go | 52 ++++++++++++++++++++++- input_definition_test.go | 92 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 239 insertions(+), 22 deletions(-) diff --git a/handler.go b/handler.go index 56f86d39e..7fae39217 100644 --- a/handler.go +++ b/handler.go @@ -1629,7 +1629,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1637,18 +1637,24 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + for fr, bs := range bits { + err := index.InputBits(fr, bs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } // JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { - return ErrInputDefinitionNotFound + return nil, ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error var columnLabel string @@ -1662,11 +1668,12 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri for key, _ := range req { _, ok := validFields[key] if !ok { - return fmt.Errorf("field not found: %s", key) + return nil, fmt.Errorf("field not found: %s", key) } } var bits []*Bit + setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data //var colValue uint64 @@ -1675,24 +1682,22 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } value, ok := req[columnLabel] if !ok { - return fmt.Errorf("columnLabel required") + return nil, fmt.Errorf("columnLabel required") } colValue, ok := value.(float64) if !ok { - return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } for _, action := range field.Actions { - bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + frame := action.Frame + bit, err := HandleAction(action, req[field.Name], uint64(colValue)) if err != nil { - return fmt.Errorf("error handling action: %s", action.ValueDestination) + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - bits = append(bits, bit) + //bits = append(bits, bit) + setBits[frame] = append(bits, bit) } } - return nil -} - -func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { - return nil, nil + return setBits, nil } diff --git a/handler_test.go b/handler_test.go index 0c3691f5b..ec79f837a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" @@ -1178,12 +1179,27 @@ var defaultBody = ` { "frames":[ { - "name":"event-time", + "name":"cab-type", "options": { "timeQuantum":"YMD", "inverseEnabled":false, "cacheType":"ranked" } + }, + { + "name":"add-ons", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + }, + { + "name":"distance-miles", + "options": { + "timeQuantum":"YMD", + "cacheType":"ranked" + } } ], "fields":[ @@ -1198,8 +1214,8 @@ var defaultBody = ` "frame":"cab-type", "valueDestination":"mapping", "valueMap":{ - "Green":1, - "Yellow":2 + "green":1, + "yellow":2 } } ] @@ -1246,13 +1262,14 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "with-pet": true + "withPet": true }]`) h := NewHandler() h.Holder = hldr.Holder h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { diff --git a/index.go b/index.go index 69afdbbf9..8d5d4065b 100644 --- a/index.go +++ b/index.go @@ -755,3 +755,26 @@ func (i *Index) openInputDefinition() error { } return nil } + +// InputBits Process the []Bit though the Frame import process +func (i *Index) InputBits(frame string, bits []*Bit) error { + var rowIDs, columnIDs []uint64 + timestamps := make([]*time.Time, len(bits)) + f := i.Frame(frame) + if f == nil { + return fmt.Errorf("Frame not found: %s", frame) + } + + for i, bit := range bits { + rowIDs = append(rowIDs, bit.RowID) + columnIDs = append(columnIDs, bit.ColumnID) + + // Convert timestamps to time.Time. + if bit.Timestamp > 0 { + t := time.Unix(0, bit.Timestamp) + timestamps[i] = &t + } + } + + return f.Import(rowIDs, columnIDs, timestamps) +} diff --git a/index_test.go b/index_test.go index 49ab014b3..d7df43da3 100644 --- a/index_test.go +++ b/index_test.go @@ -454,3 +454,37 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } } + +func TestIndex_InputBits(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } + + // Create frame. + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + var bits []*pilosa.Bit + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) + bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + + err := index.InputBits("f", bits) + if err != nil { + t.Fatal(err) + } + + f := index.Frame("f") + v := f.View(pilosa.ViewStandard) + fragment := v.Fragment(0) + + // Verify the Bits were set + if a := fragment.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{0, 1}) { + t.Fatalf("unexpected bits: %+v", a) + } +} diff --git a/input_definition.go b/input_definition.go index c2d16c947..47236b1cf 100644 --- a/input_definition.go +++ b/input_definition.go @@ -15,12 +15,13 @@ package pilosa import ( + "fmt" "io/ioutil" "os" "path/filepath" "errors" - "fmt" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -295,7 +296,6 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } return nil } - func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired @@ -315,3 +315,51 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) } return nil } + +// HandleAction Process the input data with its action and return a bit to be imported later +// Note: if the Bit should not be set then nil is returned with no error +// From the JSON marshalling the possible types are: float64, boolean, string +// TODO handle Timestams +func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + var err error + var bit Bit + bit.ColumnID = colID + + switch a.ValueDestination { + case Mapping: + v, ok := value.(string) + if !ok { + return nil, fmt.Errorf("Mapping value must be a string %v", value) + } + bit.RowID, ok = a.ValueMap[v] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", v) + } + case SingleRowBool: + switch value.(type) { + case bool: + if value.(bool) { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + case float64: + if value.(float64) >= 1 { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + default: + return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) + } + case ValueToRow: + v, ok := value.(float64) + if !ok { + return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) + } + bit.RowID = uint64(v) + default: + return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) + } + return &bit, err +} diff --git a/input_definition_test.go b/input_definition_test.go index aa6acd5f8..0e078536e 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -18,9 +18,10 @@ import ( "encoding/json" "testing" + "strings" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" - "strings" ) func TestInputDefinition_Open(t *testing.T) { @@ -151,3 +152,92 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected frame required error, actual error: %s", err) } } + +func TestHandleAction(t *testing.T) { + var value interface{} + colID := uint64(0) + rowID := uint64(100) + action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + + value = 1 + b, err := pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected integer type is not handled by single-row-boolean") + } else if !strings.Contains(err.Error(), "single-row-boolean value") { + t.Fatalf("Expected single-row-boolean value error, actual error: %s", err) + } + + value = "1" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore strings, only accept boolean") + } + + value = "t" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + + value = float64(1.5) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } + } + value = float64(0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = false + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = true + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.ColumnID != 0 { + t.Fatalf("Unexpected ColumnID %v", b.ColumnID) + } + } + + action.ValueDestination = pilosa.ValueToRow + rowID = 101 + value = float64(25.0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 25 { + t.Fatalf("Unexpected RowID %v", b.RowID) + } + } + value = "25" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type float64") + } + + action.ValueDestination = pilosa.Mapping + value = "test" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + value = 25 + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + action.ValueDestination = "test" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "Unrecognized Value Destination") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + +} From c441e55bb2acc9b760a7936fa3d3ff547614cdb0 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 14:56:27 -0500 Subject: [PATCH 08/17] Prefaced validValueDestination values Code cleanup --- handler.go | 2 +- input_definition.go | 31 +++++++++++++++++-------------- input_definition_test.go | 18 +++++++++--------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/handler.go b/handler.go index 7fae39217..71d6c25ef 100644 --- a/handler.go +++ b/handler.go @@ -1665,7 +1665,7 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri columnLabel = field.Name } } - for key, _ := range req { + for key := range req { _, ok := validFields[key] if !ok { return nil, fmt.Errorf("field not found: %s", key) diff --git a/input_definition.go b/input_definition.go index 47236b1cf..56b5d7705 100644 --- a/input_definition.go +++ b/input_definition.go @@ -26,13 +26,14 @@ import ( "github.com/pilosa/pilosa/internal" ) +// Action Mapping types const ( - Mapping = "mapping" - ValueToRow = "value-to-row" - SingleRowBool = "single-row-boolean" + InputMapping = "mapping" + InputValueToRow = "value-to-row" + InputSingleRowBool = "single-row-boolean" ) -var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -108,13 +109,12 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.ValueDestination == SingleRowBool && action.Frame != "" { + if action.ValueDestination == InputSingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) - } else { - countRowID[action.Frame] = action.RowID } + countRowID[action.Frame] = action.RowID } actions = append(actions, Action{ Frame: action.Frame, @@ -124,7 +124,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { }) } if field.PrimaryKey { - numPrimaryKey += 1 + numPrimaryKey++ } if numPrimaryKey > 1 { @@ -289,6 +289,7 @@ func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { return &def, nil } +// AddFrame manually add frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) if err := i.saveMeta(); err != nil { @@ -296,19 +297,21 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } return nil } + +// ValidateAction ensures the input definition action conforms to our specification. func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } validValues := make(map[string]bool) - for _, val := range ValidValueDestination { + for _, val := range validValueDestination { validValues[val] = true } if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } switch action.ValueDestination { - case Mapping: + case InputMapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } @@ -319,14 +322,14 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestams +// TODO handle Timestamps func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID switch a.ValueDestination { - case Mapping: + case InputMapping: v, ok := value.(string) if !ok { return nil, fmt.Errorf("Mapping value must be a string %v", value) @@ -335,7 +338,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { if !ok { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } - case SingleRowBool: + case InputSingleRowBool: switch value.(type) { case bool: if value.(bool) { @@ -352,7 +355,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } - case ValueToRow: + case InputValueToRow: v, ok := value.(float64) if !ok { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) diff --git a/input_definition_test.go b/input_definition_test.go index 0e078536e..8adcbc1cc 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -114,13 +114,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -128,8 +128,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -137,7 +137,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -145,7 +145,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.InputSingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { @@ -157,7 +157,7 @@ func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) rowID := uint64(100) - action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} value = 1 b, err := pilosa.HandleAction(action, value, colID) @@ -206,7 +206,7 @@ func TestHandleAction(t *testing.T) { } } - action.ValueDestination = pilosa.ValueToRow + action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) b, err = pilosa.HandleAction(action, value, colID) @@ -221,7 +221,7 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type float64") } - action.ValueDestination = pilosa.Mapping + action.ValueDestination = pilosa.InputMapping value = "test" b, err = pilosa.HandleAction(action, value, colID) if b != nil { From 98fa937af2094444336b99d23e5b1358e9d2797b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 15:19:03 -0500 Subject: [PATCH 09/17] remove prints --- handler.go | 1 - handler_test.go | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 71d6c25ef..341c2d395 100644 --- a/handler.go +++ b/handler.go @@ -1511,7 +1511,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } - fmt.Println() // Find index. index := h.Holder.Index(indexName) diff --git a/handler_test.go b/handler_test.go index ec79f837a..0fe1f6c47 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,11 +19,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -32,6 +27,11 @@ import ( "reflect" "strings" "testing" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1269,7 +1269,6 @@ func TestHandler_CreateInput(t *testing.T) { h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) - fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { From d8e7769f5b48d20bcc27649790d9aa5779b67699 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:51:57 -0500 Subject: [PATCH 10/17] handle input nil bit --- index.go | 3 +++ index_test.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index 8d5d4065b..7c97c0498 100644 --- a/index.go +++ b/index.go @@ -766,6 +766,9 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { } for i, bit := range bits { + if bit == nil { + continue + } rowIDs = append(rowIDs, bit.RowID) columnIDs = append(columnIDs, bit.ColumnID) diff --git a/index_test.go b/index_test.go index d7df43da3..8a7367636 100644 --- a/index_test.go +++ b/index_test.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -456,6 +457,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } func TestIndex_InputBits(t *testing.T) { + var bits []*pilosa.Bit index := MustOpenIndex() defer index.Close() @@ -464,17 +466,22 @@ func TestIndex_InputBits(t *testing.T) { t.Fatal(err) } + err := index.InputBits("f", bits) + if !strings.Contains(err.Error(), "Frame not found") { + t.Fatalf("Expected Frame not found error, actual error: %s", err) + } + // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - var bits []*pilosa.Bit bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + bits = append(bits, nil) - err := index.InputBits("f", bits) + err = index.InputBits("f", bits) if err != nil { t.Fatal(err) } From 66900e48369f53851744a0e2836e59d94e8bd8e9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:52:51 -0500 Subject: [PATCH 11/17] accumulate set bits per frame, don't mix into one set across frames --- handler.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 341c2d395..a44f05ae9 100644 --- a/handler.go +++ b/handler.go @@ -1628,7 +1628,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.InputJsonDataParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1649,8 +1649,8 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { +// InputJsonDataParser validate input json file and execute SetBit +func (h *Handler) InputJsonDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { return nil, ErrInputDefinitionNotFound @@ -1671,11 +1671,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } } - var bits []*Bit setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - //var colValue uint64 if _, ok := req[field.Name]; !ok { continue } @@ -1694,8 +1692,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - //bits = append(bits, bit) - setBits[frame] = append(bits, bit) + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } } } return setBits, nil From def02c45e73cc059cf44292a22eb6cb753c418d7 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 08:50:33 -0500 Subject: [PATCH 12/17] The Input process must respect the Action Frame assignments --- handler_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index 0fe1f6c47..37fd9f25c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1274,6 +1274,32 @@ func TestHandler_CreateInput(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + + // Verify the bits set per frame + // f := index.Frame("cab-type") + f0 := index.Frame("distance-miles") + v0 := f0.View(pilosa.ViewStandard) + fragment0 := v0.Fragment(0) + + // Verify the distanceMiles Bit was set + if a := fragment0.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + + f1 := index.Frame("add-ons") + v1 := f1.View(pilosa.ViewStandard) + fragment1 := v1.Fragment(0) + + // Verify the add-ons frame does not have a distanceMiles Bit set + // The Input process must respect the Action Frame assignments + if a := fragment1.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{}) { + t.Fatalf("unexpected bits: %+v", a) + } + // Verify the withPet Bit was set + if a := fragment1.Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + } func TestInput_JSON(t *testing.T) { @@ -1325,8 +1351,8 @@ func TestInput_JSON(t *testing.T) { t.Fatalf("Expect error: %s, actual: %s", test.err, body) } } - } + func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { var req pilosa.InputDefinitionInfo err := json.Unmarshal(body, &req) From 16db983bf1d12f0fd73dc7b4dda0e56e74a6729d Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:02:25 -0500 Subject: [PATCH 13/17] removed float64 check in single row boolean --- input_definition.go | 6 ------ input_definition_test.go | 16 ++++++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/input_definition.go b/input_definition.go index 56b5d7705..40b8451b8 100644 --- a/input_definition.go +++ b/input_definition.go @@ -346,12 +346,6 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { } else { // value is not True. return nil, err } - case float64: - if value.(float64) >= 1 { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } diff --git a/input_definition_test.go b/input_definition_test.go index 8adcbc1cc..595f57f10 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -179,17 +179,10 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - value = float64(1.5) + value = float64(1) b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - if b.RowID != 100 { - t.Fatalf("Unexpected rowID %v", b.RowID) - } - } - value = float64(0) - b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - t.Fatalf("Expected Ignore values that do not equate to True") + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false @@ -204,6 +197,9 @@ func TestHandleAction(t *testing.T) { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) } + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } } action.ValueDestination = pilosa.InputValueToRow From 7b66fcf26c8dc0d9c8fc514509d2f93b3a16486b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:39:11 -0500 Subject: [PATCH 14/17] cleanup type detection for consistency --- input_definition.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index 40b8451b8..a8e2e709e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -339,16 +339,14 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } case InputSingleRowBool: - switch value.(type) { - case bool: - if value.(bool) { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } - default: + v, ok := value.(bool) + if !ok { return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } + if v == false { // False returns a nil error and nil bit. + return nil, err + } + bit.RowID = *a.RowID case InputValueToRow: v, ok := value.(float64) if !ok { From b2bbff6a33bc5e6c00a9fde9f8695a4cd0bb37a6 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 11:07:54 -0500 Subject: [PATCH 15/17] conver invalid input name --- input_definition_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/input_definition_test.go b/input_definition_test.go index 595f57f10..753681e51 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -32,8 +32,14 @@ func TestInputDefinition_Open(t *testing.T) { frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) + if !strings.Contains(err.Error(), "invalid index or frame's name") { + t.Fatalf("Expected Invalid name error, actual error: %s", err) + } + + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } From d87f1bbca1fa088670eefecce7e82e87dbc5f5ae Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 14:06:12 -0500 Subject: [PATCH 16/17] Input Definition Struct encoding error tests --- input_definition.go | 2 +- input_definition_test.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/input_definition.go b/input_definition.go index a8e2e709e..aba684fb6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -232,7 +232,7 @@ func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) return &field, nil } -// Action descripes the mapping method for the field in the InputDefinition. +// Action describes the mapping method for the field in the InputDefinition. type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` diff --git a/input_definition_test.go b/input_definition_test.go index 753681e51..925adb724 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -120,12 +120,6 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} - _, err = act.Encode() - if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { - t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) - } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -159,6 +153,21 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } } +func TestActionEncoding(t *testing.T) { + action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err := action.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } + + field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} + info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} + _, err = info.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } +} + func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) From 1f90461a092fa74a26552825e48f9666c4e141a4 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 27 Jun 2017 16:06:21 -0500 Subject: [PATCH 17/17] rebase input-definition --- handler.go | 57 ++++++++------ handler_test.go | 154 ++++++++++++++++++++++++++++++++++---- index.go | 22 +++--- index_test.go | 33 +++++---- input_definition.go | 15 +--- input_definition_test.go | 12 +-- internal/private.pb.go | 155 ++++++++++++++------------------------- internal/private.proto | 1 - pilosa.go | 8 +- server/server_test.go | 2 +- 10 files changed, 276 insertions(+), 183 deletions(-) diff --git a/handler.go b/handler.go index a44f05ae9..99695ee7e 100644 --- a/handler.go +++ b/handler.go @@ -1500,10 +1500,18 @@ func errorString(err error) string { return err.Error() } +// handlePostInputDefinition handles POST /input-definition request. func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1512,13 +1520,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - + // Encode InputDefinition to its internal representation. def, err := req.Encode() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1526,6 +1528,22 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } def.Name = inputDefName + // Validate columnLabel and duplicate primaryKey. + numPrimaryKey := 0 + for _, field := range def.Fields { + if field.PrimaryKey { + numPrimaryKey += 1 + if field.Name != index.columnLabel { + http.Error(w, ErrInputDefinitionColumnLabel.Error(), http.StatusBadRequest) + return + } + } + } + if numPrimaryKey > 1 { + http.Error(w, ErrInputDefinitionPrimaryKey.Error(), http.StatusBadRequest) + return + } + // Create InputDefinition. _, err = index.CreateInputDefinition(def) if err == ErrInputDefinitionExists { @@ -1539,32 +1557,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ Index: indexName, - Name: inputDefName, Definition: def, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } +// handleGetInputDefinition handles GET /input-definition request. func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - //Find index. + // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } + inputDef, _ := index.inputDefinitions[inputDefName] - //inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ Frames: inputDef.frames, Fields: inputDef.fields, @@ -1574,6 +1590,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques } +// handleDeleteInputDefinition handles DELETE /input-definition request. func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1581,9 +1598,7 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1599,15 +1614,15 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req Name: inputDefName, }) if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type postInputDefinitionResponse struct{} +type defaultInputDefinitionResponse struct{} func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] @@ -1644,7 +1659,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 37fd9f25c..05c0db4d4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1084,7 +1084,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { }], "fields": [ { - "name": "id", + "name": "columnID", "primaryKey": true }, { @@ -1112,14 +1112,133 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusConflict { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionExists.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + +} + +// Ensure throwing error if there's duplicated primaryKey field. +func TestHandler_DuplicatePrimaryKey(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + + //Ensure throwing error if there's duplicated primaryKey field + invalidPrimaryKey := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + }, + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionPrimaryKey.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Eusure throwing error if primary field's name doesn't match columnLabel + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + unmatchColumnBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Eusure throwing error if request body is invalid + jsonErrorBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + + + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `unexpected EOF`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + } // Ensure handler can delete a input definition. func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + // Test index not found + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Test input definition is deleted + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} @@ -1128,11 +1247,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { if err != nil { t.Fatal(err) } - - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -1143,30 +1258,41 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } } -// Return existing input definition +// Ensure handler can get existing input definition func TestHandler_GetInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + + // Return error if index does not exist + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) + } + + // Return existing input definition + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } - response := &pilosa.InputDefinitionInfo{Frames: inputDef.Frames(), Fields: inputDef.Fields()} expect, err := json.Marshal(response) if err != nil { t.Fatal(err) } - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) diff --git a/index.go b/index.go index 7c97c0498..54a8f1433 100644 --- a/index.go +++ b/index.go @@ -51,14 +51,14 @@ type Index struct { // Frames by name. frames map[string]*Frame - // Max Slice on any node in the cluster, according to this node + // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - // Column attribute storage and cache + // Column attribute storage and cache. columnAttrStore *AttrStore - // InputDefinition by name + // InputDefinitions by name. inputDefinitions map[string]*InputDefinition broadcaster Broadcaster @@ -336,7 +336,7 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } -// InputDefinitionPath returns the path to a inputdefinition in the index. +// InputDefinitionPath returns the path to an input definition in the index. func (i *Index) InputDefinitionPath() string { return filepath.Join(i.path, InputDefinitionDir) } @@ -618,10 +618,10 @@ type IndexOptions struct { } // Encode converts i into its internal representation. -func (o *IndexOptions) Encode() *internal.IndexMeta { +func (i *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ - ColumnLabel: o.ColumnLabel, - TimeQuantum: string(o.TimeQuantum), + ColumnLabel: i.ColumnLabel, + TimeQuantum: string(i.TimeQuantum), } } @@ -656,9 +656,9 @@ func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefin func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { if pb.Name == "" { - return nil, errors.New("input-definition name required") + return nil, ErrInputDefinitionNameRequired } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { - return nil, errors.New("frames and fields are required") + return nil, ErrInputDefinitionAttrsRequired } for _, fr := range pb.Frames { @@ -702,7 +702,7 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { return inputDef, nil } -// DeleteInputDefinition removes a input definition from the index. +// DeleteInputDefinition removes an input definition from the index. func (i *Index) DeleteInputDefinition(name string) error { i.mu.Lock() defer i.mu.Unlock() @@ -742,7 +742,7 @@ func (i *Index) openInputDefinition() error { input.Open() i.inputDefinitions[file.Name()] = input - // Create frame if it doesn't exist + // Create frame if it doesn't exist. for _, fr := range input.frames { _, err := i.CreateFrame(fr.Name, fr.Options) if err == ErrFrameExists { diff --git a/index_test.go b/index_test.go index 8a7367636..aaebe288d 100644 --- a/index_test.go +++ b/index_test.go @@ -379,16 +379,31 @@ func TestIndex_CreateInputDefinition(t *testing.T) { } } +// Ensure create input definition handle correct error func TestIndex_CreateExistingInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() + // Test frames and fields are required + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionAttrsRequired { + t.Fatal(err) + } + + //Test input definition name is required + def = internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err = index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionNameRequired { + t.Fatal(err) + } + // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} - _, err := index.CreateInputDefinition(&def) + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -398,18 +413,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } } -func TestIndex_CreateEmptyInputDefinition(t *testing.T) { - index := MustOpenIndex() - defer index.Close() - - // Create Input Definition. - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} - _, err := index.CreateInputDefinition(&def) - if err.Error() != "frames and fields are required" { - t.Fatal(err) - } -} - +// Ensure to delete existing input definition. func TestIndex_DeleteInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() @@ -434,6 +438,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { } } +// Ensure that frame in input definition will be created when server restart func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() diff --git a/input_definition.go b/input_definition.go index aba684fb6..bef566416 100644 --- a/input_definition.go +++ b/input_definition.go @@ -82,7 +82,7 @@ func (i *InputDefinition) Open() error { return nil } -// LoadDefinition loads the protobuf format of a defition +// LoadDefinition loads the protobuf format of a definition. func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name @@ -101,7 +101,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } - numPrimaryKey := 0 countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action @@ -123,13 +122,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: &action.RowID, }) } - if field.PrimaryKey { - numPrimaryKey++ - } - - if numPrimaryKey > 1 { - return errors.New("duplicate primaryKey with other field") - } inputField := InputDefinitionField{ Name: field.Name, @@ -155,7 +147,7 @@ func (i *InputDefinition) loadMeta() error { return i.LoadDefinition(&pb) } -//saveMeta writes meta data for the input definition file. +// saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { return err @@ -253,6 +245,7 @@ func (o *Action) Encode() (*internal.InputDefinitionAction, error) { }, nil } +// convert pointer to uint64 func convert(x *uint64) uint64 { if x != nil { return *x @@ -266,7 +259,7 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -// InputDefinitionInfo the json message format to create an InputDefinition. +// InputDefinitionInfo represents the json message format needed to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` Fields []InputDefinitionField `json:"fields"` diff --git a/input_definition_test.go b/input_definition_test.go index 925adb724..112d8eac9 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -129,16 +129,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} - err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { - t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) - } - - action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} + field1 := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { diff --git a/internal/private.pb.go b/internal/private.pb.go index 8194a13de..abde718e9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -314,7 +314,6 @@ func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { type CreateInputDefinitionMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` } @@ -1160,12 +1159,6 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Name) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } if m.Definition != nil { dAtA[i] = 0x1a i++ @@ -1695,10 +1688,6 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Name) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.Definition != nil { l = m.Definition.Size() n += 1 + l + sovPrivate(uint64(l)) @@ -4284,35 +4273,6 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { } m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Definition", wireType) @@ -5030,63 +4990,62 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 915 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, + // 912 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, + 0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, 0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd, - 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0x66, 0xda, 0x49, 0xcc, 0x81, 0x0b, - 0x12, 0xcf, 0x80, 0xc4, 0x91, 0x97, 0xe1, 0x08, 0x6f, 0x80, 0xc2, 0x85, 0x37, 0xe0, 0x8a, 0xba, - 0xba, 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x2d, 0xdc, 0xba, 0xbe, 0xae, 0x9f, 0xaf, 0xaa, 0xab, 0x6a, - 0x06, 0xd6, 0xa7, 0x69, 0x78, 0xae, 0x34, 0x1e, 0x4c, 0xd3, 0x44, 0x27, 0xbc, 0x15, 0xc6, 0x1a, - 0xd3, 0x58, 0x45, 0xe2, 0x0b, 0x68, 0x0f, 0xe2, 0x53, 0xbc, 0x1c, 0xa2, 0x56, 0x7c, 0x17, 0xd6, - 0x0e, 0x93, 0x68, 0x36, 0x89, 0x3f, 0x57, 0x27, 0x18, 0x75, 0xd9, 0x2e, 0xdb, 0x6b, 0xcb, 0x32, - 0x64, 0x34, 0x9e, 0x85, 0x13, 0xfc, 0x6a, 0xa6, 0x62, 0x3d, 0x9b, 0x74, 0x6b, 0x56, 0xa3, 0x04, - 0x89, 0x3f, 0x18, 0xb4, 0x9f, 0xa4, 0x6a, 0x82, 0xe4, 0x71, 0x1b, 0x5a, 0x32, 0xb9, 0x28, 0xbb, - 0xcb, 0x65, 0xfe, 0x0e, 0x6c, 0x0c, 0xe2, 0x73, 0x4c, 0x33, 0x3c, 0x8a, 0xd5, 0x49, 0x84, 0xa7, - 0xe4, 0xae, 0x25, 0x2b, 0x28, 0xdf, 0x81, 0xf6, 0xa1, 0x1a, 0xbf, 0xc4, 0x67, 0xf3, 0x29, 0x76, - 0xeb, 0xe4, 0xa4, 0x00, 0xf2, 0xdb, 0x51, 0xf8, 0x3d, 0x76, 0x83, 0x5d, 0xb6, 0xb7, 0x2e, 0x0b, - 0xa0, 0xca, 0xb7, 0xb1, 0xc4, 0x97, 0x0b, 0x78, 0x43, 0xaa, 0xf8, 0x45, 0xce, 0xa1, 0x49, 0x1c, - 0x16, 0x30, 0x21, 0x60, 0x63, 0x30, 0x99, 0x26, 0xa9, 0x96, 0x98, 0x4d, 0x93, 0x38, 0x43, 0xde, - 0x81, 0xfa, 0x51, 0x9a, 0xba, 0x94, 0xcc, 0x51, 0xfc, 0x00, 0x9d, 0xc7, 0x51, 0x32, 0x3e, 0xeb, - 0x2b, 0xad, 0x24, 0x7e, 0x37, 0xc3, 0x4c, 0xf3, 0x2d, 0x68, 0x50, 0x71, 0x9d, 0x9e, 0x15, 0x0c, - 0x4a, 0x05, 0x72, 0xd5, 0xb3, 0x82, 0x41, 0xc9, 0x9e, 0x32, 0x0c, 0xa4, 0x15, 0x0c, 0x3a, 0x8a, - 0xc2, 0xb1, 0xcd, 0x2c, 0x90, 0x56, 0xe0, 0x1c, 0x82, 0xe7, 0x21, 0x5e, 0xb8, 0x74, 0xe8, 0x2c, - 0x06, 0x70, 0xa7, 0x14, 0xdf, 0xd1, 0xbc, 0x0b, 0x4d, 0x99, 0x5c, 0x0c, 0xfa, 0x59, 0x97, 0xed, - 0xd6, 0xf7, 0x02, 0xe9, 0x24, 0x2a, 0x1a, 0xbd, 0xaa, 0xb9, 0xaa, 0xd1, 0x55, 0x01, 0x88, 0x7b, - 0xd0, 0xa0, 0x0a, 0x9a, 0x2c, 0x0b, 0x5b, 0x73, 0x14, 0xbf, 0x30, 0xb8, 0x33, 0x54, 0x97, 0x44, - 0x23, 0xcb, 0xc3, 0x1c, 0x43, 0x3b, 0x07, 0x49, 0x7b, 0xed, 0xc1, 0xfe, 0x81, 0x6f, 0xb1, 0x83, - 0x25, 0xfd, 0x02, 0x39, 0x8a, 0x75, 0x3a, 0x97, 0x85, 0xf1, 0xf6, 0xc7, 0xb0, 0xb1, 0x78, 0x69, - 0x38, 0x9c, 0xe1, 0xdc, 0x57, 0xfa, 0x0c, 0xe7, 0xa6, 0x26, 0xe7, 0x2a, 0x9a, 0xd9, 0xfa, 0x05, - 0xd2, 0x0a, 0x1f, 0xd5, 0x3e, 0x60, 0xe2, 0x1b, 0xe0, 0x87, 0x29, 0x2a, 0x8d, 0xe4, 0x60, 0x88, - 0x59, 0xa6, 0x5e, 0xe0, 0xea, 0x57, 0xb0, 0x95, 0xad, 0x95, 0x2b, 0xbb, 0x03, 0xed, 0x41, 0xe6, - 0xfa, 0x8f, 0x5e, 0xa2, 0x25, 0x0b, 0x40, 0xec, 0x03, 0xef, 0x63, 0x84, 0x1a, 0xdd, 0xc8, 0xdc, - 0xe0, 0x5f, 0x8c, 0x3c, 0x97, 0xdb, 0x75, 0xf9, 0x7d, 0x08, 0xcc, 0xb4, 0x10, 0x95, 0xb5, 0x07, - 0x6f, 0x16, 0xa5, 0xcb, 0x47, 0x53, 0x92, 0x82, 0x08, 0xbd, 0x53, 0x37, 0x61, 0xb7, 0x24, 0x78, - 0x4d, 0x9b, 0xf9, 0x50, 0xf5, 0x6a, 0xa8, 0x7c, 0x66, 0x5d, 0xa8, 0x4f, 0x7c, 0xae, 0xff, 0x35, - 0x94, 0xe8, 0x3b, 0xd4, 0xb4, 0xeb, 0x53, 0x73, 0x6b, 0x6d, 0xe8, 0xbc, 0x3a, 0xe5, 0x2a, 0x8f, - 0xbf, 0x99, 0x0b, 0xf9, 0x6a, 0x6e, 0x2a, 0x95, 0x33, 0x8b, 0xc8, 0x37, 0x96, 0x9b, 0xb0, 0x5c, - 0xe6, 0xf7, 0xa1, 0x49, 0x51, 0xb3, 0x6e, 0x40, 0xbd, 0xbb, 0x59, 0x61, 0x23, 0xdd, 0xb5, 0x19, - 0x27, 0xd7, 0xe4, 0x0d, 0x3b, 0x4e, 0x56, 0xe2, 0x47, 0xd0, 0x19, 0xc4, 0xd3, 0x99, 0xee, 0xe3, - 0xb7, 0x61, 0x1c, 0xea, 0x30, 0x89, 0xb3, 0x6e, 0x93, 0x5c, 0xdd, 0x2b, 0x33, 0x5a, 0xd0, 0x90, - 0x4b, 0x26, 0xe2, 0x27, 0x06, 0x9b, 0x15, 0x70, 0x45, 0xd2, 0x9e, 0x6f, 0xed, 0x66, 0xbe, 0x0f, - 0xa1, 0xf9, 0x24, 0xc4, 0xe8, 0x34, 0xeb, 0xd6, 0x49, 0xb1, 0xb7, 0x92, 0x0d, 0xa9, 0x49, 0xa7, - 0x2d, 0x7e, 0x65, 0xb0, 0x75, 0x9d, 0xc2, 0xb5, 0x6c, 0x7a, 0x00, 0x5f, 0xa6, 0xe1, 0x44, 0xa5, - 0xf3, 0xcf, 0x70, 0xee, 0x56, 0x78, 0x09, 0xe1, 0x5f, 0xc3, 0xdd, 0x8a, 0xaf, 0x4f, 0xc7, 0xb6, - 0x44, 0x96, 0xd4, 0xdb, 0x2b, 0x49, 0x59, 0x3d, 0xb9, 0xc2, 0x5c, 0xfc, 0xc3, 0xe0, 0xad, 0x6b, - 0xaf, 0x8a, 0x7e, 0x64, 0xe5, 0xd6, 0xdf, 0x87, 0xce, 0x73, 0xb3, 0x2a, 0xfa, 0x98, 0xe9, 0x30, - 0x56, 0x46, 0xd3, 0x35, 0xec, 0x12, 0xce, 0x07, 0xd0, 0x22, 0x6c, 0xa8, 0xa6, 0x8e, 0xe6, 0x7b, - 0xb7, 0xd0, 0x3c, 0xf0, 0xfa, 0x76, 0xa7, 0xe5, 0xe6, 0x86, 0x0c, 0x6d, 0x5d, 0xbf, 0xc2, 0x49, - 0xd8, 0x7e, 0x04, 0xeb, 0x0b, 0x06, 0xaf, 0xb4, 0xe7, 0x7e, 0x64, 0xb0, 0xe3, 0x97, 0xcb, 0x02, - 0x95, 0x9b, 0xc7, 0xd4, 0xbf, 0x5e, 0xad, 0xf4, 0x7a, 0x1f, 0x02, 0x14, 0xe6, 0x6e, 0x2b, 0xdc, - 0xd0, 0xb4, 0x25, 0x65, 0x71, 0x0c, 0x3b, 0x7e, 0x1b, 0xfe, 0x3f, 0x12, 0x42, 0x01, 0x3c, 0x4d, - 0x4e, 0x71, 0xa4, 0x95, 0x9e, 0x65, 0x46, 0xe3, 0x38, 0xc9, 0xb4, 0x6f, 0x32, 0x73, 0xa6, 0x6d, - 0xad, 0x95, 0xce, 0x37, 0x0c, 0x09, 0xfc, 0x5d, 0x78, 0x9d, 0x9c, 0xa2, 0xef, 0xa5, 0xcd, 0xca, - 0x02, 0x90, 0xfe, 0x5e, 0x3c, 0x82, 0xf5, 0xc3, 0x68, 0x96, 0x69, 0x4c, 0x5d, 0x94, 0x7d, 0x68, - 0x98, 0x98, 0xfe, 0x7b, 0xb5, 0x55, 0x58, 0x16, 0x54, 0xa4, 0x55, 0x11, 0x0f, 0x61, 0x8d, 0x5a, - 0x68, 0x34, 0x7e, 0x89, 0x13, 0x45, 0xf3, 0x67, 0xc7, 0x8a, 0x2d, 0xcd, 0xdf, 0xc2, 0x1c, 0x8d, - 0xa0, 0xb1, 0x7a, 0x6e, 0x38, 0x04, 0xf4, 0x47, 0xe3, 0x0a, 0x41, 0x3f, 0x33, 0x1d, 0xa8, 0x0f, - 0x43, 0xfb, 0x0c, 0x75, 0x69, 0x8e, 0x84, 0xa8, 0x4b, 0xea, 0x1d, 0x83, 0xa8, 0xcb, 0xc7, 0x9d, - 0xdf, 0xae, 0x7a, 0xec, 0xf7, 0xab, 0x1e, 0xfb, 0xf3, 0xaa, 0xc7, 0x7e, 0xfe, 0xab, 0xf7, 0xda, - 0x49, 0x93, 0x7e, 0xea, 0xde, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb5, 0x12, 0x7e, 0xdc, 0xe5, - 0x09, 0x00, 0x00, + 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0xa6, 0xdb, 0x49, 0xcc, 0x81, 0x23, + 0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0x8a, 0xba, 0xba, + 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x85, 0x5b, 0xd7, 0xeb, 0xd7, 0x55, 0xaf, 0x6a, 0xaa, 0xca, 0x86, + 0xf5, 0x69, 0x16, 0x9d, 0x4b, 0x8d, 0x07, 0xd3, 0x2c, 0xd5, 0x29, 0x6b, 0x45, 0x89, 0xc6, 0x2c, + 0x91, 0x31, 0xff, 0x0a, 0xda, 0x83, 0xe4, 0x14, 0x2f, 0x87, 0xa8, 0x25, 0xdb, 0x85, 0xb5, 0xc3, + 0x34, 0x9e, 0x4d, 0x92, 0x2f, 0xe5, 0x09, 0xc6, 0xdd, 0x60, 0x37, 0xd8, 0x6b, 0x8b, 0x32, 0x64, + 0x18, 0x2f, 0xa2, 0x09, 0x7e, 0x33, 0x93, 0x89, 0x9e, 0x4d, 0xba, 0x35, 0xcb, 0x28, 0x41, 0xfc, + 0xcf, 0x00, 0xda, 0xcf, 0x32, 0x39, 0x41, 0xf2, 0xb8, 0x0d, 0x2d, 0x91, 0x5e, 0x94, 0xdd, 0xe5, + 0x36, 0x7b, 0x0f, 0x36, 0x06, 0xc9, 0x39, 0x66, 0x0a, 0x8f, 0x12, 0x79, 0x12, 0xe3, 0x29, 0xb9, + 0x6b, 0x89, 0x0a, 0xca, 0x76, 0xa0, 0x7d, 0x28, 0xc7, 0xaf, 0xf1, 0xc5, 0x7c, 0x8a, 0xdd, 0x3a, + 0x39, 0x29, 0x80, 0xfc, 0x76, 0x14, 0xfd, 0x88, 0xdd, 0x70, 0x37, 0xd8, 0x5b, 0x17, 0x05, 0x50, + 0xd5, 0xdb, 0x58, 0xd2, 0xcb, 0x38, 0xbc, 0x25, 0x64, 0xf2, 0x2a, 0xd7, 0xd0, 0x24, 0x0d, 0x0b, + 0x18, 0xe7, 0xb0, 0x31, 0x98, 0x4c, 0xd3, 0x4c, 0x0b, 0x54, 0xd3, 0x34, 0x51, 0xc8, 0x3a, 0x50, + 0x3f, 0xca, 0x32, 0x97, 0x92, 0x39, 0xf2, 0x9f, 0xa0, 0xf3, 0x34, 0x4e, 0xc7, 0x67, 0x7d, 0xa9, + 0xa5, 0xc0, 0x1f, 0x66, 0xa8, 0x34, 0xdb, 0x82, 0x06, 0x15, 0xd7, 0xf1, 0xac, 0x61, 0x50, 0x2a, + 0x90, 0xab, 0x9e, 0x35, 0x0c, 0x4a, 0xef, 0x29, 0xc3, 0x50, 0x58, 0xc3, 0xa0, 0xa3, 0x38, 0x1a, + 0xdb, 0xcc, 0x42, 0x61, 0x0d, 0xc6, 0x20, 0x7c, 0x19, 0xe1, 0x85, 0x4b, 0x87, 0xce, 0x7c, 0x00, + 0xf7, 0x4a, 0xf1, 0x9d, 0xcc, 0xfb, 0xd0, 0x14, 0xe9, 0xc5, 0xa0, 0xaf, 0xba, 0xc1, 0x6e, 0x7d, + 0x2f, 0x14, 0xce, 0xa2, 0xa2, 0xd1, 0x57, 0x35, 0x57, 0x35, 0xba, 0x2a, 0x00, 0xfe, 0x00, 0x1a, + 0x54, 0x41, 0x93, 0x65, 0xf1, 0xd6, 0x1c, 0xf9, 0xaf, 0x01, 0xdc, 0x1b, 0xca, 0x4b, 0x92, 0xa1, + 0xf2, 0x30, 0xc7, 0xd0, 0xce, 0x41, 0x62, 0xaf, 0x3d, 0xda, 0x3f, 0xf0, 0x2d, 0x76, 0xb0, 0xc4, + 0x2f, 0x90, 0xa3, 0x44, 0x67, 0x73, 0x51, 0x3c, 0xde, 0xfe, 0x14, 0x36, 0x16, 0x2f, 0x8d, 0x86, + 0x33, 0x9c, 0xfb, 0x4a, 0x9f, 0xe1, 0xdc, 0xd4, 0xe4, 0x5c, 0xc6, 0x33, 0x5b, 0xbf, 0x50, 0x58, + 0xe3, 0x93, 0xda, 0x47, 0x01, 0xff, 0x0e, 0xd8, 0x61, 0x86, 0x52, 0x23, 0x39, 0x18, 0xa2, 0x52, + 0xf2, 0x15, 0xae, 0xfe, 0x0a, 0xb6, 0xb2, 0xb5, 0x72, 0x65, 0x77, 0xa0, 0x3d, 0x50, 0xae, 0xff, + 0xe8, 0x4b, 0xb4, 0x44, 0x01, 0xf0, 0x7d, 0x60, 0x7d, 0x8c, 0x51, 0xa3, 0x1b, 0x99, 0x1b, 0xfc, + 0xf3, 0x91, 0xd7, 0x72, 0x3b, 0x97, 0x3d, 0x84, 0xd0, 0x4c, 0x0b, 0x49, 0x59, 0x7b, 0xf4, 0x76, + 0x51, 0xba, 0x7c, 0x34, 0x05, 0x11, 0x78, 0xe4, 0x9d, 0xba, 0x09, 0xbb, 0x25, 0xc1, 0x6b, 0xda, + 0xcc, 0x87, 0xaa, 0x57, 0x43, 0xe5, 0x33, 0xeb, 0x42, 0x7d, 0xe6, 0x73, 0xfd, 0xbf, 0xa1, 0x78, + 0xdf, 0xa1, 0xa6, 0x5d, 0x9f, 0x9b, 0x5b, 0xfb, 0x86, 0xce, 0xab, 0x53, 0xae, 0xea, 0xf8, 0x27, + 0x70, 0x21, 0xef, 0xe6, 0xa6, 0x52, 0x39, 0xb3, 0x88, 0x7c, 0x63, 0xb9, 0x09, 0xcb, 0x6d, 0xf6, + 0x10, 0x9a, 0x14, 0x55, 0x75, 0x43, 0xea, 0xdd, 0xcd, 0x8a, 0x1a, 0xe1, 0xae, 0xcd, 0x38, 0xb9, + 0x26, 0x6f, 0xd8, 0x71, 0xb2, 0x16, 0x3b, 0x82, 0xce, 0x20, 0x99, 0xce, 0x74, 0x1f, 0xbf, 0x8f, + 0x92, 0x48, 0x47, 0x69, 0xa2, 0xba, 0x4d, 0x72, 0xf5, 0xa0, 0xac, 0x68, 0x81, 0x21, 0x96, 0x9e, + 0xf0, 0x9f, 0x03, 0xd8, 0xac, 0x80, 0x2b, 0x92, 0xf6, 0x7a, 0x6b, 0x37, 0xeb, 0x7d, 0x0c, 0xcd, + 0x67, 0x11, 0xc6, 0xa7, 0xaa, 0x5b, 0x27, 0x62, 0x6f, 0xa5, 0x1a, 0xa2, 0x09, 0xc7, 0xe6, 0xbf, + 0x05, 0xb0, 0x75, 0x1d, 0xe1, 0x5a, 0x35, 0x3d, 0x80, 0xaf, 0xb3, 0x68, 0x22, 0xb3, 0xf9, 0x17, + 0x38, 0x77, 0x2b, 0xbc, 0x84, 0xb0, 0x6f, 0xe1, 0x7e, 0xc5, 0xd7, 0xe7, 0x63, 0x5b, 0x22, 0x2b, + 0xea, 0xdd, 0x95, 0xa2, 0x2c, 0x4f, 0xac, 0x78, 0xce, 0xff, 0x0d, 0xe0, 0x9d, 0x6b, 0xaf, 0x8a, + 0x7e, 0x0c, 0xca, 0xad, 0xbf, 0x0f, 0x9d, 0x97, 0x66, 0x55, 0xf4, 0x51, 0xe9, 0x28, 0x91, 0x86, + 0xe9, 0x1a, 0x76, 0x09, 0x67, 0x03, 0x68, 0x11, 0x36, 0x94, 0x53, 0x27, 0xf3, 0x83, 0x5b, 0x64, + 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xf9, 0x73, 0x23, 0x86, 0xb6, 0xae, 0x5f, 0xe1, 0x64, 0x6c, 0x3f, + 0x81, 0xf5, 0x85, 0x07, 0x77, 0xda, 0x73, 0x29, 0xec, 0xf8, 0xdd, 0xb2, 0xa0, 0xe4, 0xe6, 0x29, + 0xfd, 0x18, 0xa0, 0xa0, 0xba, 0x05, 0x70, 0x43, 0x7f, 0x96, 0xc8, 0xfc, 0x18, 0x76, 0xfc, 0xe2, + 0xbb, 0x43, 0x40, 0xdf, 0x2d, 0xb5, 0xa2, 0x5b, 0xb8, 0x04, 0x78, 0x9e, 0x9e, 0xe2, 0x48, 0x4b, + 0x3d, 0x53, 0x86, 0x71, 0x9c, 0x2a, 0xed, 0xfb, 0xc9, 0x9c, 0x69, 0x31, 0x6b, 0xa9, 0xf3, 0x65, + 0x42, 0x06, 0x7b, 0x1f, 0xde, 0x24, 0xa7, 0xe8, 0xdb, 0x66, 0xb3, 0x32, 0xeb, 0xc2, 0xdf, 0xf3, + 0x27, 0xb0, 0x7e, 0x18, 0xcf, 0x94, 0xc6, 0xcc, 0x45, 0xd9, 0x87, 0x86, 0x89, 0xe9, 0x7f, 0x9a, + 0xb6, 0x8a, 0x97, 0x85, 0x14, 0x61, 0x29, 0xfc, 0x31, 0xac, 0x51, 0xb7, 0x8c, 0xc6, 0xaf, 0x71, + 0x22, 0x69, 0xd4, 0xec, 0x04, 0x05, 0x4b, 0xa3, 0xb6, 0x30, 0x32, 0x23, 0x68, 0xac, 0x1e, 0x11, + 0x06, 0x21, 0xfd, 0x79, 0x71, 0x85, 0xa0, 0xff, 0x2d, 0x1d, 0xa8, 0x0f, 0x23, 0xfb, 0x19, 0xea, + 0xc2, 0x1c, 0x09, 0x91, 0x97, 0xd4, 0x26, 0x06, 0x91, 0x97, 0x4f, 0x3b, 0xbf, 0x5f, 0xf5, 0x82, + 0x3f, 0xae, 0x7a, 0xc1, 0x5f, 0x57, 0xbd, 0xe0, 0x97, 0xbf, 0x7b, 0x6f, 0x9c, 0x34, 0xe9, 0xff, + 0xdb, 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x29, 0x07, 0x36, 0x04, 0xd0, 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index f69b0de7b..3173b12a2 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -103,7 +103,6 @@ message InputDefinitionAction { message CreateInputDefinitionMessage { string Index = 1; - string Name = 2; InputDefinition Definition = 3; } diff --git a/pilosa.go b/pilosa.go index 3edcbfad9..bde8cc2c7 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,8 +36,11 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrInputDefinitionNotFound = errors.New("input-definition not found") + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionPrimaryKey = errors.New("input-definition can only contain one PrimaryKey") + ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") + ErrInputDefinitionNameRequired = errors.New("input-definition name required") + ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") @@ -45,6 +48,7 @@ var ( ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") ErrRangeCacheNotAllowed = errors.New("range cache not allowed") ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/server/server_test.go b/server/server_test.go index 6f4570709..b6af8c423 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -540,7 +540,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { "cacheType": "ranked", "timeQuantum": "YMD" }}], - "fields": [{"name": "id", + "fields": [{"name": "columnID", "primaryKey": true }]} `); err != nil {