diff --git a/handler.go b/handler.go index 464506ae8..bf8dbdd7e 100644 --- a/handler.go +++ b/handler.go @@ -1505,7 +1505,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { inputDefName := mux.Vars(r)["input-definition"] // Decode request. - var req internal.InputDefinition + var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1519,14 +1519,24 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } - // Add the name - req.Name = inputDefName + def := req.Encode() + def.Name = inputDefName + + // Create InputDefinition. + _, err = index.CreateInputDefinition(def) + if err == ErrInputDefinitionExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ Index: indexName, Name: inputDefName, - Definition: &req, + Definition: def, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) @@ -1584,14 +1594,4 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) } } -type InputFrame struct { - Name string `json:"name,omitempty"` - Options FrameOptions `json:"options,omitempty"` -} - -type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []Field `json:"fields"` -} - type postInputDefinitionResponse struct{} diff --git a/handler_test.go b/handler_test.go index a89244d49..45ca9f021 100644 --- a/handler_test.go +++ b/handler_test.go @@ -215,7 +215,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code, w.Body.String()) + t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) } @@ -1120,10 +1120,11 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -1138,7 +1139,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } else if index.InputDefinition("test") != nil { - t.Fatalf("unexpected result: %s", index.InputDefinition("test")) + t.Fatalf("unexpected result: %v", index.InputDefinition("test")) } } @@ -1148,10 +1149,11 @@ func TestHandler_GetInputDefinition(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } diff --git a/index.go b/index.go index 4dc40501b..581da85a4 100644 --- a/index.go +++ b/index.go @@ -613,24 +613,30 @@ type importData struct { } // CreateInputDefinition creates a new input definition. -func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { - +func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { // Ensure input definition doesn't already exist. - if i.inputDefinitions[name] != nil { + if i.inputDefinitions[pb.Name] != nil { return nil, ErrInputDefinitionExists } - return i.createInputDefinition(name, frames, field) + return i.createInputDefinition(pb) } -func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) { - if name == "" { +func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { + if pb.Name == "" { return nil, errors.New("input-definition name required") - } else if len(frames) == 0 || len(fields) == 0 { + } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { return nil, errors.New("frames and fields are required") } - for _, fr := range frames { - _, err := i.CreateFrame(fr.Name, fr.Options) + for _, fr := range pb.Frames { + opt := FrameOptions{ + RowLabel: fr.Meta.RowLabel, + InverseEnabled: fr.Meta.InverseEnabled, + CacheType: fr.Meta.CacheType, + CacheSize: fr.Meta.CacheSize, + TimeQuantum: TimeQuantum(fr.Meta.TimeQuantum), + } + _, err := i.CreateFrame(fr.Name, opt) if err == ErrFrameExists { continue } else if err != nil { @@ -639,17 +645,16 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [ } // Initialize input definition. - inputDef, err := i.newInputDefinition(i.InputDefPath(), name) + inputDef, err := i.newInputDefinition(i.InputDefPath(), pb.Name) if err != nil { return nil, err } - inputDef.frames = frames - inputDef.fields = fields + inputDef.LoadDefinition(pb) if err = inputDef.saveMeta(); err != nil { return nil, err } - i.inputDefinitions[name] = inputDef + i.inputDefinitions[pb.Name] = inputDef return inputDef, nil } diff --git a/index_test.go b/index_test.go index 125be84c4..5d5e03ab3 100644 --- a/index_test.go +++ b/index_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/pilosa/pilosa" - "reflect" + "github.com/pilosa/pilosa/internal" ) // Ensure index can open and retrieve a frame. @@ -237,7 +237,7 @@ func TestIndex_InvalidName(t *testing.T) { } index, err := pilosa.NewIndex(path, "ABC") if index != nil { - t.Fatalf("unexpected index name %s", index) + t.Fatalf("unexpected index name %v", index) } } @@ -246,16 +246,17 @@ func TestIndex_CreateInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) - } else if inputDef.Frames()[0] != frames { - t.Fatalf("unexpected input definition frames", inputDef.Frames()) - } else if !reflect.DeepEqual(inputDef.Fields()[0], fields) { - t.Fatalf("unexpected input definition actions", inputDef.Fields()) + } else if inputDef.Frames()[0].Name != frames.Name { + t.Fatalf("unexpected input definition frames %v", inputDef.Frames()) + } else if inputDef.Fields()[0].Name != fields.Name { + t.Fatalf("unexpected input definition actions %v", inputDef.Fields()) } } @@ -264,14 +265,15 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } - _, err = index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + _, err = index.CreateInputDefinition(&def) if err != pilosa.ErrInputDefinitionExists { t.Fatal(err) } @@ -282,7 +284,8 @@ func TestIndex_CreateEmptyInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{}) + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) if err.Error() != "frames and fields are required" { t.Fatal(err) } @@ -293,10 +296,11 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } else if index.InputDefinition("test") == nil { @@ -309,5 +313,4 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { } else if index.InputDefinition("test") != nil { t.Fatal("input definition isn't deleted") } - } diff --git a/input_definition.go b/input_definition.go index baa8bf215..d02d4813f 100644 --- a/input_definition.go +++ b/input_definition.go @@ -53,16 +53,8 @@ func (i *InputDefinition) Open() error { return nil } -func (i *InputDefinition) loadMeta() error { - var pb internal.InputDefinition - buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name)) - if err != nil { - return err - } else { - if err := proto.Unmarshal(buf, &pb); err != nil { - return err - } - } +// LoadDefinition loads the protobuf format of a defition +func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name for _, fr := range pb.Frames { @@ -98,9 +90,23 @@ func (i *InputDefinition) loadMeta() error { } i.fields = append(i.fields, inputField) } + return nil } +func (i *InputDefinition) loadMeta() error { + var pb internal.InputDefinition + buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name)) + if err != nil { + return err + } + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + + return i.LoadDefinition(&pb) +} + //saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { @@ -157,15 +163,61 @@ func (i *InputDefinition) saveMeta() error { return nil } +// Field descripes a single field mapping in the InputDefinition. type Field struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` Actions []Action `json:"actions,omitempty"` } +// Encode converts Field into its internal representation. +func (o *Field) Encode() *internal.InputDefinitionField { + field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { + field.Actions = append(field.Actions, action.Encode()) + } + return &field +} + +// Action descripes the mapping method for the field in the InputDefinition. type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` RowID uint64 `json:"rowID,omitempty"` } + +// Encode converts Action into its internal representation. +func (o *Action) Encode() *internal.Action { + return &internal.Action{ + Frame: o.Frame, + ValueDestination: o.ValueDestination, + ValueMap: o.ValueMap, + RowID: o.RowID, + } +} + +// InputFrame defines the frame used in the input definition. +type InputFrame struct { + Name string `json:"name,omitempty"` + Options FrameOptions `json:"options,omitempty"` +} + +// InputDefinitionInfo the json message format to create an InputDefinition. +type InputDefinitionInfo struct { + Frames []InputFrame `json:"frames"` + Fields []Field `json:"fields"` +} + +// Encode converts InputDefinitionInfo into its internal representation. +func (o *InputDefinitionInfo) Encode() *internal.InputDefinition { + var def internal.InputDefinition + for _, f := range o.Frames { + def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) + } + for _, f := range o.Fields { + def.Fields = append(def.Fields, f.Encode()) + } + + return &def +} diff --git a/input_definition_test.go b/input_definition_test.go index 5fd78c60c..037a97c28 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -15,8 +15,11 @@ package pilosa_test import ( - "github.com/pilosa/pilosa" + "encoding/json" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" ) func TestInputDefinition_Open(t *testing.T) { @@ -24,10 +27,11 @@ func TestInputDefinition_Open(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -36,3 +40,56 @@ func TestInputDefinition_Open(t *testing.T) { t.Fatal(err) } } + +// Verify the InputDefinition Encoding to the internal format +func TestInputDefinition_Encoding(t *testing.T) { + inputBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": true, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "id", + "primaryKey": true + }, + { + "name": "cabType", + "actions": [ + { + "frame": "cab-type", + "valueDestination": "mapping", + "valueMap": { + "Green": 1, + "Yellow": 2 + } + } + ] + } + ] + }`) + var def pilosa.InputDefinitionInfo + err := json.Unmarshal(inputBody, &def) + if err != nil { + t.Fatal(err) + } + + internalDef := def.Encode() + + if internalDef.Frames[0].Name != "event-time" { + t.Fatalf("unexpected frame: %v", internalDef) + } else if internalDef.Frames[0].Meta.CacheType != "ranked" { + t.Fatalf("unexpected frame meta data: %v", internalDef) + } else if len(internalDef.Fields) != 2 { + t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) + } else if len(internalDef.Fields[1].Actions) != 1 { + t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions) + } else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" { + t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) + } +} diff --git a/server.go b/server.go index d9b262088..6903c216b 100644 --- a/server.go +++ b/server.go @@ -311,15 +311,27 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { CacheSize: obj.Meta.CacheSize, TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), } - _, err := idx.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } + err := s.createFrame(idx, obj.Frame, opt) + return err case *internal.DeleteFrameMessage: idx := s.Holder.Index(obj.Index) if err := idx.DeleteFrame(obj.Frame); err != nil { return err } + case *internal.CreateInputDefinitionMessage: + idx := s.Holder.Index(obj.Index) + if idx == nil { + return fmt.Errorf("Local Index not found: %s", obj.Index) + } + idx.CreateInputDefinition(obj.Definition) + } + return nil +} + +func (s *Server) createFrame(idx *Index, frame string, opt FrameOptions) error { + _, err := idx.CreateFrame(frame, opt) + if err != nil { + return err } return nil }