Use the internal.inputdefinition object for broadcasting to other nodes. Create encoding methods to translate the JSON request to create a definition.

handler.go

@@ -1505,7 +1505,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) {
 	inputDefName := mux.Vars(r)["input-definition"]
 
 	// Decode request.
-	var req internal.InputDefinition
+	var req InputDefinitionInfo
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
@@ -1519,14 +1519,24 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Add the name
-	req.Name = inputDefName
+	def := req.Encode()
+	def.Name = inputDefName
+
+	// Create InputDefinition.
+	_, err = index.CreateInputDefinition(def)
+	if err == ErrInputDefinitionExists {
+		http.Error(w, err.Error(), http.StatusConflict)
+		return
+	} else if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 
 	err = h.Broadcaster.SendSync(
 		&internal.CreateInputDefinitionMessage{
 			Index:      indexName,
 			Name:       inputDefName,
-			Definition: &req,
+			Definition: def,
 		})
 	if err != nil {
 		h.logger().Printf("problem sending CreateInputDefinition message: %s", err)
@@ -1584,14 +1594,4 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-type InputFrame struct {
-	Name    string       `json:"name,omitempty"`
-	Options FrameOptions `json:"options,omitempty"`
-}
-
-type InputDefinitionInfo struct {
-	Frames []InputFrame `json:"frames"`
-	Fields []Field      `json:"fields"`
-}
-
 type postInputDefinitionResponse struct{}

handler_test.go

@@ -215,7 +215,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
 	w := httptest.NewRecorder()
 	h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
 	if w.Code != http.StatusOK {
-		t.Fatalf("unexpected status code: %d", w.Code, w.Body.String())
+		t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
 	} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
 		t.Fatalf("unexpected body: %q", body)
 	}
@@ -1120,10 +1120,11 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
 	defer hldr.Close()
 	index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
 
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	_, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	_, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1138,7 +1139,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) {
 	} else if body := w.Body.String(); body != `{}`+"\n" {
 		t.Fatalf("unexpected body: %s", body)
 	} else if index.InputDefinition("test") != nil {
-		t.Fatalf("unexpected result: %s", index.InputDefinition("test"))
+		t.Fatalf("unexpected result: %v", index.InputDefinition("test"))
 	}
 }
 
@@ -1148,10 +1149,11 @@ func TestHandler_GetInputDefinition(t *testing.T) {
 	defer hldr.Close()
 	index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
 
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	inputDef, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	}

index.go

@@ -613,24 +613,30 @@ type importData struct {
 }
 
 // CreateInputDefinition creates a new input definition.
-func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) {
-
+func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) {
 	// Ensure input definition doesn't already exist.
-	if i.inputDefinitions[name] != nil {
+	if i.inputDefinitions[pb.Name] != nil {
 		return nil, ErrInputDefinitionExists
 	}
-	return i.createInputDefinition(name, frames, field)
+	return i.createInputDefinition(pb)
 }
 
-func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) {
-	if name == "" {
+func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) {
+	if pb.Name == "" {
 		return nil, errors.New("input-definition name required")
-	} else if len(frames) == 0 || len(fields) == 0 {
+	} else if len(pb.Frames) == 0 || len(pb.Fields) == 0 {
 		return nil, errors.New("frames and fields are required")
 	}
 
-	for _, fr := range frames {
-		_, err := i.CreateFrame(fr.Name, fr.Options)
+	for _, fr := range pb.Frames {
+		opt := FrameOptions{
+			RowLabel:       fr.Meta.RowLabel,
+			InverseEnabled: fr.Meta.InverseEnabled,
+			CacheType:      fr.Meta.CacheType,
+			CacheSize:      fr.Meta.CacheSize,
+			TimeQuantum:    TimeQuantum(fr.Meta.TimeQuantum),
+		}
+		_, err := i.CreateFrame(fr.Name, opt)
 		if err == ErrFrameExists {
 			continue
 		} else if err != nil {
@@ -639,17 +645,16 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [
 	}
 
 	// Initialize input definition.
-	inputDef, err := i.newInputDefinition(i.InputDefPath(), name)
+	inputDef, err := i.newInputDefinition(i.InputDefPath(), pb.Name)
 	if err != nil {
 		return nil, err
 	}
 
-	inputDef.frames = frames
-	inputDef.fields = fields
+	inputDef.LoadDefinition(pb)
 	if err = inputDef.saveMeta(); err != nil {
 		return nil, err
 	}
-	i.inputDefinitions[name] = inputDef
+	i.inputDefinitions[pb.Name] = inputDef
 	return inputDef, nil
 }
 

index_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 
 	"github.com/pilosa/pilosa"
-	"reflect"
+	"github.com/pilosa/pilosa/internal"
 )
 
 // Ensure index can open and retrieve a frame.
@@ -237,7 +237,7 @@ func TestIndex_InvalidName(t *testing.T) {
 	}
 	index, err := pilosa.NewIndex(path, "ABC")
 	if index != nil {
-		t.Fatalf("unexpected index name %s", index)
+		t.Fatalf("unexpected index name %v", index)
 	}
 }
 
@@ -246,16 +246,17 @@ func TestIndex_CreateInputDefinition(t *testing.T) {
 	defer index.Close()
 
 	// Create Input Definition.
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	inputDef, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
-	} else if inputDef.Frames()[0] != frames {
-		t.Fatalf("unexpected input definition frames", inputDef.Frames())
-	} else if !reflect.DeepEqual(inputDef.Fields()[0], fields) {
-		t.Fatalf("unexpected input definition actions", inputDef.Fields())
+	} else if inputDef.Frames()[0].Name != frames.Name {
+		t.Fatalf("unexpected input definition frames %v", inputDef.Frames())
+	} else if inputDef.Fields()[0].Name != fields.Name {
+		t.Fatalf("unexpected input definition actions %v", inputDef.Fields())
 	}
 }
 
@@ -264,14 +265,15 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) {
 	defer index.Close()
 
 	// Create Input Definition.
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	_, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	_, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	_, err = index.CreateInputDefinition(&def)
 	if err != pilosa.ErrInputDefinitionExists {
 		t.Fatal(err)
 	}
@@ -282,7 +284,8 @@ func TestIndex_CreateEmptyInputDefinition(t *testing.T) {
 	defer index.Close()
 
 	// Create Input Definition.
-	_, err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{})
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}}
+	_, err := index.CreateInputDefinition(&def)
 	if err.Error() != "frames and fields are required" {
 		t.Fatal(err)
 	}
@@ -293,10 +296,11 @@ func TestIndex_DeleteInputDefinition(t *testing.T) {
 	defer index.Close()
 
 	// Create Input Definition.
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	_, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	_, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	} else if index.InputDefinition("test") == nil {
@@ -309,5 +313,4 @@ func TestIndex_DeleteInputDefinition(t *testing.T) {
 	} else if index.InputDefinition("test") != nil {
 		t.Fatal("input definition isn't deleted")
 	}
-
 }

input_definition.go

@@ -53,16 +53,8 @@ func (i *InputDefinition) Open() error {
 	return nil
 }
 
-func (i *InputDefinition) loadMeta() error {
-	var pb internal.InputDefinition
-	buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name))
-	if err != nil {
-		return err
-	} else {
-		if err := proto.Unmarshal(buf, &pb); err != nil {
-			return err
-		}
-	}
+// LoadDefinition loads the protobuf format of a defition
+func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
 	// Copy metadata fields.
 	i.name = pb.Name
 	for _, fr := range pb.Frames {
@@ -98,9 +90,23 @@ func (i *InputDefinition) loadMeta() error {
 		}
 		i.fields = append(i.fields, inputField)
 	}
+
 	return nil
 }
 
+func (i *InputDefinition) loadMeta() error {
+	var pb internal.InputDefinition
+	buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name))
+	if err != nil {
+		return err
+	}
+	if err := proto.Unmarshal(buf, &pb); err != nil {
+		return err
+	}
+
+	return i.LoadDefinition(&pb)
+}
+
 //saveMeta writes meta data for the input definition file.
 func (i *InputDefinition) saveMeta() error {
 	if err := os.MkdirAll(i.path, 0777); err != nil {
@@ -157,15 +163,61 @@ func (i *InputDefinition) saveMeta() error {
 	return nil
 }
 
+// Field descripes a single field mapping in the InputDefinition.
 type Field struct {
 	Name       string   `json:"name,omitempty"`
 	PrimaryKey bool     `json:"primaryKey,omitempty"`
 	Actions    []Action `json:"actions,omitempty"`
 }
 
+// Encode converts Field into its internal representation.
+func (o *Field) Encode() *internal.InputDefinitionField {
+	field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey}
+	for _, action := range o.Actions {
+		field.Actions = append(field.Actions, action.Encode())
+	}
+	return &field
+}
+
+// Action descripes the mapping method for the field in the InputDefinition.
 type Action struct {
 	Frame            string            `json:"frame,omitempty"`
 	ValueDestination string            `json:"valueDestination,omitempty"`
 	ValueMap         map[string]uint64 `json:"valueMap,omitempty"`
 	RowID            uint64            `json:"rowID,omitempty"`
 }
+
+// Encode converts Action into its internal representation.
+func (o *Action) Encode() *internal.Action {
+	return &internal.Action{
+		Frame:            o.Frame,
+		ValueDestination: o.ValueDestination,
+		ValueMap:         o.ValueMap,
+		RowID:            o.RowID,
+	}
+}
+
+// InputFrame defines the frame used in the input definition.
+type InputFrame struct {
+	Name    string       `json:"name,omitempty"`
+	Options FrameOptions `json:"options,omitempty"`
+}
+
+// InputDefinitionInfo the json message format to create an InputDefinition.
+type InputDefinitionInfo struct {
+	Frames []InputFrame `json:"frames"`
+	Fields []Field      `json:"fields"`
+}
+
+// Encode converts InputDefinitionInfo into its internal representation.
+func (o *InputDefinitionInfo) Encode() *internal.InputDefinition {
+	var def internal.InputDefinition
+	for _, f := range o.Frames {
+		def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()})
+	}
+	for _, f := range o.Fields {
+		def.Fields = append(def.Fields, f.Encode())
+	}
+
+	return &def
+}

input_definition_test.go

@@ -15,8 +15,11 @@
 package pilosa_test
 
 import (
-	"github.com/pilosa/pilosa"
+	"encoding/json"
 	"testing"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/internal"
 )
 
 func TestInputDefinition_Open(t *testing.T) {
@@ -24,10 +27,11 @@ func TestInputDefinition_Open(t *testing.T) {
 	defer index.Close()
 
 	// Create Input Definition.
-	frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}}
-	action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
-	fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
-	inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields})
+	frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}}
+	action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}}
+	fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}}
+	def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
+	inputDef, err := index.CreateInputDefinition(&def)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -36,3 +40,56 @@ func TestInputDefinition_Open(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+// Verify the InputDefinition Encoding to the internal format
+func TestInputDefinition_Encoding(t *testing.T) {
+	inputBody := []byte(`
+			{
+			"frames":[{
+				"name":"event-time",
+				"options":{
+					"timeQuantum": "YMD",
+					"inverseEnabled": true,
+					"cacheType": "ranked"
+				}
+			}],
+			"fields": [
+				{
+					"name": "id",
+					"primaryKey": true
+				},
+				{
+					"name": "cabType",
+					"actions": [
+						{
+							"frame": "cab-type",
+							"valueDestination": "mapping",
+							"valueMap": {
+								"Green": 1,
+								"Yellow": 2
+							}
+						}
+					]
+				}
+			]
+		}`)
+	var def pilosa.InputDefinitionInfo
+	err := json.Unmarshal(inputBody, &def)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	internalDef := def.Encode()
+
+	if internalDef.Frames[0].Name != "event-time" {
+		t.Fatalf("unexpected frame: %v", internalDef)
+	} else if internalDef.Frames[0].Meta.CacheType != "ranked" {
+		t.Fatalf("unexpected frame meta data: %v", internalDef)
+	} else if len(internalDef.Fields) != 2 {
+		t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields))
+	} else if len(internalDef.Fields[1].Actions) != 1 {
+		t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions)
+	} else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" {
+		t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0])
+	}
+}

server.go

@@ -311,15 +311,27 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 			CacheSize:      obj.Meta.CacheSize,
 			TimeQuantum:    TimeQuantum(obj.Meta.TimeQuantum),
 		}
-		_, err := idx.CreateFrame(obj.Frame, opt)
-		if err != nil {
-			return err
-		}
+		err := s.createFrame(idx, obj.Frame, opt)
+		return err
 	case *internal.DeleteFrameMessage:
 		idx := s.Holder.Index(obj.Index)
 		if err := idx.DeleteFrame(obj.Frame); err != nil {
 			return err
 		}
+	case *internal.CreateInputDefinitionMessage:
+		idx := s.Holder.Index(obj.Index)
+		if idx == nil {
+			return fmt.Errorf("Local Index not found: %s", obj.Index)
+		}
+		idx.CreateInputDefinition(obj.Definition)
+	}
+	return nil
+}
+
+func (s *Server) createFrame(idx *Index, frame string, opt FrameOptions) error {
+	_, err := idx.CreateFrame(frame, opt)
+	if err != nil {
+		return err
 	}
 	return nil
 }