load input-definition when startup servers

handler.go

@@ -1515,9 +1515,6 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	fmt.Println(req.Frames)
-	fmt.Println(req.Fields)
-
 	// Find index.
 	index := h.Holder.Index(indexName)
 	if index == nil {
@@ -1558,14 +1555,14 @@ func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) {
 		}
 		return
 	}
-
-	inputDef := index.inputDefinition(inputDefName)
-	res, err := json.Marshal(inputDef)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+	inputDef, _ := index.inputDefinitions[inputDefName]
+	inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields}
+	if err := json.NewEncoder(w).Encode(getInputDefinitionResponse{
+		InputDefinition: inputInfo,
+	}); err != nil {
+		h.logger().Printf("write status response error: %s", err)
 	}
-	fmt.Println("RESPONSE", string(res))
+
 }
 
 func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) {
@@ -1581,7 +1578,7 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// Delete frame from the index.
+	// Delete input definition from the index.
 	if err := index.DeleteInputDefinition(inputDefName); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -1594,6 +1591,15 @@ type postInputDefinition struct {
 }
 
 type InputFrame struct {
-	Name    string
-	Options FrameOptions
+	Name    string       `json:"name,omitempty"`
+	Options FrameOptions `json:"options,omitempty"`
+}
+
+type getInputDefinitionResponse struct {
+	InputDefinition InputDefinitionInfo `json:"input-definition"`
+}
+
+type InputDefinitionInfo struct {
+	Frames []InputFrame `json:"frames"`
+	Fields []Field      `json:"fields"`
 }

index.go

@@ -31,7 +31,8 @@ import (
 
 // Default index settings.
 const (
-	DefaultColumnLabel = "columnID"
+	DefaultColumnLabel  = "columnID"
+	InputDefinitionRepo = ".input-definitions"
 )
 
 // Index represents a container for frames.
@@ -154,6 +155,44 @@ func (i *Index) Open() error {
 		return err
 	}
 
+	if err := i.openInputDefinition(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// openInputDefinition opens and initializes the input definitions inside the index.
+func (i *Index) openInputDefinition() error {
+	inputDefs, err := os.Open(i.path)
+	if err != nil {
+		return err
+	}
+	defer inputDefs.Close()
+
+	fis, err := inputDefs.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, fi := range fis {
+		if fi.Name() != InputDefinitionRepo {
+			continue
+		}
+		inputPath, err := os.Open(i.InputDefPath())
+		if err != nil {
+			return err
+		}
+		inputFiles, err := inputPath.Readdir(0)
+		for _, file := range inputFiles {
+			input, err := i.newInputDefinition(i.InputDefPath(), file.Name())
+			if err != nil {
+				return err
+			}
+			input.Open()
+			i.inputDefinitions[file.Name()] = input
+		}
+
+	}
 	return nil
 }
 
@@ -171,13 +210,12 @@ func (i *Index) openFrames() error {
 	}
 
 	for _, fi := range fis {
-		if !fi.IsDir() || fi.Name() == ".input-definitions" {
+		if !fi.IsDir() || fi.Name() == InputDefinitionRepo {
 			continue
 		}
 
 		fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
 		if err != nil {
-			fmt.Println(err)
 			return ErrName
 		}
 		if err := fr.Open(); err != nil {
@@ -334,7 +372,7 @@ func (i *Index) FramePath(name string) string { return filepath.Join(i.path, nam
 
 // InputDefPath returns the path to a inputdefinition in the index.
 func (i *Index) InputDefPath() string {
-	return filepath.Join(i.path, ".input-definitions")
+	return filepath.Join(i.path, InputDefinitionRepo)
 }
 
 // Frame returns a frame in the index by name.
@@ -410,6 +448,7 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [
 		return nil, err
 	}
 	i.inputDefinitions[name] = inputDef
+	fmt.Printf("%+v\n", i.inputDefinitions[name])
 	return inputDef, nil
 }
 
@@ -501,8 +540,6 @@ func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error)
 	if err != nil {
 		return nil, err
 	}
-	f.LogOutput = i.LogOutput
-	f.Stats = i.Stats.WithTags(fmt.Sprintf("input-definition:%s", name))
 	f.broadcaster = i.broadcaster
 	return f, nil
 }

input_definition.go

@@ -3,7 +3,6 @@ package pilosa
 import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,8 +13,6 @@ type InputDefinition struct {
 	path        string
 	index       string
 	broadcaster Broadcaster
-	Stats       StatsClient
-	LogOutput   io.Writer
 	frames      []InputFrame
 	fields      []Field
 }
@@ -51,7 +48,6 @@ func (i *InputDefinition) Open() error {
 		if err := i.loadMeta(); err != nil {
 			return err
 		}
-
 		return nil
 	}(); err != nil {
 		return err
@@ -71,12 +67,43 @@ func (i *InputDefinition) loadMeta() error {
 	}
 	// Copy metadata fields.
 	i.name = pb.Name
-	i.frames = pb.Frames
-	i.fields = pb.InputDefinitionFields
+	for _, fr := range pb.Frames {
+		frameMeta := fr.Meta
+		inputFrame := InputFrame{
+			Name: fr.Name,
+			Options: FrameOptions{
+				RowLabel:       frameMeta.RowLabel,
+				InverseEnabled: frameMeta.InverseEnabled,
+				CacheSize:      frameMeta.CacheSize,
+				CacheType:      frameMeta.CacheType,
+				TimeQuantum:    TimeQuantum(frameMeta.TimeQuantum),
+			},
+		}
+		i.frames = append(i.frames, inputFrame)
+	}
+
+	for _, field := range pb.InputDefinitionFields {
+		var actions []Action
+		for _, action := range field.Actions {
+			actions = append(actions, Action{
+				Frame:            action.Frame,
+				ValueDestination: action.ValueDestination,
+				ValueMap:         action.ValueMap,
+				RowID:            action.RowID,
+			})
+		}
+
+		inputField := Field{
+			Name:       field.Name,
+			PrimaryKey: field.PrimaryKey,
+			Actions:    actions,
+		}
+		i.fields = append(i.fields, inputField)
+	}
 	return nil
 }
 
-//saveMeta writes meta data for the frame.
+//saveMeta writes meta data for the input definition file.
 func (i *InputDefinition) saveMeta() error {
 	// Marshal metadata.
 	var frames []*internal.Frame
@@ -129,12 +156,6 @@ func (i *InputDefinition) saveMeta() error {
 	return nil
 }
 
-// FrameOptions represents options to set when initializing a frame.
-type InputDefinitionMeta struct {
-	Frames []Frame `json:"frames,omitempty"`
-	Fields []Field `json:"fields,omitempty"`
-}
-
 type Field struct {
 	Name       string   `json:"name,omitempty"`
 	PrimaryKey bool     `json:"primaryKey,omitempty"`
@@ -147,11 +168,3 @@ type Action struct {
 	ValueMap         map[string]uint64 `json:"valueMap,omitempty"`
 	RowID            uint64            `json:"rowID,omitempty"`
 }
-
-// Encode converts o into its internal representation.
-//func (o *InputDefinitionMeta) Encode() *internal.InputDefinitionMeta {
-//	return &internal.InputDefinitionMeta{
-//		Frames:                o.Frames,
-//		InputDefinitionFields: o.Fields,
-//	}
-//}