From c80bedb4023e19aa985fb49a9ff2ec875143ac72 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 8 Jun 2017 14:23:46 -0500 Subject: [PATCH 01/99] create input-definiton --- handler.go | 61 ++ index.go | 57 +- input_definition.go | 103 ++++ internal/private.pb.go | 1307 ++++++++++++++++++++++++++++++++++++++-- internal/private.proto | 31 + internal/public.pb.go | 2 +- pilosa.go | 11 +- 7 files changed, 1515 insertions(+), 57 deletions(-) create mode 100644 input_definition.go diff --git a/handler.go b/handler.go index 2ced1f66e..c0dbd8ba6 100644 --- a/handler.go +++ b/handler.go @@ -129,6 +129,10 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetDefinition).Methods("GET") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostDefinition).Methods("POST") + router.HandleFunc("/index/{index}input-definition/{input-definition}", handler.handleDeleteDefinition).Methods("DELETE") + return router } @@ -1495,3 +1499,60 @@ func errorString(err error) string { } return err.Error() } + +func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + + // Decode request. + var req postInputDefinition + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the frame + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + fmt.Println(req) + + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + + // Create InputDefinition. + _, err = index.CreateInputDefinition(inputDefName, req.Frames, req.Fields) + if err == ErrInputDefinitionExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + //err = h.Broadcaster.SendSync( + // &internal.CreateInputDefinitionMessage{ + // Index: indexName, + // InputDefinition: inputDefName, + // Meta: req.Options.Encode(), + // }) + //if err != nil { + // h.logger().Printf("problem sending CreateFrame message: %s", err) + //} +} + +func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { + +} + +func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) { + +} + +type postInputDefinition struct { + Frames []Frame `json:"frames,omitempty"` + Fields []Field `json:"field,omitempty"` +} diff --git a/index.go b/index.go index 8a1b34aed..420e460e6 100644 --- a/index.go +++ b/index.go @@ -57,6 +57,9 @@ type Index struct { // Column attribute storage and cache columnAttrStore *AttrStore + // InputDefinition by name + inputDefinitions map[string]*InputDefinition + broadcaster Broadcaster Stats StatsClient @@ -71,9 +74,10 @@ func NewIndex(path, name string) (*Index, error) { } return &Index{ - path: path, - name: name, - frames: make(map[string]*Frame), + path: path, + name: name, + frames: make(map[string]*Frame), + inputDefinitions: make(map[string]*InputDefinition), remoteMaxSlice: 0, remoteMaxInverseSlice: 0, @@ -327,6 +331,11 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } +// InputDefPath returns the path to a inputdefinition in the index. +func (i *Index) InputDefPath(name string) string { + return filepath.Join(i.path, ".input-definitions", name) +} + // Frame returns a frame in the index by name. func (i *Index) Frame(name string) *Frame { i.mu.Lock() @@ -362,6 +371,37 @@ func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { return i.createFrame(name, opt) } +// CreateInputDefinition creates a new input definition. +func (i *Index) CreateInputDefinition(name string, frames []Frame, field []Field) (*InputDefinition, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Ensure frame doesn't already exist. + if i.inputDefinitions[name] != nil { + return nil, ErrInputDefinitionExists + } + return i.createInputDefinition(name, frames, field) +} + +func (i *Index) createInputDefinition(name string, frames []Frame, field []Field) (*InputDefinition, error) { + if name == "" { + return nil, errors.New("input-definition name required") + } + + // Initialize frame. + inputDef, err := i.newInputDefinition(i.InputDefPath(name), name) + fmt.Println(inputDef) + if err != nil { + return nil, err + } + // Open frame. + if err := inputDef.Open(); err != nil { + return nil, err + } + i.inputDefinitions[name] = inputDef + return inputDef, nil +} + // CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { i.mu.Lock() @@ -445,6 +485,17 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { return f, nil } +func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) { + f, err := NewInputDefinition(path, i.name, name) + if err != nil { + return nil, err + } + f.LogOutput = i.LogOutput + f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.broadcaster = i.broadcaster + return f, nil +} + // DeleteFrame removes a frame from the index. func (i *Index) DeleteFrame(name string) error { i.mu.Lock() diff --git a/input_definition.go b/input_definition.go new file mode 100644 index 000000000..0b168a25d --- /dev/null +++ b/input_definition.go @@ -0,0 +1,103 @@ +package pilosa + +import ( + "fmt" + "io" + "os" +) + +type InputDefinition struct { + name string + path string + index string + broadcaster Broadcaster + Stats StatsClient + LogOutput io.Writer +} + +func NewInputDefinition(path, index, name string) (*InputDefinition, error) { + err := ValidateName(name) + if err != nil { + return nil, err + } + + return &InputDefinition{ + path: path, + index: index, + name: name, + }, nil +} + +// Name returns the name of input definition was initialized with. +func (i *InputDefinition) Name() string { return i.name } + +// Index returns the index name of the input definition was initialized with. +func (i *InputDefinition) Index() string { return i.index } + +// Path returns the path of the input definition was initialized with. +func (i *InputDefinition) Path() string { return i.path } + +func (i *InputDefinition) Open() error { + fmt.Println(i.path) + if err := func() error { + // Ensure the frame's path exists. + if err := os.MkdirAll(i.path, 0777); err != nil { + return err + } + + //if err := i.loadMeta(); err != nil { + // return err + //} + + return nil + }(); err != nil { + return err + } + + return nil +} + +// saveMeta writes meta data for the frame. +//func (f *InputDefinition) saveMeta() error { +// // Marshal metadata. +// buf, err := proto.Marshal(&internal.InputDefinitionMeta{ +// Frames: f. +// }) +// if err != nil { +// return err +// } +// +// // Write to meta file. +// if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { +// return err +// } +// +// return nil +//} + +// FrameOptions represents options to set when initializing a frame. +type InputDefinitionMeta struct { + Frames []Frame `json:"frames,omitempty"` + Fields []Field `json:"fields,omitempty"` +} + +type Field struct { + Name string `json:"name,omitempty"` + PrimaryKey bool `json:"primaryKey,omitempty"` + Actions []Action `json:"action,omitempty"` +} + +type Action struct { + Frame string `json:"frame,omitempty"` + ValueDestination string `json:"valueDestination,omitempty"` + ValueMap map[string]uint64 `json:"valueMap,omitempty"` + RowID uint64 `json:"rowID,omitempty"` +} + +// Encode converts o into its internal representation. +//func (o *InputDefinitionMeta) Encode() *internal.InputDefinitionMeta { +// return &internal.InputDefinitionMeta{ +// Frames: o.Frames, +// InputDefinitionFields: o.Fields, +// } +//} diff --git a/internal/private.pb.go b/internal/private.pb.go index eba920486..1c546e74e 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -23,6 +23,11 @@ DeleteFrameMessage Frame Index + InputDefinition + InputDefinitionMeta + InputDefinitionField + Action + CreateInputDefinitionMessage NodeStatus ClusterStatus */ @@ -208,11 +213,12 @@ func (m *Frame) GetMeta() *FrameMeta { } type Index struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` - Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Slices []uint64 `protobuf:"varint,5,rep,packed,name=Slices" json:"Slices,omitempty"` + InputDefinitions []*InputDefinition `protobuf:"bytes,6,rep,name=InputDefinitions" json:"InputDefinitions,omitempty"` } func (m *Index) Reset() { *m = Index{} } @@ -234,6 +240,111 @@ func (m *Index) GetFrames() []*Frame { return nil } +func (m *Index) GetInputDefinitions() []*InputDefinition { + if m != nil { + return m.InputDefinitions + } + return nil +} + +type InputDefinition struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *InputDefinitionMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *InputDefinition) Reset() { *m = InputDefinition{} } +func (m *InputDefinition) String() string { return proto.CompactTextString(m) } +func (*InputDefinition) ProtoMessage() {} +func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } + +func (m *InputDefinition) GetMeta() *InputDefinitionMeta { + if m != nil { + return m.Meta + } + return nil +} + +type InputDefinitionMeta struct { + Frames []*Frame `protobuf:"bytes,1,rep,name=Frames" json:"Frames,omitempty"` + InputDefinitionFields []*InputDefinitionField `protobuf:"bytes,2,rep,name=InputDefinitionFields" json:"InputDefinitionFields,omitempty"` +} + +func (m *InputDefinitionMeta) Reset() { *m = InputDefinitionMeta{} } +func (m *InputDefinitionMeta) String() string { return proto.CompactTextString(m) } +func (*InputDefinitionMeta) ProtoMessage() {} +func (*InputDefinitionMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } + +func (m *InputDefinitionMeta) GetFrames() []*Frame { + if m != nil { + return m.Frames + } + return nil +} + +func (m *InputDefinitionMeta) GetInputDefinitionFields() []*InputDefinitionField { + if m != nil { + return m.InputDefinitionFields + } + return nil +} + +type InputDefinitionField struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` + Meta *Action `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } +func (m *InputDefinitionField) String() string { return proto.CompactTextString(m) } +func (*InputDefinitionField) ProtoMessage() {} +func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } + +func (m *InputDefinitionField) GetMeta() *Action { + if m != nil { + return m.Meta + } + return nil +} + +type Action struct { + Frame string `protobuf:"bytes,1,opt,name=Frame,proto3" json:"Frame,omitempty"` + ValueDestination string `protobuf:"bytes,2,opt,name=ValueDestination,proto3" json:"ValueDestination,omitempty"` + ValueMap map[string]uint64 `protobuf:"bytes,3,rep,name=ValueMap" json:"ValueMap,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + RowID uint64 `protobuf:"varint,4,opt,name=RowID,proto3" json:"RowID,omitempty"` +} + +func (m *Action) Reset() { *m = Action{} } +func (m *Action) String() string { return proto.CompactTextString(m) } +func (*Action) ProtoMessage() {} +func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } + +func (m *Action) GetValueMap() map[string]uint64 { + if m != nil { + return m.ValueMap + } + return nil +} + +type CreateInputDefinitionMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + InputDefinition string `protobuf:"bytes,2,opt,name=InputDefinition,proto3" json:"InputDefinition,omitempty"` + Meta *InputDefinitionMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinitionMessage{} } +func (m *CreateInputDefinitionMessage) String() string { return proto.CompactTextString(m) } +func (*CreateInputDefinitionMessage) ProtoMessage() {} +func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { + return fileDescriptorPrivate, []int{18} +} + +func (m *CreateInputDefinitionMessage) GetMeta() *InputDefinitionMeta { + if m != nil { + return m.Meta + } + return nil +} + type NodeStatus struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -243,7 +354,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeStatus) GetIndexes() []*Index { if m != nil { @@ -259,7 +370,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *ClusterStatus) GetNodes() []*NodeStatus { if m != nil { @@ -283,6 +394,11 @@ func init() { proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Index)(nil), "internal.Index") + proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") + proto.RegisterType((*InputDefinitionMeta)(nil), "internal.InputDefinitionMeta") + proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") + proto.RegisterType((*Action)(nil), "internal.Action") + proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") } @@ -824,6 +940,229 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j11)) i += copy(dAtA[i:], dAtA12[:j11]) } + if len(m.InputDefinitions) > 0 { + for _, msg := range m.InputDefinitions { + dAtA[i] = 0x32 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *InputDefinition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InputDefinition) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n13, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 + } + return i, nil +} + +func (m *InputDefinitionMeta) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InputDefinitionMeta) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Frames) > 0 { + for _, msg := range m.Frames { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.InputDefinitionFields) > 0 { + for _, msg := range m.InputDefinitionFields { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *InputDefinitionField) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.PrimaryKey { + dAtA[i] = 0x10 + i++ + if m.PrimaryKey { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Meta != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n14, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n14 + } + return i, nil +} + +func (m *Action) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Action) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Frame) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.ValueDestination) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ValueDestination))) + i += copy(dAtA[i:], m.ValueDestination) + } + if len(m.ValueMap) > 0 { + for k, _ := range m.ValueMap { + dAtA[i] = 0x1a + i++ + v := m.ValueMap[k] + mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(v)) + } + } + if m.RowID != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.RowID)) + } + return i, nil +} + +func (m *CreateInputDefinitionMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.InputDefinition) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.InputDefinition))) + i += copy(dAtA[i:], m.InputDefinition) + } + if m.Meta != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n15, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n15 + } return i, nil } @@ -1158,6 +1497,104 @@ func (m *Index) Size() (n int) { } n += 1 + sovPrivate(uint64(l)) + l } + if len(m.InputDefinitions) > 0 { + for _, e := range m.InputDefinitions { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *InputDefinition) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *InputDefinitionMeta) Size() (n int) { + var l int + _ = l + if len(m.Frames) > 0 { + for _, e := range m.Frames { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + if len(m.InputDefinitionFields) > 0 { + for _, e := range m.InputDefinitionFields { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *InputDefinitionField) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.PrimaryKey { + n += 2 + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *Action) Size() (n int) { + var l int + _ = l + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.ValueDestination) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.ValueMap) > 0 { + for k, v := range m.ValueMap { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.RowID != 0 { + n += 1 + sovPrivate(uint64(m.RowID)) + } + return n +} + +func (m *CreateInputDefinitionMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.InputDefinition) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -3059,6 +3496,767 @@ func (m *Index) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InputDefinitions = append(m.InputDefinitions, &InputDefinition{}) + if err := m.InputDefinitions[len(m.InputDefinitions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InputDefinition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InputDefinition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InputDefinition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &InputDefinitionMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InputDefinitionMeta) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InputDefinitionMeta: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InputDefinitionMeta: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frames = append(m.Frames, &Frame{}) + if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionFields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InputDefinitionFields = append(m.InputDefinitionFields, &InputDefinitionField{}) + if err := m.InputDefinitionFields[len(m.InputDefinitionFields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InputDefinitionField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InputDefinitionField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PrimaryKey", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PrimaryKey = bool(v != 0) + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &Action{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Action) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Action: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Action: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ValueDestination", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ValueDestination = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ValueMap", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.ValueMap == nil { + m.ValueMap = make(map[string]uint64) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ValueMap[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.ValueMap[mapkey] = mapvalue + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) + } + m.RowID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RowID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateInputDefinitionMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateInputDefinitionMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinition", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InputDefinition = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &InputDefinitionMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3408,45 +4606,58 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 640 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x4e, 0x14, 0x4d, - 0x10, 0xfe, 0x67, 0x77, 0x96, 0x7f, 0xa7, 0x08, 0x08, 0x2d, 0x31, 0x23, 0x21, 0x9b, 0x4d, 0x1f, - 0x04, 0x39, 0x70, 0xc0, 0x8b, 0x51, 0x0f, 0x86, 0x5d, 0x0c, 0x93, 0x08, 0xc6, 0x5e, 0xe2, 0xd1, - 0xa4, 0x81, 0x8a, 0x4e, 0x98, 0x9d, 0x59, 0xa7, 0x7b, 0x80, 0xf5, 0xe0, 0x73, 0x98, 0x78, 0xf2, - 0x01, 0x7c, 0x0f, 0x8f, 0x3e, 0x82, 0xc1, 0x17, 0x31, 0x5d, 0xdd, 0x33, 0xb3, 0x0e, 0x22, 0xd1, - 0x5b, 0xd7, 0x57, 0xd5, 0xf5, 0x7d, 0xfd, 0x4d, 0xd5, 0xc0, 0xc2, 0x24, 0x8f, 0xcf, 0xa4, 0xc6, - 0xad, 0x49, 0x9e, 0xe9, 0x8c, 0x75, 0xe3, 0x54, 0x63, 0x9e, 0xca, 0x84, 0xbf, 0x80, 0x20, 0x4a, - 0x4f, 0xf0, 0x62, 0x1f, 0xb5, 0x64, 0x7d, 0x98, 0x1f, 0x64, 0x49, 0x31, 0x4e, 0x9f, 0xcb, 0x23, - 0x4c, 0x42, 0xaf, 0xef, 0x6d, 0x04, 0x62, 0x16, 0x32, 0x15, 0x87, 0xf1, 0x18, 0x5f, 0x16, 0x32, - 0xd5, 0xc5, 0x38, 0x6c, 0xd9, 0x8a, 0x19, 0x88, 0x7f, 0xf1, 0x20, 0x78, 0x96, 0xcb, 0x31, 0x52, - 0xc7, 0x55, 0xe8, 0x8a, 0xec, 0x7c, 0xb6, 0x5d, 0x15, 0xb3, 0x7b, 0xb0, 0x18, 0xa5, 0x67, 0x98, - 0x2b, 0xdc, 0x4d, 0xe5, 0x51, 0x82, 0x27, 0xd4, 0xae, 0x2b, 0x1a, 0x28, 0x5b, 0x83, 0x60, 0x20, - 0x8f, 0xdf, 0xe2, 0xe1, 0x74, 0x82, 0x61, 0x9b, 0x9a, 0xd4, 0x40, 0x95, 0x1d, 0xc5, 0xef, 0x31, - 0xf4, 0xfb, 0xde, 0xc6, 0x82, 0xa8, 0x81, 0xa6, 0xde, 0xce, 0x55, 0xbd, 0x1c, 0x16, 0xa3, 0xf1, - 0x24, 0xcb, 0xb5, 0x40, 0x35, 0xc9, 0x52, 0x85, 0x6c, 0x09, 0xda, 0xbb, 0x79, 0xee, 0xe4, 0x9a, - 0x23, 0xff, 0x00, 0x4b, 0x3b, 0x49, 0x76, 0x7c, 0x3a, 0x94, 0x5a, 0x0a, 0x7c, 0x57, 0xa0, 0xd2, - 0x6c, 0x05, 0x3a, 0x64, 0x9c, 0xab, 0xb3, 0x81, 0x41, 0xe9, 0xf1, 0xce, 0x19, 0x1b, 0x18, 0x94, - 0xee, 0x93, 0x7a, 0x5f, 0xd8, 0xc0, 0xa0, 0xa3, 0x24, 0x3e, 0xb6, 0xaa, 0x7d, 0x61, 0x03, 0xc6, - 0xc0, 0x7f, 0x15, 0xe3, 0xb9, 0x93, 0x4a, 0x67, 0x1e, 0xc1, 0xf2, 0x0c, 0xbf, 0x93, 0x79, 0x07, - 0xe6, 0x44, 0x76, 0x1e, 0x0d, 0x55, 0xe8, 0xf5, 0xdb, 0x1b, 0xbe, 0x70, 0x11, 0x19, 0x42, 0x5f, - 0xcc, 0xa4, 0x5a, 0x94, 0xaa, 0x01, 0x7e, 0x17, 0x3a, 0xe4, 0x8e, 0x79, 0x65, 0x7d, 0xd7, 0x1c, - 0xf9, 0x27, 0x0f, 0x96, 0xf7, 0xe5, 0x05, 0xc9, 0x50, 0x15, 0xcd, 0x1e, 0x04, 0x15, 0x48, 0xd5, - 0xf3, 0xdb, 0x9b, 0x5b, 0xe5, 0xf8, 0x6c, 0x5d, 0xa9, 0xaf, 0x91, 0xdd, 0x54, 0xe7, 0x53, 0x51, - 0x5f, 0x5e, 0x7d, 0x02, 0x8b, 0xbf, 0x26, 0x8d, 0x86, 0x53, 0x9c, 0x96, 0x4e, 0x9f, 0xe2, 0xd4, - 0x78, 0x72, 0x26, 0x93, 0xc2, 0xfa, 0xe7, 0x0b, 0x1b, 0x3c, 0x6a, 0x3d, 0xf4, 0xf8, 0x6b, 0x60, - 0x83, 0x1c, 0xa5, 0x46, 0x6a, 0xb0, 0x8f, 0x4a, 0xc9, 0x37, 0x78, 0xfd, 0x57, 0xb0, 0xce, 0xb6, - 0x66, 0x9d, 0x5d, 0x83, 0x20, 0x52, 0x6e, 0xb6, 0xe8, 0x4b, 0x74, 0x45, 0x0d, 0xf0, 0x4d, 0x60, - 0x43, 0x4c, 0x50, 0xa3, 0x5b, 0x87, 0x3f, 0xf4, 0xe7, 0xa3, 0x52, 0xcb, 0xcd, 0xb5, 0x6c, 0x1d, - 0x7c, 0xb3, 0x09, 0x24, 0x65, 0x7e, 0xfb, 0x76, 0x6d, 0x5d, 0xb5, 0x76, 0x82, 0x0a, 0x78, 0x5c, - 0x36, 0x75, 0xdb, 0x73, 0xc3, 0x03, 0x7f, 0x33, 0x66, 0x25, 0x55, 0xbb, 0x49, 0x55, 0xed, 0xa3, - 0xa3, 0x7a, 0x5a, 0xbe, 0xf5, 0x5f, 0xa9, 0xf8, 0xd0, 0xa1, 0x66, 0x5c, 0x0f, 0x4c, 0xd6, 0xde, - 0xa1, 0xf3, 0xf5, 0x4f, 0x6e, 0xea, 0xf8, 0xec, 0x39, 0xca, 0xbf, 0x6b, 0xd3, 0x70, 0xce, 0xfc, - 0x64, 0xca, 0xc1, 0x72, 0x1b, 0x56, 0xc5, 0x6c, 0x1d, 0xe6, 0x88, 0x55, 0x85, 0x3e, 0xcd, 0xee, - 0xad, 0x86, 0x1a, 0xe1, 0xd2, 0x66, 0x9d, 0xdc, 0x90, 0x77, 0xec, 0x3a, 0xd9, 0x88, 0x4b, 0x80, - 0x83, 0xec, 0x04, 0x47, 0x5a, 0xea, 0x42, 0x19, 0x9d, 0x7b, 0x99, 0xd2, 0xa5, 0x4e, 0x73, 0xa6, - 0x69, 0xd3, 0x52, 0x57, 0x0e, 0x51, 0xc0, 0xee, 0xc3, 0xff, 0xa4, 0x13, 0x55, 0xd8, 0x6e, 0x32, - 0x53, 0x42, 0x94, 0x79, 0xfe, 0x18, 0x16, 0x06, 0x49, 0xa1, 0x34, 0xe6, 0x8e, 0x65, 0x13, 0x3a, - 0x86, 0xb3, 0xdc, 0xb7, 0x95, 0xfa, 0x66, 0x2d, 0x45, 0xd8, 0x92, 0x9d, 0xa5, 0xaf, 0x97, 0x3d, - 0xef, 0xdb, 0x65, 0xcf, 0xfb, 0x7e, 0xd9, 0xf3, 0x3e, 0xfe, 0xe8, 0xfd, 0x77, 0x34, 0x47, 0xff, - 0xf8, 0x07, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x1c, 0x79, 0x08, 0xf4, 0x05, 0x00, 0x00, + // 848 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0xa6, 0xe3, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0xed, 0x0d, 0x68, 0x36, 0x0a, 0x96, 0x35, 0x42, + 0xac, 0xf1, 0xc1, 0xd2, 0x86, 0x0b, 0xda, 0xe5, 0x00, 0x6b, 0x7b, 0xb5, 0x16, 0x78, 0x81, 0x76, + 0xb4, 0xe2, 0x84, 0xd4, 0xb1, 0x0b, 0x18, 0x65, 0x3c, 0x33, 0x4c, 0xf7, 0x24, 0x6b, 0x0e, 0xbc, + 0x03, 0x12, 0x07, 0x24, 0x9e, 0x81, 0xf7, 0xe0, 0xc8, 0x95, 0x1b, 0x0a, 0x17, 0x1e, 0x03, 0xf5, + 0xcf, 0xfc, 0x78, 0x6c, 0x67, 0x09, 0xb7, 0xa9, 0xaf, 0xfe, 0xbe, 0xaa, 0xae, 0x2a, 0x1b, 0x0e, + 0xe2, 0xc4, 0xbf, 0xe2, 0x12, 0x07, 0x71, 0x12, 0xc9, 0x88, 0x36, 0xfd, 0x50, 0x62, 0x12, 0xf2, + 0xc0, 0xfb, 0x1c, 0x5a, 0x93, 0x70, 0x81, 0xaf, 0xa6, 0x28, 0x39, 0xed, 0xc2, 0xfe, 0x30, 0x0a, + 0xd2, 0x65, 0xf8, 0x19, 0xbf, 0xc0, 0xc0, 0x25, 0x5d, 0xd2, 0x6b, 0xb1, 0x32, 0xa4, 0x2c, 0xce, + 0xfd, 0x25, 0x7e, 0x99, 0xf2, 0x50, 0xa6, 0x4b, 0x77, 0xcf, 0x58, 0x94, 0x20, 0xef, 0x37, 0x02, + 0xad, 0x67, 0x09, 0x5f, 0xa2, 0x8e, 0x78, 0x02, 0x4d, 0x16, 0x5d, 0x97, 0xc3, 0xe5, 0x32, 0x7d, + 0x0f, 0x0e, 0x27, 0xe1, 0x15, 0x26, 0x02, 0xc7, 0x21, 0xbf, 0x08, 0x70, 0xa1, 0xc3, 0x35, 0x59, + 0x05, 0xa5, 0xa7, 0xd0, 0x1a, 0xf2, 0xf9, 0x77, 0x78, 0xbe, 0x8a, 0xd1, 0xad, 0xe9, 0x20, 0x05, + 0x90, 0x6b, 0x67, 0xfe, 0x0f, 0xe8, 0x3a, 0x5d, 0xd2, 0x3b, 0x60, 0x05, 0x50, 0xe5, 0x5b, 0xdf, + 0xe4, 0xeb, 0xc1, 0xe1, 0x64, 0x19, 0x47, 0x89, 0x64, 0x28, 0xe2, 0x28, 0x14, 0x48, 0xdb, 0x50, + 0x1b, 0x27, 0x89, 0xa5, 0xab, 0x3e, 0xbd, 0x1f, 0xa1, 0xfd, 0x34, 0x88, 0xe6, 0x97, 0x23, 0x2e, + 0x39, 0xc3, 0xef, 0x53, 0x14, 0x92, 0x1e, 0x43, 0x5d, 0x37, 0xce, 0xda, 0x19, 0x41, 0xa1, 0xba, + 0x78, 0xdb, 0x19, 0x23, 0x28, 0x54, 0xfb, 0x6b, 0xf6, 0x0e, 0x33, 0x82, 0x42, 0x67, 0x81, 0x3f, + 0x37, 0xac, 0x1d, 0x66, 0x04, 0x4a, 0xc1, 0x79, 0xe9, 0xe3, 0xb5, 0xa5, 0xaa, 0xbf, 0xbd, 0x09, + 0xdc, 0x2b, 0xe5, 0xb7, 0x34, 0xdf, 0x86, 0x06, 0x8b, 0xae, 0x27, 0x23, 0xe1, 0x92, 0x6e, 0xad, + 0xe7, 0x30, 0x2b, 0xe9, 0x86, 0xe8, 0x17, 0x53, 0xaa, 0x3d, 0xad, 0x2a, 0x00, 0xef, 0x01, 0xd4, + 0x75, 0x77, 0x54, 0x95, 0x85, 0xaf, 0xfa, 0xf4, 0x7e, 0x25, 0x70, 0x6f, 0xca, 0x5f, 0x69, 0x1a, + 0x22, 0x4f, 0xf3, 0x1c, 0x5a, 0x39, 0xa8, 0xad, 0xf7, 0xcf, 0xfa, 0x83, 0x6c, 0x7c, 0x06, 0x1b, + 0xf6, 0x05, 0x32, 0x0e, 0x65, 0xb2, 0x62, 0x85, 0xf3, 0xc9, 0x47, 0x70, 0xb8, 0xae, 0x54, 0x1c, + 0x2e, 0x71, 0x95, 0x75, 0xfa, 0x12, 0x57, 0xaa, 0x27, 0x57, 0x3c, 0x48, 0x4d, 0xff, 0x1c, 0x66, + 0x84, 0xc7, 0x7b, 0x1f, 0x12, 0xef, 0x6b, 0xa0, 0xc3, 0x04, 0xb9, 0x44, 0x1d, 0x60, 0x8a, 0x42, + 0xf0, 0x6f, 0x71, 0xf7, 0x2b, 0x98, 0xce, 0xee, 0x95, 0x3b, 0x7b, 0x0a, 0xad, 0x89, 0xb0, 0xb3, + 0xa5, 0x5f, 0xa2, 0xc9, 0x0a, 0xc0, 0xeb, 0x03, 0x1d, 0x61, 0x80, 0x12, 0xed, 0x3a, 0xdc, 0x12, + 0xdf, 0x9b, 0x65, 0x5c, 0x5e, 0x6f, 0x4b, 0x1f, 0x82, 0xa3, 0x36, 0x41, 0x53, 0xd9, 0x3f, 0xbb, + 0x5f, 0xb4, 0x2e, 0x5f, 0x3b, 0xa6, 0x0d, 0x3c, 0x3f, 0x0b, 0x6a, 0xb7, 0xe7, 0x35, 0x05, 0x6e, + 0x19, 0xb3, 0x2c, 0x55, 0xad, 0x9a, 0x2a, 0xdf, 0x47, 0x9b, 0xea, 0xe3, 0xac, 0xd6, 0xff, 0x9b, + 0xca, 0x1b, 0x59, 0x54, 0x8d, 0xeb, 0x0b, 0xa5, 0x35, 0x3e, 0xfa, 0x7b, 0x77, 0xc9, 0x55, 0x1e, + 0xff, 0x10, 0x9b, 0xf2, 0x6e, 0x61, 0x2a, 0x9d, 0x53, 0x47, 0x26, 0x1b, 0x2c, 0xbb, 0x61, 0xb9, + 0x4c, 0x1f, 0x42, 0x43, 0x67, 0x15, 0xae, 0xa3, 0x67, 0xf7, 0xa8, 0xc2, 0x86, 0x59, 0xb5, 0x5a, + 0x27, 0x3b, 0xe4, 0x75, 0xb3, 0x4e, 0x46, 0xa2, 0x63, 0x68, 0x4f, 0xc2, 0x38, 0x95, 0x23, 0xfc, + 0xc6, 0x0f, 0x7d, 0xe9, 0x47, 0xa1, 0x70, 0x1b, 0x3a, 0xd4, 0x83, 0x32, 0xa3, 0x35, 0x0b, 0xb6, + 0xe1, 0xe2, 0x7d, 0x05, 0x47, 0x15, 0x6c, 0x6b, 0xcd, 0x8f, 0xd6, 0x6a, 0x7e, 0x67, 0x67, 0x86, + 0x52, 0x13, 0x7f, 0x26, 0x70, 0x7f, 0x8b, 0xb6, 0x54, 0x39, 0xb9, 0xbd, 0xf2, 0x73, 0x78, 0xab, + 0xe2, 0xff, 0xcc, 0xc7, 0x60, 0x61, 0x8e, 0xc7, 0xfe, 0x59, 0x67, 0x27, 0x09, 0x6d, 0xc6, 0xb6, + 0x3b, 0x7b, 0x31, 0x1c, 0x6f, 0x53, 0x6c, 0xad, 0xba, 0x03, 0xf0, 0x45, 0xe2, 0x2f, 0x79, 0xb2, + 0xfa, 0x14, 0x57, 0xf6, 0x57, 0xa0, 0x84, 0xd0, 0x77, 0xd7, 0x06, 0xbb, 0x5d, 0x10, 0xfa, 0x64, + 0xae, 0xdb, 0x6d, 0x1a, 0xf1, 0x27, 0x81, 0x86, 0x01, 0x8a, 0xa1, 0x25, 0xe5, 0xfd, 0xe8, 0x43, + 0xfb, 0xa5, 0xba, 0x27, 0x23, 0x14, 0xd2, 0x0f, 0xb9, 0xb2, 0xb4, 0x53, 0xbd, 0x81, 0xd3, 0xc7, + 0xd0, 0xd4, 0xd8, 0x94, 0xc7, 0x6e, 0xad, 0xda, 0x07, 0x93, 0x65, 0x90, 0x19, 0x98, 0x4b, 0x97, + 0xdb, 0xab, 0xec, 0xfa, 0x16, 0x67, 0x87, 0x5d, 0x0b, 0x27, 0x4f, 0xe0, 0x60, 0xcd, 0xe1, 0x4e, + 0xd7, 0xef, 0x27, 0x02, 0xa7, 0xd9, 0xc9, 0xa9, 0x3c, 0xf5, 0x6d, 0xcb, 0xdb, 0xdb, 0x98, 0x3a, + 0x5b, 0xf0, 0xc6, 0x30, 0x3e, 0x5a, 0x6b, 0xf1, 0x7f, 0x1a, 0x3c, 0x0e, 0xf0, 0x22, 0x5a, 0xe0, + 0x4c, 0x72, 0x99, 0x0a, 0xf5, 0xae, 0xcf, 0x23, 0x21, 0xb3, 0x77, 0x55, 0xdf, 0xfa, 0x0e, 0x4b, + 0x2e, 0xf3, 0xdb, 0xa1, 0x05, 0xfa, 0x3e, 0xbc, 0xa9, 0xd9, 0xa1, 0xb0, 0x9d, 0x3d, 0xaa, 0xac, + 0x36, 0xcb, 0xf4, 0xde, 0x13, 0x38, 0x18, 0x06, 0xa9, 0x90, 0x98, 0xd8, 0x2c, 0x7d, 0xa8, 0xab, + 0x9c, 0xd9, 0x4c, 0x1f, 0x17, 0x9e, 0x05, 0x15, 0x66, 0x4c, 0x9e, 0xb6, 0x7f, 0xbf, 0xe9, 0x90, + 0x3f, 0x6e, 0x3a, 0xe4, 0xaf, 0x9b, 0x0e, 0xf9, 0xe5, 0xef, 0xce, 0x1b, 0x17, 0x0d, 0xfd, 0xef, + 0xe7, 0x83, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x8e, 0x46, 0x6d, 0x0e, 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index c7c792f4a..2cc51595b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -77,6 +77,37 @@ message Index { uint64 MaxSlice = 3; repeated Frame Frames = 4; repeated uint64 Slices = 5; + repeated InputDefinition InputDefinitions = 6; +} + +message InputDefinition { + string Name = 1; + InputDefinitionMeta Meta = 2; +} + +message InputDefinitionMeta { + repeated Frame Frames = 1; + repeated InputDefinitionField InputDefinitionFields = 2; +} + +message InputDefinitionField { + string Name = 1; + bool PrimaryKey = 2; + Action Meta = 3; +} + +message Action { + string Frame = 1; + string ValueDestination = 2; + map ValueMap = 3; + uint64 RowID = 4; + +} + +message CreateInputDefinitionMessage { + string Index = 1; + string InputDefinition = 2; + InputDefinitionMeta Meta = 3; } message NodeStatus { diff --git a/internal/public.pb.go b/internal/public.pb.go index 9eb005c17..75d79133f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -2576,7 +2576,7 @@ func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ // 576 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4, 0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b, diff --git a/pilosa.go b/pilosa.go index 13ade19ea..504df3b65 100644 --- a/pilosa.go +++ b/pilosa.go @@ -30,11 +30,12 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrFrameNotFound = errors.New("frame not found") + ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From 8ca7c3b229445abe7f6c5d318252c9d15dff4616 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 8 Jun 2017 15:48:00 -0500 Subject: [PATCH 02/99] create input-defintion file --- index.go | 16 ++++++++++------ input_definition.go | 38 ++++++++++++++++++++------------------ internal/private.proto | 8 ++------ 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/index.go b/index.go index 420e460e6..a3776e48e 100644 --- a/index.go +++ b/index.go @@ -332,8 +332,8 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } // InputDefPath returns the path to a inputdefinition in the index. -func (i *Index) InputDefPath(name string) string { - return filepath.Join(i.path, ".input-definitions", name) +func (i *Index) InputDefPath() string { + return filepath.Join(i.path, ".input-definitions") } // Frame returns a frame in the index by name. @@ -376,7 +376,7 @@ func (i *Index) CreateInputDefinition(name string, frames []Frame, field []Field i.mu.Lock() defer i.mu.Unlock() - // Ensure frame doesn't already exist. + // Ensure input definition doesn't already exist. if i.inputDefinitions[name] != nil { return nil, ErrInputDefinitionExists } @@ -388,16 +388,20 @@ func (i *Index) createInputDefinition(name string, frames []Frame, field []Field return nil, errors.New("input-definition name required") } - // Initialize frame. - inputDef, err := i.newInputDefinition(i.InputDefPath(name), name) + // Initialize input definition. + inputDef, err := i.newInputDefinition(i.InputDefPath(), name) fmt.Println(inputDef) if err != nil { return nil, err } - // Open frame. + // Open input definition. if err := inputDef.Open(); err != nil { return nil, err } + + if err = inputDef.saveMeta(); err != nil { + return nil, err + } i.inputDefinitions[name] = inputDef return inputDef, nil } diff --git a/input_definition.go b/input_definition.go index 0b168a25d..789ade7cc 100644 --- a/input_definition.go +++ b/input_definition.go @@ -3,7 +3,9 @@ package pilosa import ( "fmt" "io" + "io/ioutil" "os" + "path/filepath" ) type InputDefinition struct { @@ -40,7 +42,7 @@ func (i *InputDefinition) Path() string { return i.path } func (i *InputDefinition) Open() error { fmt.Println(i.path) if err := func() error { - // Ensure the frame's path exists. + // Ensure the input definition's path exists. if err := os.MkdirAll(i.path, 0777); err != nil { return err } @@ -57,23 +59,23 @@ func (i *InputDefinition) Open() error { return nil } -// saveMeta writes meta data for the frame. -//func (f *InputDefinition) saveMeta() error { -// // Marshal metadata. -// buf, err := proto.Marshal(&internal.InputDefinitionMeta{ -// Frames: f. -// }) -// if err != nil { -// return err -// } -// -// // Write to meta file. -// if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { -// return err -// } -// -// return nil -//} +//saveMeta writes meta data for the frame. +func (f *InputDefinition) saveMeta() error { + // Marshal metadata. + //buf, err := proto.Marshal(&internal.InputDefinitionMeta{ + // Frames: f. + //}) + //if err != nil { + // return err + //} + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, f.name), []byte("123"), 0666); err != nil { + return err + } + + return nil +} // FrameOptions represents options to set when initializing a frame. type InputDefinitionMeta struct { diff --git a/internal/private.proto b/internal/private.proto index 2cc51595b..660633f2c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -82,12 +82,8 @@ message Index { message InputDefinition { string Name = 1; - InputDefinitionMeta Meta = 2; -} - -message InputDefinitionMeta { - repeated Frame Frames = 1; - repeated InputDefinitionField InputDefinitionFields = 2; + repeated Frame Frames = 2; + repeated InputDefinitionField InputDefinitionFields = 3; } message InputDefinitionField { From 7c28e2c3f3c7ac778a3cbc8f8154c0c3d252bee6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 9 Jun 2017 10:27:11 -0500 Subject: [PATCH 03/99] InputFrame struct --- handler.go | 10 +- index.go | 4 +- input_definition.go | 8 +- input_definition_test.go | 1 + internal/private.pb.go | 333 ++++++++------------------------------- internal/private.proto | 1 - 6 files changed, 82 insertions(+), 275 deletions(-) create mode 100644 input_definition_test.go diff --git a/handler.go b/handler.go index c0dbd8ba6..00ced11ce 100644 --- a/handler.go +++ b/handler.go @@ -1514,7 +1514,6 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - fmt.Println(req) // Find index. index := h.Holder.Index(indexName) @@ -1553,6 +1552,11 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) } type postInputDefinition struct { - Frames []Frame `json:"frames,omitempty"` - Fields []Field `json:"field,omitempty"` + Frames []InputFrame `json:"frames,omitempty"` + Fields []Field `json:"fields,omitempty"` } + +type InputFrame struct { + Name string + Options FrameOptions +} \ No newline at end of file diff --git a/index.go b/index.go index a3776e48e..9a4f88149 100644 --- a/index.go +++ b/index.go @@ -372,7 +372,7 @@ func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { } // CreateInputDefinition creates a new input definition. -func (i *Index) CreateInputDefinition(name string, frames []Frame, field []Field) (*InputDefinition, error) { +func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { i.mu.Lock() defer i.mu.Unlock() @@ -383,7 +383,7 @@ func (i *Index) CreateInputDefinition(name string, frames []Frame, field []Field return i.createInputDefinition(name, frames, field) } -func (i *Index) createInputDefinition(name string, frames []Frame, field []Field) (*InputDefinition, error) { +func (i *Index) createInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { if name == "" { return nil, errors.New("input-definition name required") } diff --git a/input_definition.go b/input_definition.go index 789ade7cc..3aa6e6657 100644 --- a/input_definition.go +++ b/input_definition.go @@ -1,7 +1,6 @@ package pilosa import ( - "fmt" "io" "io/ioutil" "os" @@ -15,6 +14,8 @@ type InputDefinition struct { broadcaster Broadcaster Stats StatsClient LogOutput io.Writer + frames []Frame + fields []Field } func NewInputDefinition(path, index, name string) (*InputDefinition, error) { @@ -40,9 +41,7 @@ func (i *InputDefinition) Index() string { return i.index } func (i *InputDefinition) Path() string { return i.path } func (i *InputDefinition) Open() error { - fmt.Println(i.path) if err := func() error { - // Ensure the input definition's path exists. if err := os.MkdirAll(i.path, 0777); err != nil { return err } @@ -55,7 +54,6 @@ func (i *InputDefinition) Open() error { }(); err != nil { return err } - return nil } @@ -70,7 +68,7 @@ func (f *InputDefinition) saveMeta() error { //} // Write to meta file. - if err := ioutil.WriteFile(filepath.Join(f.path, f.name), []byte("123"), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(f.path, f.name), []byte(""), 0666); err != nil { return err } diff --git a/input_definition_test.go b/input_definition_test.go new file mode 100644 index 000000000..191b5d182 --- /dev/null +++ b/input_definition_test.go @@ -0,0 +1 @@ +package pilosa diff --git a/internal/private.pb.go b/internal/private.pb.go index 1c546e74e..48d410476 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -24,7 +24,6 @@ Frame Index InputDefinition - InputDefinitionMeta InputDefinitionField Action CreateInputDefinitionMessage @@ -248,8 +247,9 @@ func (m *Index) GetInputDefinitions() []*InputDefinition { } type InputDefinition struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *InputDefinitionMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Frames []*Frame `protobuf:"bytes,2,rep,name=Frames" json:"Frames,omitempty"` + InputDefinitionFields []*InputDefinitionField `protobuf:"bytes,3,rep,name=InputDefinitionFields" json:"InputDefinitionFields,omitempty"` } func (m *InputDefinition) Reset() { *m = InputDefinition{} } @@ -257,31 +257,14 @@ func (m *InputDefinition) String() string { return proto.CompactTextS func (*InputDefinition) ProtoMessage() {} func (*InputDefinition) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } -func (m *InputDefinition) GetMeta() *InputDefinitionMeta { - if m != nil { - return m.Meta - } - return nil -} - -type InputDefinitionMeta struct { - Frames []*Frame `protobuf:"bytes,1,rep,name=Frames" json:"Frames,omitempty"` - InputDefinitionFields []*InputDefinitionField `protobuf:"bytes,2,rep,name=InputDefinitionFields" json:"InputDefinitionFields,omitempty"` -} - -func (m *InputDefinitionMeta) Reset() { *m = InputDefinitionMeta{} } -func (m *InputDefinitionMeta) String() string { return proto.CompactTextString(m) } -func (*InputDefinitionMeta) ProtoMessage() {} -func (*InputDefinitionMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } - -func (m *InputDefinitionMeta) GetFrames() []*Frame { +func (m *InputDefinition) GetFrames() []*Frame { if m != nil { return m.Frames } return nil } -func (m *InputDefinitionMeta) GetInputDefinitionFields() []*InputDefinitionField { +func (m *InputDefinition) GetInputDefinitionFields() []*InputDefinitionField { if m != nil { return m.InputDefinitionFields } @@ -297,7 +280,7 @@ type InputDefinitionField struct { func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } func (m *InputDefinitionField) String() string { return proto.CompactTextString(m) } func (*InputDefinitionField) ProtoMessage() {} -func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *InputDefinitionField) GetMeta() *Action { if m != nil { @@ -316,7 +299,7 @@ type Action struct { func (m *Action) Reset() { *m = Action{} } func (m *Action) String() string { return proto.CompactTextString(m) } func (*Action) ProtoMessage() {} -func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *Action) GetValueMap() map[string]uint64 { if m != nil { @@ -326,23 +309,15 @@ func (m *Action) GetValueMap() map[string]uint64 { } type CreateInputDefinitionMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - InputDefinition string `protobuf:"bytes,2,opt,name=InputDefinition,proto3" json:"InputDefinition,omitempty"` - Meta *InputDefinitionMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + InputDefinition string `protobuf:"bytes,2,opt,name=InputDefinition,proto3" json:"InputDefinition,omitempty"` } func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinitionMessage{} } func (m *CreateInputDefinitionMessage) String() string { return proto.CompactTextString(m) } func (*CreateInputDefinitionMessage) ProtoMessage() {} func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{18} -} - -func (m *CreateInputDefinitionMessage) GetMeta() *InputDefinitionMeta { - if m != nil { - return m.Meta - } - return nil + return fileDescriptorPrivate, []int{17} } type NodeStatus struct { @@ -354,7 +329,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeStatus) GetIndexes() []*Index { if m != nil { @@ -370,7 +345,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *ClusterStatus) GetNodes() []*NodeStatus { if m != nil { @@ -395,7 +370,6 @@ func init() { proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") - proto.RegisterType((*InputDefinitionMeta)(nil), "internal.InputDefinitionMeta") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") proto.RegisterType((*Action)(nil), "internal.Action") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") @@ -976,37 +950,9 @@ func (m *InputDefinition) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if m.Meta != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n13, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n13 - } - return i, nil -} - -func (m *InputDefinitionMeta) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *InputDefinitionMeta) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l if len(m.Frames) > 0 { for _, msg := range m.Frames { - dAtA[i] = 0xa + dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) @@ -1018,7 +964,7 @@ func (m *InputDefinitionMeta) MarshalTo(dAtA []byte) (int, error) { } if len(m.InputDefinitionFields) > 0 { for _, msg := range m.InputDefinitionFields { - dAtA[i] = 0x12 + dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) @@ -1066,11 +1012,11 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n14, err := m.Meta.MarshalTo(dAtA[i:]) + n13, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n13 } return i, nil } @@ -1153,16 +1099,6 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.InputDefinition))) i += copy(dAtA[i:], m.InputDefinition) } - if m.Meta != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n15, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n15 - } return i, nil } @@ -1513,16 +1449,6 @@ func (m *InputDefinition) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - -func (m *InputDefinitionMeta) Size() (n int) { - var l int - _ = l if len(m.Frames) > 0 { for _, e := range m.Frames { l = e.Size() @@ -1591,10 +1517,6 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } return n } @@ -3607,89 +3529,6 @@ func (m *InputDefinition) Unmarshal(dAtA []byte) error { m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &InputDefinitionMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *InputDefinitionMeta) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: InputDefinitionMeta: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: InputDefinitionMeta: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) } @@ -3720,7 +3559,7 @@ func (m *InputDefinitionMeta) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 2: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionFields", wireType) } @@ -4224,39 +4063,6 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { } m.InputDefinition = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &InputDefinitionMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4606,58 +4412,57 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 848 bytes of a gzipped FileDescriptorProto + // 820 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xa6, 0xe3, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0xed, 0x0d, 0x68, 0x36, 0x0a, 0x96, 0x35, 0x42, - 0xac, 0xf1, 0xc1, 0xd2, 0x86, 0x0b, 0xda, 0xe5, 0x00, 0x6b, 0x7b, 0xb5, 0x16, 0x78, 0x81, 0x76, - 0xb4, 0xe2, 0x84, 0xd4, 0xb1, 0x0b, 0x18, 0x65, 0x3c, 0x33, 0x4c, 0xf7, 0x24, 0x6b, 0x0e, 0xbc, - 0x03, 0x12, 0x07, 0x24, 0x9e, 0x81, 0xf7, 0xe0, 0xc8, 0x95, 0x1b, 0x0a, 0x17, 0x1e, 0x03, 0xf5, - 0xcf, 0xfc, 0x78, 0x6c, 0x67, 0x09, 0xb7, 0xa9, 0xaf, 0xfe, 0xbe, 0xaa, 0xae, 0x2a, 0x1b, 0x0e, - 0xe2, 0xc4, 0xbf, 0xe2, 0x12, 0x07, 0x71, 0x12, 0xc9, 0x88, 0x36, 0xfd, 0x50, 0x62, 0x12, 0xf2, - 0xc0, 0xfb, 0x1c, 0x5a, 0x93, 0x70, 0x81, 0xaf, 0xa6, 0x28, 0x39, 0xed, 0xc2, 0xfe, 0x30, 0x0a, - 0xd2, 0x65, 0xf8, 0x19, 0xbf, 0xc0, 0xc0, 0x25, 0x5d, 0xd2, 0x6b, 0xb1, 0x32, 0xa4, 0x2c, 0xce, - 0xfd, 0x25, 0x7e, 0x99, 0xf2, 0x50, 0xa6, 0x4b, 0x77, 0xcf, 0x58, 0x94, 0x20, 0xef, 0x37, 0x02, - 0xad, 0x67, 0x09, 0x5f, 0xa2, 0x8e, 0x78, 0x02, 0x4d, 0x16, 0x5d, 0x97, 0xc3, 0xe5, 0x32, 0x7d, - 0x0f, 0x0e, 0x27, 0xe1, 0x15, 0x26, 0x02, 0xc7, 0x21, 0xbf, 0x08, 0x70, 0xa1, 0xc3, 0x35, 0x59, - 0x05, 0xa5, 0xa7, 0xd0, 0x1a, 0xf2, 0xf9, 0x77, 0x78, 0xbe, 0x8a, 0xd1, 0xad, 0xe9, 0x20, 0x05, - 0x90, 0x6b, 0x67, 0xfe, 0x0f, 0xe8, 0x3a, 0x5d, 0xd2, 0x3b, 0x60, 0x05, 0x50, 0xe5, 0x5b, 0xdf, - 0xe4, 0xeb, 0xc1, 0xe1, 0x64, 0x19, 0x47, 0x89, 0x64, 0x28, 0xe2, 0x28, 0x14, 0x48, 0xdb, 0x50, - 0x1b, 0x27, 0x89, 0xa5, 0xab, 0x3e, 0xbd, 0x1f, 0xa1, 0xfd, 0x34, 0x88, 0xe6, 0x97, 0x23, 0x2e, - 0x39, 0xc3, 0xef, 0x53, 0x14, 0x92, 0x1e, 0x43, 0x5d, 0x37, 0xce, 0xda, 0x19, 0x41, 0xa1, 0xba, - 0x78, 0xdb, 0x19, 0x23, 0x28, 0x54, 0xfb, 0x6b, 0xf6, 0x0e, 0x33, 0x82, 0x42, 0x67, 0x81, 0x3f, - 0x37, 0xac, 0x1d, 0x66, 0x04, 0x4a, 0xc1, 0x79, 0xe9, 0xe3, 0xb5, 0xa5, 0xaa, 0xbf, 0xbd, 0x09, - 0xdc, 0x2b, 0xe5, 0xb7, 0x34, 0xdf, 0x86, 0x06, 0x8b, 0xae, 0x27, 0x23, 0xe1, 0x92, 0x6e, 0xad, - 0xe7, 0x30, 0x2b, 0xe9, 0x86, 0xe8, 0x17, 0x53, 0xaa, 0x3d, 0xad, 0x2a, 0x00, 0xef, 0x01, 0xd4, - 0x75, 0x77, 0x54, 0x95, 0x85, 0xaf, 0xfa, 0xf4, 0x7e, 0x25, 0x70, 0x6f, 0xca, 0x5f, 0x69, 0x1a, - 0x22, 0x4f, 0xf3, 0x1c, 0x5a, 0x39, 0xa8, 0xad, 0xf7, 0xcf, 0xfa, 0x83, 0x6c, 0x7c, 0x06, 0x1b, - 0xf6, 0x05, 0x32, 0x0e, 0x65, 0xb2, 0x62, 0x85, 0xf3, 0xc9, 0x47, 0x70, 0xb8, 0xae, 0x54, 0x1c, - 0x2e, 0x71, 0x95, 0x75, 0xfa, 0x12, 0x57, 0xaa, 0x27, 0x57, 0x3c, 0x48, 0x4d, 0xff, 0x1c, 0x66, - 0x84, 0xc7, 0x7b, 0x1f, 0x12, 0xef, 0x6b, 0xa0, 0xc3, 0x04, 0xb9, 0x44, 0x1d, 0x60, 0x8a, 0x42, - 0xf0, 0x6f, 0x71, 0xf7, 0x2b, 0x98, 0xce, 0xee, 0x95, 0x3b, 0x7b, 0x0a, 0xad, 0x89, 0xb0, 0xb3, - 0xa5, 0x5f, 0xa2, 0xc9, 0x0a, 0xc0, 0xeb, 0x03, 0x1d, 0x61, 0x80, 0x12, 0xed, 0x3a, 0xdc, 0x12, - 0xdf, 0x9b, 0x65, 0x5c, 0x5e, 0x6f, 0x4b, 0x1f, 0x82, 0xa3, 0x36, 0x41, 0x53, 0xd9, 0x3f, 0xbb, - 0x5f, 0xb4, 0x2e, 0x5f, 0x3b, 0xa6, 0x0d, 0x3c, 0x3f, 0x0b, 0x6a, 0xb7, 0xe7, 0x35, 0x05, 0x6e, - 0x19, 0xb3, 0x2c, 0x55, 0xad, 0x9a, 0x2a, 0xdf, 0x47, 0x9b, 0xea, 0xe3, 0xac, 0xd6, 0xff, 0x9b, - 0xca, 0x1b, 0x59, 0x54, 0x8d, 0xeb, 0x0b, 0xa5, 0x35, 0x3e, 0xfa, 0x7b, 0x77, 0xc9, 0x55, 0x1e, - 0xff, 0x10, 0x9b, 0xf2, 0x6e, 0x61, 0x2a, 0x9d, 0x53, 0x47, 0x26, 0x1b, 0x2c, 0xbb, 0x61, 0xb9, - 0x4c, 0x1f, 0x42, 0x43, 0x67, 0x15, 0xae, 0xa3, 0x67, 0xf7, 0xa8, 0xc2, 0x86, 0x59, 0xb5, 0x5a, - 0x27, 0x3b, 0xe4, 0x75, 0xb3, 0x4e, 0x46, 0xa2, 0x63, 0x68, 0x4f, 0xc2, 0x38, 0x95, 0x23, 0xfc, - 0xc6, 0x0f, 0x7d, 0xe9, 0x47, 0xa1, 0x70, 0x1b, 0x3a, 0xd4, 0x83, 0x32, 0xa3, 0x35, 0x0b, 0xb6, - 0xe1, 0xe2, 0x7d, 0x05, 0x47, 0x15, 0x6c, 0x6b, 0xcd, 0x8f, 0xd6, 0x6a, 0x7e, 0x67, 0x67, 0x86, - 0x52, 0x13, 0x7f, 0x26, 0x70, 0x7f, 0x8b, 0xb6, 0x54, 0x39, 0xb9, 0xbd, 0xf2, 0x73, 0x78, 0xab, - 0xe2, 0xff, 0xcc, 0xc7, 0x60, 0x61, 0x8e, 0xc7, 0xfe, 0x59, 0x67, 0x27, 0x09, 0x6d, 0xc6, 0xb6, - 0x3b, 0x7b, 0x31, 0x1c, 0x6f, 0x53, 0x6c, 0xad, 0xba, 0x03, 0xf0, 0x45, 0xe2, 0x2f, 0x79, 0xb2, - 0xfa, 0x14, 0x57, 0xf6, 0x57, 0xa0, 0x84, 0xd0, 0x77, 0xd7, 0x06, 0xbb, 0x5d, 0x10, 0xfa, 0x64, - 0xae, 0xdb, 0x6d, 0x1a, 0xf1, 0x27, 0x81, 0x86, 0x01, 0x8a, 0xa1, 0x25, 0xe5, 0xfd, 0xe8, 0x43, - 0xfb, 0xa5, 0xba, 0x27, 0x23, 0x14, 0xd2, 0x0f, 0xb9, 0xb2, 0xb4, 0x53, 0xbd, 0x81, 0xd3, 0xc7, - 0xd0, 0xd4, 0xd8, 0x94, 0xc7, 0x6e, 0xad, 0xda, 0x07, 0x93, 0x65, 0x90, 0x19, 0x98, 0x4b, 0x97, - 0xdb, 0xab, 0xec, 0xfa, 0x16, 0x67, 0x87, 0x5d, 0x0b, 0x27, 0x4f, 0xe0, 0x60, 0xcd, 0xe1, 0x4e, - 0xd7, 0xef, 0x27, 0x02, 0xa7, 0xd9, 0xc9, 0xa9, 0x3c, 0xf5, 0x6d, 0xcb, 0xdb, 0xdb, 0x98, 0x3a, - 0x5b, 0xf0, 0xc6, 0x30, 0x3e, 0x5a, 0x6b, 0xf1, 0x7f, 0x1a, 0x3c, 0x0e, 0xf0, 0x22, 0x5a, 0xe0, - 0x4c, 0x72, 0x99, 0x0a, 0xf5, 0xae, 0xcf, 0x23, 0x21, 0xb3, 0x77, 0x55, 0xdf, 0xfa, 0x0e, 0x4b, - 0x2e, 0xf3, 0xdb, 0xa1, 0x05, 0xfa, 0x3e, 0xbc, 0xa9, 0xd9, 0xa1, 0xb0, 0x9d, 0x3d, 0xaa, 0xac, - 0x36, 0xcb, 0xf4, 0xde, 0x13, 0x38, 0x18, 0x06, 0xa9, 0x90, 0x98, 0xd8, 0x2c, 0x7d, 0xa8, 0xab, - 0x9c, 0xd9, 0x4c, 0x1f, 0x17, 0x9e, 0x05, 0x15, 0x66, 0x4c, 0x9e, 0xb6, 0x7f, 0xbf, 0xe9, 0x90, - 0x3f, 0x6e, 0x3a, 0xe4, 0xaf, 0x9b, 0x0e, 0xf9, 0xe5, 0xef, 0xce, 0x1b, 0x17, 0x0d, 0xfd, 0xef, - 0xe7, 0x83, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x8e, 0x46, 0x6d, 0x0e, 0x09, 0x00, 0x00, + 0x10, 0x66, 0xec, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0x6d, 0x02, 0x9a, 0x8d, 0x22, 0x2b, 0x6a, 0x21, + 0x36, 0xe4, 0x90, 0xc3, 0x72, 0x41, 0xbb, 0x1c, 0x60, 0xe3, 0xac, 0xd6, 0x82, 0x2c, 0xd0, 0x8e, + 0xf6, 0x88, 0xd4, 0xb1, 0x0b, 0x18, 0x65, 0x3c, 0x33, 0x4c, 0xf7, 0x24, 0x6b, 0x0e, 0x3c, 0x07, + 0x12, 0x57, 0xae, 0xbc, 0x07, 0x47, 0xae, 0xdc, 0x50, 0xb8, 0xf0, 0x18, 0xa8, 0xab, 0x7b, 0x7e, + 0x3c, 0xb6, 0xb3, 0xca, 0xde, 0xa6, 0xbe, 0xfa, 0xfb, 0xaa, 0xba, 0xaa, 0x6c, 0xd8, 0x4e, 0xb3, + 0xf0, 0x5a, 0x6a, 0x3c, 0x49, 0xb3, 0x44, 0x27, 0xac, 0x17, 0xc6, 0x1a, 0xb3, 0x58, 0x46, 0xfc, + 0x6b, 0xe8, 0x8f, 0xe3, 0x19, 0xbe, 0x3e, 0x47, 0x2d, 0xd9, 0x21, 0x6c, 0x9d, 0x26, 0x51, 0x3e, + 0x8f, 0xbf, 0x92, 0x97, 0x18, 0x05, 0xde, 0xa1, 0x77, 0xd4, 0x17, 0x75, 0xc8, 0x58, 0x5c, 0x84, + 0x73, 0xfc, 0x36, 0x97, 0xb1, 0xce, 0xe7, 0x41, 0xcb, 0x5a, 0xd4, 0x20, 0xfe, 0x87, 0x07, 0xfd, + 0xe7, 0x99, 0x9c, 0x23, 0x45, 0xdc, 0x87, 0x9e, 0x48, 0x6e, 0xea, 0xe1, 0x4a, 0x99, 0x7d, 0x04, + 0x3b, 0xe3, 0xf8, 0x1a, 0x33, 0x85, 0x67, 0xb1, 0xbc, 0x8c, 0x70, 0x46, 0xe1, 0x7a, 0xa2, 0x81, + 0xb2, 0x03, 0xe8, 0x9f, 0xca, 0xe9, 0x8f, 0x78, 0xb1, 0x48, 0x31, 0x68, 0x53, 0x90, 0x0a, 0x28, + 0xb5, 0x93, 0xf0, 0x67, 0x0c, 0xfc, 0x43, 0xef, 0x68, 0x5b, 0x54, 0x40, 0x93, 0x6f, 0x67, 0x95, + 0x2f, 0x87, 0x9d, 0xf1, 0x3c, 0x4d, 0x32, 0x2d, 0x50, 0xa5, 0x49, 0xac, 0x90, 0x0d, 0xa0, 0x7d, + 0x96, 0x65, 0x8e, 0xae, 0xf9, 0xe4, 0xbf, 0xc0, 0xe0, 0x59, 0x94, 0x4c, 0xaf, 0x46, 0x52, 0x4b, + 0x81, 0x3f, 0xe5, 0xa8, 0x34, 0xdb, 0x83, 0x0e, 0x35, 0xce, 0xd9, 0x59, 0xc1, 0xa0, 0x54, 0xbc, + 0xeb, 0x8c, 0x15, 0x0c, 0x4a, 0xfe, 0xc4, 0xde, 0x17, 0x56, 0x30, 0xe8, 0x24, 0x0a, 0xa7, 0x96, + 0xb5, 0x2f, 0xac, 0xc0, 0x18, 0xf8, 0xaf, 0x42, 0xbc, 0x71, 0x54, 0xe9, 0x9b, 0x8f, 0xe1, 0x41, + 0x2d, 0xbf, 0xa3, 0xf9, 0x01, 0x74, 0x45, 0x72, 0x33, 0x1e, 0xa9, 0xc0, 0x3b, 0x6c, 0x1f, 0xf9, + 0xc2, 0x49, 0xd4, 0x10, 0x7a, 0x31, 0xa3, 0x6a, 0x91, 0xaa, 0x02, 0xf8, 0x43, 0xe8, 0x50, 0x77, + 0x4c, 0x95, 0x95, 0xaf, 0xf9, 0xe4, 0xbf, 0x79, 0xf0, 0xe0, 0x5c, 0xbe, 0x26, 0x1a, 0xaa, 0x4c, + 0xf3, 0x02, 0xfa, 0x25, 0x48, 0xd6, 0x5b, 0x8f, 0x8f, 0x4f, 0x8a, 0xf1, 0x39, 0x59, 0xb1, 0xaf, + 0x90, 0xb3, 0x58, 0x67, 0x0b, 0x51, 0x39, 0xef, 0x7f, 0x06, 0x3b, 0xcb, 0x4a, 0xc3, 0xe1, 0x0a, + 0x17, 0x45, 0xa7, 0xaf, 0x70, 0x61, 0x7a, 0x72, 0x2d, 0xa3, 0xdc, 0xf6, 0xcf, 0x17, 0x56, 0x78, + 0xd2, 0xfa, 0xd4, 0xe3, 0xdf, 0x01, 0x3b, 0xcd, 0x50, 0x6a, 0xa4, 0x00, 0xe7, 0xa8, 0x94, 0xfc, + 0x01, 0x37, 0xbf, 0x82, 0xed, 0x6c, 0xab, 0xde, 0xd9, 0x03, 0xe8, 0x8f, 0x95, 0x9b, 0x2d, 0x7a, + 0x89, 0x9e, 0xa8, 0x00, 0x7e, 0x0c, 0x6c, 0x84, 0x11, 0x6a, 0x74, 0xeb, 0x70, 0x47, 0x7c, 0x3e, + 0x29, 0xb8, 0xbc, 0xd9, 0x96, 0x3d, 0x02, 0xdf, 0x6c, 0x02, 0x51, 0xd9, 0x7a, 0xfc, 0x5e, 0xd5, + 0xba, 0x72, 0xed, 0x04, 0x19, 0xf0, 0xb0, 0x08, 0xea, 0xb6, 0xe7, 0x0d, 0x05, 0xae, 0x19, 0xb3, + 0x22, 0x55, 0xbb, 0x99, 0xaa, 0xdc, 0x47, 0x97, 0xea, 0xf3, 0xa2, 0xd6, 0xb7, 0x4d, 0xc5, 0x47, + 0x0e, 0x35, 0xe3, 0xfa, 0xd2, 0x68, 0xad, 0x0f, 0x7d, 0x6f, 0x2e, 0xb9, 0xc9, 0xe3, 0x3f, 0xcf, + 0xa5, 0xbc, 0x5f, 0x98, 0x46, 0xe7, 0xcc, 0x91, 0x29, 0x06, 0xcb, 0x6d, 0x58, 0x29, 0xb3, 0x47, + 0xd0, 0xa5, 0xac, 0x2a, 0xf0, 0x69, 0x76, 0x77, 0x1b, 0x6c, 0x84, 0x53, 0x9b, 0x75, 0x72, 0x43, + 0xde, 0xb1, 0xeb, 0x64, 0x25, 0x76, 0x06, 0x83, 0x71, 0x9c, 0xe6, 0x7a, 0x84, 0xdf, 0x87, 0x71, + 0xa8, 0xc3, 0x24, 0x56, 0x41, 0x97, 0x42, 0x3d, 0xac, 0x33, 0x5a, 0xb2, 0x10, 0x2b, 0x2e, 0xfc, + 0x77, 0x0f, 0x76, 0x1b, 0xe0, 0x86, 0xa2, 0x0b, 0xbe, 0xad, 0xbb, 0xf9, 0x5e, 0xc0, 0xfb, 0x8d, + 0x78, 0xcf, 0x43, 0x8c, 0x66, 0x2a, 0x68, 0x93, 0xdf, 0x70, 0x23, 0x39, 0x32, 0x13, 0xeb, 0x9d, + 0x79, 0x0a, 0x7b, 0xeb, 0x14, 0x6b, 0xa9, 0x0e, 0x01, 0xbe, 0xc9, 0xc2, 0xb9, 0xcc, 0x16, 0x5f, + 0xe2, 0xc2, 0xdd, 0xee, 0x1a, 0xc2, 0x3e, 0x5c, 0x1a, 0xc7, 0x41, 0x45, 0xe8, 0x8b, 0x29, 0x35, + 0xc9, 0xce, 0xc0, 0xdf, 0x1e, 0x74, 0x2d, 0x50, 0x8d, 0x9a, 0x57, 0x9f, 0xea, 0x63, 0x18, 0xbc, + 0x32, 0x57, 0x60, 0x84, 0x4a, 0x87, 0xb1, 0x34, 0x96, 0x6e, 0x16, 0x57, 0x70, 0xf6, 0x04, 0x7a, + 0x84, 0x9d, 0xcb, 0x74, 0xb5, 0x0f, 0x36, 0xcb, 0x49, 0x61, 0x60, 0xef, 0x53, 0x69, 0x6f, 0xb2, + 0xd3, 0x05, 0x2d, 0xce, 0x31, 0x09, 0xfb, 0x4f, 0x61, 0x7b, 0xc9, 0xe1, 0x9e, 0x37, 0xeb, 0xa0, + 0xb8, 0x13, 0x4b, 0x3d, 0xbd, 0x7b, 0xe3, 0x8e, 0x56, 0x26, 0xc5, 0xd5, 0xdb, 0x84, 0xb9, 0x04, + 0x78, 0x99, 0xcc, 0x70, 0xa2, 0xa5, 0xce, 0x95, 0x79, 0xa3, 0x17, 0x89, 0xd2, 0xc5, 0x1b, 0x99, + 0x6f, 0xba, 0x84, 0x5a, 0xea, 0x72, 0x7b, 0x49, 0x60, 0x1f, 0xc3, 0xbb, 0x94, 0x0a, 0x8b, 0x69, + 0xd9, 0x6d, 0x2c, 0x97, 0x28, 0xf4, 0xfc, 0x29, 0x6c, 0x9f, 0x46, 0xb9, 0xd2, 0x98, 0xb9, 0x2c, + 0xc7, 0xd0, 0x31, 0x39, 0x8b, 0xdf, 0x82, 0xbd, 0xca, 0xb3, 0xa2, 0x22, 0xac, 0xc9, 0xb3, 0xc1, + 0x9f, 0xb7, 0x43, 0xef, 0xaf, 0xdb, 0xa1, 0xf7, 0xcf, 0xed, 0xd0, 0xfb, 0xf5, 0xdf, 0xe1, 0x3b, + 0x97, 0x5d, 0xfa, 0xff, 0xf1, 0xc9, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4a, 0x74, 0xd0, 0x83, + 0x90, 0x08, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 660633f2c..1c494c080 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -103,7 +103,6 @@ message Action { message CreateInputDefinitionMessage { string Index = 1; string InputDefinition = 2; - InputDefinitionMeta Meta = 3; } message NodeStatus { From fdacca31474d018f810890dfe1e03dac6364c9fc Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 12 Jun 2017 00:11:20 -0500 Subject: [PATCH 04/99] save inputdefinition as protobuf to file --- handler.go | 45 +++++++++++-- holder.go | 1 + index.go | 39 ++++++++++- input_definition.go | 82 ++++++++++++++++++---- internal/private.pb.go | 150 +++++++++++++++++++++-------------------- internal/private.proto | 2 +- 6 files changed, 223 insertions(+), 96 deletions(-) diff --git a/handler.go b/handler.go index 00ced11ce..c2d77e4a4 100644 --- a/handler.go +++ b/handler.go @@ -131,7 +131,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostDefinition).Methods("POST") - router.HandleFunc("/index/{index}input-definition/{input-definition}", handler.handleDeleteDefinition).Methods("DELETE") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteDefinition).Methods("DELETE") return router } @@ -1515,6 +1515,9 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } + fmt.Println(req.Frames) + fmt.Println(req.Fields) + // Find index. index := h.Holder.Index(indexName) if index == nil { @@ -1544,19 +1547,53 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + //Find index. + index := h.Holder.Index(indexName) + if index == nil { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } + return + } + + inputDef := index.inputDefinition(inputDefName) + res, err := json.Marshal(inputDef) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fmt.Println("RESPONSE", string(res)) } func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } + return + } + + // Delete frame from the index. + if err := index.DeleteInputDefinition(inputDefName); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } type postInputDefinition struct { Frames []InputFrame `json:"frames,omitempty"` - Fields []Field `json:"fields,omitempty"` + Fields []Field `json:"fields,omitempty"` } type InputFrame struct { - Name string + Name string Options FrameOptions -} \ No newline at end of file +} diff --git a/holder.go b/holder.go index 59fef1560..22289e4a8 100644 --- a/holder.go +++ b/holder.go @@ -96,6 +96,7 @@ func (h *Holder) Open() error { index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { + h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { diff --git a/index.go b/index.go index 9a4f88149..aa02abccb 100644 --- a/index.go +++ b/index.go @@ -171,12 +171,13 @@ func (i *Index) openFrames() error { } for _, fi := range fis { - if !fi.IsDir() { + if !fi.IsDir() || fi.Name() == ".input-definitions" { continue } fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { + fmt.Println(err) return ErrName } if err := fr.Open(); err != nil { @@ -345,6 +346,8 @@ func (i *Index) Frame(name string) *Frame { func (i *Index) frame(name string) *Frame { return i.frames[name] } +func (i *Index) inputDefinition(name string) *InputDefinition { return i.inputDefinitions[name] } + // Frames returns a list of all frames in the index. func (i *Index) Frames() []*Frame { i.mu.Lock() @@ -383,9 +386,11 @@ func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field [] return i.createInputDefinition(name, frames, field) } -func (i *Index) createInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { +func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) { if name == "" { return nil, errors.New("input-definition name required") + } else if len(frames) == 0 || len(fields) == 0 { + return nil, errors.New("frames and fields are required") } // Initialize input definition. @@ -399,6 +404,8 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, field [] return nil, err } + inputDef.frames = frames + inputDef.fields = fields if err = inputDef.saveMeta(); err != nil { return nil, err } @@ -495,7 +502,7 @@ func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) return nil, err } f.LogOutput = i.LogOutput - f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.Stats = i.Stats.WithTags(fmt.Sprintf("input-definition:%s", name)) f.broadcaster = i.broadcaster return f, nil } @@ -527,6 +534,32 @@ func (i *Index) DeleteFrame(name string) error { return nil } +// DeleteFrame removes a frame from the index. +func (i *Index) DeleteInputDefinition(name string) error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ignore if input definition doesn't exist. + f := i.inputDefinition(name) + if f == nil { + return nil + } + + //if err := f.Close(); err != nil { + // return err + //} + + // Delete input definition file. + if err := os.Remove(filepath.Join(i.InputDefPath(), name)); err != nil { + return err + } + + // Remove reference. + delete(i.inputDefinitions, name) + + return nil +} + type indexSlice []*Index func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/input_definition.go b/input_definition.go index 3aa6e6657..8e1729dbe 100644 --- a/input_definition.go +++ b/input_definition.go @@ -1,6 +1,8 @@ package pilosa import ( + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" "io" "io/ioutil" "os" @@ -14,8 +16,8 @@ type InputDefinition struct { broadcaster Broadcaster Stats StatsClient LogOutput io.Writer - frames []Frame - fields []Field + frames []InputFrame + fields []Field } func NewInputDefinition(path, index, name string) (*InputDefinition, error) { @@ -46,9 +48,9 @@ func (i *InputDefinition) Open() error { return err } - //if err := i.loadMeta(); err != nil { - // return err - //} + if err := i.loadMeta(); err != nil { + return err + } return nil }(); err != nil { @@ -57,18 +59,70 @@ func (i *InputDefinition) Open() error { return nil } +func (i *InputDefinition) loadMeta() error { + var pb internal.InputDefinition + buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name)) + if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + // Copy metadata fields. + i.name = pb.Name + i.frames = pb.Frames + i.fields = pb.InputDefinitionFields + return nil +} + //saveMeta writes meta data for the frame. -func (f *InputDefinition) saveMeta() error { +func (i *InputDefinition) saveMeta() error { // Marshal metadata. - //buf, err := proto.Marshal(&internal.InputDefinitionMeta{ - // Frames: f. - //}) - //if err != nil { - // return err - //} + var frames []*internal.Frame + for _, fr := range i.frames { + frameMeta := &internal.FrameMeta{ + RowLabel: fr.Options.RowLabel, + InverseEnabled: fr.Options.InverseEnabled, + CacheType: fr.Options.CacheType, + CacheSize: fr.Options.CacheSize, + TimeQuantum: string(fr.Options.TimeQuantum), + } + frame := &internal.Frame{Name: fr.Name, Meta: frameMeta} + frames = append(frames, frame) + } + + var fields []*internal.InputDefinitionField + for _, field := range i.fields { + var actions []*internal.Action + for _, action := range field.Actions { + actionMeta := &internal.Action{ + Frame: action.Frame, + ValueDestination: action.ValueDestination, + ValueMap: action.ValueMap, + RowID: action.RowID, + } + actions = append(actions, actionMeta) + } + + fieldMeta := &internal.InputDefinitionField{ + Name: field.Name, + PrimaryKey: field.PrimaryKey, + Actions: actions, + } + fields = append(fields, fieldMeta) + } + buf, err := proto.Marshal(&internal.InputDefinition{ + Name: i.name, + Frames: frames, + InputDefinitionFields: fields, + }) + if err != nil { + return err + } // Write to meta file. - if err := ioutil.WriteFile(filepath.Join(f.path, f.name), []byte(""), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(i.path, i.name), buf, 0666); err != nil { return err } @@ -84,7 +138,7 @@ type InputDefinitionMeta struct { type Field struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` - Actions []Action `json:"action,omitempty"` + Actions []Action `json:"actions,omitempty"` } type Action struct { diff --git a/internal/private.pb.go b/internal/private.pb.go index 48d410476..6ef948b59 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -272,9 +272,9 @@ func (m *InputDefinition) GetInputDefinitionFields() []*InputDefinitionField { } type InputDefinitionField struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` - Meta *Action `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` + Actions []*Action `protobuf:"bytes,3,rep,name=Actions" json:"Actions,omitempty"` } func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } @@ -282,9 +282,9 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } -func (m *InputDefinitionField) GetMeta() *Action { +func (m *InputDefinitionField) GetActions() []*Action { if m != nil { - return m.Meta + return m.Actions } return nil } @@ -1008,15 +1008,17 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.Meta != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n13, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.Actions) > 0 { + for _, msg := range m.Actions { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n } - i += n13 } return i, nil } @@ -1474,9 +1476,11 @@ func (m *InputDefinitionField) Size() (n int) { if m.PrimaryKey { n += 2 } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) + if len(m.Actions) > 0 { + for _, e := range m.Actions { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } } return n } @@ -3691,7 +3695,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { m.PrimaryKey = bool(v != 0) case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Actions", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3715,10 +3719,8 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Meta == nil { - m.Meta = &Action{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Actions = append(m.Actions, &Action{}) + if err := m.Actions[len(m.Actions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4412,57 +4414,57 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 820 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0x66, 0xec, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0x6d, 0x02, 0x9a, 0x8d, 0x22, 0x2b, 0x6a, 0x21, - 0x36, 0xe4, 0x90, 0xc3, 0x72, 0x41, 0xbb, 0x1c, 0x60, 0xe3, 0xac, 0xd6, 0x82, 0x2c, 0xd0, 0x8e, - 0xf6, 0x88, 0xd4, 0xb1, 0x0b, 0x18, 0x65, 0x3c, 0x33, 0x4c, 0xf7, 0x24, 0x6b, 0x0e, 0x3c, 0x07, - 0x12, 0x57, 0xae, 0xbc, 0x07, 0x47, 0xae, 0xdc, 0x50, 0xb8, 0xf0, 0x18, 0xa8, 0xab, 0x7b, 0x7e, - 0x3c, 0xb6, 0xb3, 0xca, 0xde, 0xa6, 0xbe, 0xfa, 0xfb, 0xaa, 0xba, 0xaa, 0x6c, 0xd8, 0x4e, 0xb3, - 0xf0, 0x5a, 0x6a, 0x3c, 0x49, 0xb3, 0x44, 0x27, 0xac, 0x17, 0xc6, 0x1a, 0xb3, 0x58, 0x46, 0xfc, - 0x6b, 0xe8, 0x8f, 0xe3, 0x19, 0xbe, 0x3e, 0x47, 0x2d, 0xd9, 0x21, 0x6c, 0x9d, 0x26, 0x51, 0x3e, - 0x8f, 0xbf, 0x92, 0x97, 0x18, 0x05, 0xde, 0xa1, 0x77, 0xd4, 0x17, 0x75, 0xc8, 0x58, 0x5c, 0x84, - 0x73, 0xfc, 0x36, 0x97, 0xb1, 0xce, 0xe7, 0x41, 0xcb, 0x5a, 0xd4, 0x20, 0xfe, 0x87, 0x07, 0xfd, - 0xe7, 0x99, 0x9c, 0x23, 0x45, 0xdc, 0x87, 0x9e, 0x48, 0x6e, 0xea, 0xe1, 0x4a, 0x99, 0x7d, 0x04, - 0x3b, 0xe3, 0xf8, 0x1a, 0x33, 0x85, 0x67, 0xb1, 0xbc, 0x8c, 0x70, 0x46, 0xe1, 0x7a, 0xa2, 0x81, - 0xb2, 0x03, 0xe8, 0x9f, 0xca, 0xe9, 0x8f, 0x78, 0xb1, 0x48, 0x31, 0x68, 0x53, 0x90, 0x0a, 0x28, - 0xb5, 0x93, 0xf0, 0x67, 0x0c, 0xfc, 0x43, 0xef, 0x68, 0x5b, 0x54, 0x40, 0x93, 0x6f, 0x67, 0x95, - 0x2f, 0x87, 0x9d, 0xf1, 0x3c, 0x4d, 0x32, 0x2d, 0x50, 0xa5, 0x49, 0xac, 0x90, 0x0d, 0xa0, 0x7d, - 0x96, 0x65, 0x8e, 0xae, 0xf9, 0xe4, 0xbf, 0xc0, 0xe0, 0x59, 0x94, 0x4c, 0xaf, 0x46, 0x52, 0x4b, - 0x81, 0x3f, 0xe5, 0xa8, 0x34, 0xdb, 0x83, 0x0e, 0x35, 0xce, 0xd9, 0x59, 0xc1, 0xa0, 0x54, 0xbc, - 0xeb, 0x8c, 0x15, 0x0c, 0x4a, 0xfe, 0xc4, 0xde, 0x17, 0x56, 0x30, 0xe8, 0x24, 0x0a, 0xa7, 0x96, - 0xb5, 0x2f, 0xac, 0xc0, 0x18, 0xf8, 0xaf, 0x42, 0xbc, 0x71, 0x54, 0xe9, 0x9b, 0x8f, 0xe1, 0x41, - 0x2d, 0xbf, 0xa3, 0xf9, 0x01, 0x74, 0x45, 0x72, 0x33, 0x1e, 0xa9, 0xc0, 0x3b, 0x6c, 0x1f, 0xf9, - 0xc2, 0x49, 0xd4, 0x10, 0x7a, 0x31, 0xa3, 0x6a, 0x91, 0xaa, 0x02, 0xf8, 0x43, 0xe8, 0x50, 0x77, - 0x4c, 0x95, 0x95, 0xaf, 0xf9, 0xe4, 0xbf, 0x79, 0xf0, 0xe0, 0x5c, 0xbe, 0x26, 0x1a, 0xaa, 0x4c, - 0xf3, 0x02, 0xfa, 0x25, 0x48, 0xd6, 0x5b, 0x8f, 0x8f, 0x4f, 0x8a, 0xf1, 0x39, 0x59, 0xb1, 0xaf, - 0x90, 0xb3, 0x58, 0x67, 0x0b, 0x51, 0x39, 0xef, 0x7f, 0x06, 0x3b, 0xcb, 0x4a, 0xc3, 0xe1, 0x0a, - 0x17, 0x45, 0xa7, 0xaf, 0x70, 0x61, 0x7a, 0x72, 0x2d, 0xa3, 0xdc, 0xf6, 0xcf, 0x17, 0x56, 0x78, - 0xd2, 0xfa, 0xd4, 0xe3, 0xdf, 0x01, 0x3b, 0xcd, 0x50, 0x6a, 0xa4, 0x00, 0xe7, 0xa8, 0x94, 0xfc, - 0x01, 0x37, 0xbf, 0x82, 0xed, 0x6c, 0xab, 0xde, 0xd9, 0x03, 0xe8, 0x8f, 0x95, 0x9b, 0x2d, 0x7a, - 0x89, 0x9e, 0xa8, 0x00, 0x7e, 0x0c, 0x6c, 0x84, 0x11, 0x6a, 0x74, 0xeb, 0x70, 0x47, 0x7c, 0x3e, - 0x29, 0xb8, 0xbc, 0xd9, 0x96, 0x3d, 0x02, 0xdf, 0x6c, 0x02, 0x51, 0xd9, 0x7a, 0xfc, 0x5e, 0xd5, - 0xba, 0x72, 0xed, 0x04, 0x19, 0xf0, 0xb0, 0x08, 0xea, 0xb6, 0xe7, 0x0d, 0x05, 0xae, 0x19, 0xb3, - 0x22, 0x55, 0xbb, 0x99, 0xaa, 0xdc, 0x47, 0x97, 0xea, 0xf3, 0xa2, 0xd6, 0xb7, 0x4d, 0xc5, 0x47, - 0x0e, 0x35, 0xe3, 0xfa, 0xd2, 0x68, 0xad, 0x0f, 0x7d, 0x6f, 0x2e, 0xb9, 0xc9, 0xe3, 0x3f, 0xcf, - 0xa5, 0xbc, 0x5f, 0x98, 0x46, 0xe7, 0xcc, 0x91, 0x29, 0x06, 0xcb, 0x6d, 0x58, 0x29, 0xb3, 0x47, - 0xd0, 0xa5, 0xac, 0x2a, 0xf0, 0x69, 0x76, 0x77, 0x1b, 0x6c, 0x84, 0x53, 0x9b, 0x75, 0x72, 0x43, - 0xde, 0xb1, 0xeb, 0x64, 0x25, 0x76, 0x06, 0x83, 0x71, 0x9c, 0xe6, 0x7a, 0x84, 0xdf, 0x87, 0x71, - 0xa8, 0xc3, 0x24, 0x56, 0x41, 0x97, 0x42, 0x3d, 0xac, 0x33, 0x5a, 0xb2, 0x10, 0x2b, 0x2e, 0xfc, - 0x77, 0x0f, 0x76, 0x1b, 0xe0, 0x86, 0xa2, 0x0b, 0xbe, 0xad, 0xbb, 0xf9, 0x5e, 0xc0, 0xfb, 0x8d, - 0x78, 0xcf, 0x43, 0x8c, 0x66, 0x2a, 0x68, 0x93, 0xdf, 0x70, 0x23, 0x39, 0x32, 0x13, 0xeb, 0x9d, - 0x79, 0x0a, 0x7b, 0xeb, 0x14, 0x6b, 0xa9, 0x0e, 0x01, 0xbe, 0xc9, 0xc2, 0xb9, 0xcc, 0x16, 0x5f, - 0xe2, 0xc2, 0xdd, 0xee, 0x1a, 0xc2, 0x3e, 0x5c, 0x1a, 0xc7, 0x41, 0x45, 0xe8, 0x8b, 0x29, 0x35, - 0xc9, 0xce, 0xc0, 0xdf, 0x1e, 0x74, 0x2d, 0x50, 0x8d, 0x9a, 0x57, 0x9f, 0xea, 0x63, 0x18, 0xbc, - 0x32, 0x57, 0x60, 0x84, 0x4a, 0x87, 0xb1, 0x34, 0x96, 0x6e, 0x16, 0x57, 0x70, 0xf6, 0x04, 0x7a, - 0x84, 0x9d, 0xcb, 0x74, 0xb5, 0x0f, 0x36, 0xcb, 0x49, 0x61, 0x60, 0xef, 0x53, 0x69, 0x6f, 0xb2, - 0xd3, 0x05, 0x2d, 0xce, 0x31, 0x09, 0xfb, 0x4f, 0x61, 0x7b, 0xc9, 0xe1, 0x9e, 0x37, 0xeb, 0xa0, - 0xb8, 0x13, 0x4b, 0x3d, 0xbd, 0x7b, 0xe3, 0x8e, 0x56, 0x26, 0xc5, 0xd5, 0xdb, 0x84, 0xb9, 0x04, - 0x78, 0x99, 0xcc, 0x70, 0xa2, 0xa5, 0xce, 0x95, 0x79, 0xa3, 0x17, 0x89, 0xd2, 0xc5, 0x1b, 0x99, - 0x6f, 0xba, 0x84, 0x5a, 0xea, 0x72, 0x7b, 0x49, 0x60, 0x1f, 0xc3, 0xbb, 0x94, 0x0a, 0x8b, 0x69, - 0xd9, 0x6d, 0x2c, 0x97, 0x28, 0xf4, 0xfc, 0x29, 0x6c, 0x9f, 0x46, 0xb9, 0xd2, 0x98, 0xb9, 0x2c, - 0xc7, 0xd0, 0x31, 0x39, 0x8b, 0xdf, 0x82, 0xbd, 0xca, 0xb3, 0xa2, 0x22, 0xac, 0xc9, 0xb3, 0xc1, - 0x9f, 0xb7, 0x43, 0xef, 0xaf, 0xdb, 0xa1, 0xf7, 0xcf, 0xed, 0xd0, 0xfb, 0xf5, 0xdf, 0xe1, 0x3b, - 0x97, 0x5d, 0xfa, 0xff, 0xf1, 0xc9, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4a, 0x74, 0xd0, 0x83, - 0x90, 0x08, 0x00, 0x00, + // 823 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0x67, 0xed, 0xb5, 0x6b, 0xbf, 0x28, 0x89, 0x3b, 0x04, 0xb4, 0x8d, 0x22, 0x2b, 0x9a, 0x03, + 0x0d, 0x3e, 0xf8, 0x50, 0x2e, 0xa8, 0xe5, 0x00, 0xb5, 0x5d, 0xd5, 0x02, 0x17, 0x18, 0x47, 0x3d, + 0x22, 0x4d, 0xec, 0x07, 0xac, 0xb2, 0xde, 0x35, 0x3b, 0xb3, 0x4e, 0xcd, 0x81, 0xcf, 0x81, 0xc4, + 0x95, 0x2b, 0xdf, 0x83, 0x23, 0x57, 0x6e, 0x28, 0x5c, 0xf8, 0x18, 0x68, 0xfe, 0xed, 0xae, 0xd7, + 0x76, 0xaa, 0xf4, 0x36, 0xef, 0xf7, 0xfe, 0xfd, 0xe6, 0xcd, 0x7b, 0x6f, 0x17, 0x0e, 0x97, 0x69, + 0xb8, 0xe2, 0x12, 0xfb, 0xcb, 0x34, 0x91, 0x09, 0x69, 0x85, 0xb1, 0xc4, 0x34, 0xe6, 0x11, 0xfd, + 0x1a, 0xda, 0xe3, 0x78, 0x8e, 0x6f, 0x26, 0x28, 0x39, 0x39, 0x87, 0x83, 0x41, 0x12, 0x65, 0x8b, + 0xf8, 0x2b, 0x7e, 0x85, 0x51, 0xe0, 0x9d, 0x7b, 0x17, 0x6d, 0x56, 0x86, 0x94, 0xc5, 0x65, 0xb8, + 0xc0, 0x6f, 0x33, 0x1e, 0xcb, 0x6c, 0x11, 0xd4, 0x8c, 0x45, 0x09, 0xa2, 0x7f, 0x78, 0xd0, 0x7e, + 0x91, 0xf2, 0x05, 0xea, 0x88, 0xa7, 0xd0, 0x62, 0xc9, 0x4d, 0x39, 0x5c, 0x2e, 0x93, 0x8f, 0xe0, + 0x68, 0x1c, 0xaf, 0x30, 0x15, 0x38, 0x8a, 0xf9, 0x55, 0x84, 0x73, 0x1d, 0xae, 0xc5, 0x2a, 0x28, + 0x39, 0x83, 0xf6, 0x80, 0xcf, 0x7e, 0xc4, 0xcb, 0xf5, 0x12, 0x83, 0xba, 0x0e, 0x52, 0x00, 0xb9, + 0x76, 0x1a, 0xfe, 0x8c, 0x81, 0x7f, 0xee, 0x5d, 0x1c, 0xb2, 0x02, 0xa8, 0xf2, 0x6d, 0x6c, 0xf3, + 0xa5, 0x70, 0x34, 0x5e, 0x2c, 0x93, 0x54, 0x32, 0x14, 0xcb, 0x24, 0x16, 0x48, 0x3a, 0x50, 0x1f, + 0xa5, 0xa9, 0xa5, 0xab, 0x8e, 0xf4, 0x17, 0xe8, 0x3c, 0x8f, 0x92, 0xd9, 0xf5, 0x90, 0x4b, 0xce, + 0xf0, 0xa7, 0x0c, 0x85, 0x24, 0x27, 0xd0, 0xd0, 0x85, 0xb3, 0x76, 0x46, 0x50, 0xa8, 0xbe, 0xbc, + 0xad, 0x8c, 0x11, 0x14, 0xaa, 0xfd, 0x35, 0x7b, 0x9f, 0x19, 0x41, 0xa1, 0xd3, 0x28, 0x9c, 0x19, + 0xd6, 0x3e, 0x33, 0x02, 0x21, 0xe0, 0xbf, 0x0e, 0xf1, 0xc6, 0x52, 0xd5, 0x67, 0x3a, 0x86, 0x87, + 0xa5, 0xfc, 0x96, 0xe6, 0x87, 0xd0, 0x64, 0xc9, 0xcd, 0x78, 0x28, 0x02, 0xef, 0xbc, 0x7e, 0xe1, + 0x33, 0x2b, 0xe9, 0x82, 0xe8, 0x17, 0x53, 0xaa, 0x9a, 0x56, 0x15, 0x00, 0x7d, 0x04, 0x0d, 0x5d, + 0x1d, 0x75, 0xcb, 0xc2, 0x57, 0x1d, 0xe9, 0x6f, 0x1e, 0x3c, 0x9c, 0xf0, 0x37, 0x9a, 0x86, 0xc8, + 0xd3, 0xbc, 0x84, 0x76, 0x0e, 0x6a, 0xeb, 0x83, 0x27, 0xbd, 0xbe, 0x6b, 0x9f, 0xfe, 0x96, 0x7d, + 0x81, 0x8c, 0x62, 0x99, 0xae, 0x59, 0xe1, 0x7c, 0xfa, 0x19, 0x1c, 0x6d, 0x2a, 0x15, 0x87, 0x6b, + 0x5c, 0xbb, 0x4a, 0x5f, 0xe3, 0x5a, 0xd5, 0x64, 0xc5, 0xa3, 0xcc, 0xd4, 0xcf, 0x67, 0x46, 0x78, + 0x5a, 0xfb, 0xd4, 0xa3, 0xdf, 0x01, 0x19, 0xa4, 0xc8, 0x25, 0xea, 0x00, 0x13, 0x14, 0x82, 0xff, + 0x80, 0xfb, 0x5f, 0xc1, 0x54, 0xb6, 0x56, 0xae, 0xec, 0x19, 0xb4, 0xc7, 0xc2, 0xf6, 0x96, 0x7e, + 0x89, 0x16, 0x2b, 0x00, 0xda, 0x03, 0x32, 0xc4, 0x08, 0x25, 0xda, 0x71, 0xb8, 0x23, 0x3e, 0x9d, + 0x3a, 0x2e, 0x6f, 0xb7, 0x25, 0x8f, 0xc1, 0x57, 0x93, 0xa0, 0xa9, 0x1c, 0x3c, 0x79, 0xbf, 0x28, + 0x5d, 0x3e, 0x76, 0x4c, 0x1b, 0xd0, 0xd0, 0x05, 0xb5, 0xd3, 0xf3, 0x96, 0x0b, 0xee, 0x68, 0x33, + 0x97, 0xaa, 0x5e, 0x4d, 0x95, 0xcf, 0xa3, 0x4d, 0xf5, 0xb9, 0xbb, 0xeb, 0xbb, 0xa6, 0xa2, 0x43, + 0x8b, 0xaa, 0x76, 0x7d, 0xa5, 0xb4, 0xc6, 0x47, 0x9f, 0xf7, 0x5f, 0xb9, 0xca, 0xe3, 0x3f, 0xcf, + 0xa6, 0xbc, 0x5f, 0x98, 0x4a, 0xe5, 0xd4, 0x92, 0x71, 0x8d, 0x65, 0x27, 0x2c, 0x97, 0xc9, 0x63, + 0x68, 0xea, 0xac, 0x22, 0xf0, 0x75, 0xef, 0x1e, 0x57, 0xd8, 0x30, 0xab, 0x56, 0xe3, 0x64, 0x9b, + 0xbc, 0x61, 0xc6, 0xc9, 0x48, 0x64, 0x04, 0x9d, 0x71, 0xbc, 0xcc, 0xe4, 0x10, 0xbf, 0x0f, 0xe3, + 0x50, 0x86, 0x49, 0x2c, 0x82, 0xa6, 0x0e, 0xf5, 0xa8, 0xcc, 0x68, 0xc3, 0x82, 0x6d, 0xb9, 0xd0, + 0xdf, 0x3d, 0x38, 0xae, 0x80, 0x7b, 0x2e, 0xed, 0xf8, 0xd6, 0xee, 0xe6, 0x7b, 0x09, 0x1f, 0x54, + 0xe2, 0xbd, 0x08, 0x31, 0x9a, 0x8b, 0xa0, 0xae, 0xfd, 0xba, 0x7b, 0xc9, 0x69, 0x33, 0xb6, 0xdb, + 0x99, 0xae, 0xe0, 0x64, 0x97, 0x62, 0x27, 0xd5, 0x2e, 0xc0, 0x37, 0x69, 0xb8, 0xe0, 0xe9, 0xfa, + 0x4b, 0x5c, 0xdb, 0xdd, 0x5d, 0x42, 0x48, 0x0f, 0x1e, 0x7c, 0x31, 0x33, 0x05, 0x33, 0x9c, 0x3a, + 0x05, 0x27, 0xa3, 0x60, 0xce, 0x80, 0xfe, 0xed, 0x41, 0xd3, 0x9c, 0x8b, 0x86, 0xf3, 0xca, 0xbd, + 0xdd, 0x83, 0xce, 0x6b, 0xb5, 0x0b, 0x86, 0x28, 0x64, 0x18, 0x73, 0x65, 0x69, 0x3b, 0x72, 0x0b, + 0x27, 0x4f, 0xa1, 0xa5, 0xb1, 0x09, 0x5f, 0x6e, 0x57, 0xc3, 0x64, 0xe9, 0x3b, 0x03, 0xb3, 0xa5, + 0x72, 0x7b, 0x95, 0x5d, 0xef, 0x51, 0xb7, 0x94, 0xb5, 0x70, 0xfa, 0x0c, 0x0e, 0x37, 0x1c, 0xee, + 0xb9, 0xb9, 0xce, 0xdc, 0xb6, 0xd8, 0xa8, 0xec, 0xdd, 0x73, 0x77, 0xb1, 0xd5, 0x2f, 0xf6, 0xbe, + 0x55, 0x98, 0x72, 0x80, 0x57, 0xc9, 0x1c, 0xa7, 0x92, 0xcb, 0x4c, 0xa8, 0x97, 0x7a, 0x99, 0x08, + 0xe9, 0x5e, 0x4a, 0x9d, 0xf5, 0x3e, 0x94, 0x5c, 0xe6, 0x33, 0xac, 0x05, 0xf2, 0x31, 0x3c, 0xd0, + 0xa9, 0xd0, 0xbd, 0xcf, 0x71, 0x65, 0xc4, 0x98, 0xd3, 0xd3, 0x67, 0x70, 0x38, 0x88, 0x32, 0x21, + 0x31, 0xb5, 0x59, 0x7a, 0xd0, 0x50, 0x39, 0xdd, 0x17, 0xe1, 0xa4, 0xf0, 0x2c, 0xa8, 0x30, 0x63, + 0xf2, 0xbc, 0xf3, 0xe7, 0x6d, 0xd7, 0xfb, 0xeb, 0xb6, 0xeb, 0xfd, 0x73, 0xdb, 0xf5, 0x7e, 0xfd, + 0xb7, 0xfb, 0xde, 0x55, 0x53, 0xff, 0x85, 0x7c, 0xf2, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, + 0x37, 0x53, 0x60, 0x96, 0x08, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1c494c080..4329a33f7 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -89,7 +89,7 @@ message InputDefinition { message InputDefinitionField { string Name = 1; bool PrimaryKey = 2; - Action Meta = 3; + repeated Action Actions = 3; } message Action { From a62ec47eec1187849d15b17b1cc002b26872432c Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 12 Jun 2017 15:04:27 -0500 Subject: [PATCH 05/99] load input-definition when startup servers --- handler.go | 32 +++++++++++++++----------- index.go | 49 +++++++++++++++++++++++++++++++++++----- input_definition.go | 55 ++++++++++++++++++++++++++++----------------- 3 files changed, 96 insertions(+), 40 deletions(-) diff --git a/handler.go b/handler.go index c2d77e4a4..38c5f9080 100644 --- a/handler.go +++ b/handler.go @@ -1515,9 +1515,6 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } - fmt.Println(req.Frames) - fmt.Println(req.Fields) - // Find index. index := h.Holder.Index(indexName) if index == nil { @@ -1558,14 +1555,14 @@ func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { } return } - - inputDef := index.inputDefinition(inputDefName) - res, err := json.Marshal(inputDef) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + inputDef, _ := index.inputDefinitions[inputDefName] + inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} + if err := json.NewEncoder(w).Encode(getInputDefinitionResponse{ + InputDefinition: inputInfo, + }); err != nil { + h.logger().Printf("write status response error: %s", err) } - fmt.Println("RESPONSE", string(res)) + } func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) { @@ -1581,7 +1578,7 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) return } - // Delete frame from the index. + // Delete input definition from the index. if err := index.DeleteInputDefinition(inputDefName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -1594,6 +1591,15 @@ type postInputDefinition struct { } type InputFrame struct { - Name string - Options FrameOptions + Name string `json:"name,omitempty"` + Options FrameOptions `json:"options,omitempty"` +} + +type getInputDefinitionResponse struct { + InputDefinition InputDefinitionInfo `json:"input-definition"` +} + +type InputDefinitionInfo struct { + Frames []InputFrame `json:"frames"` + Fields []Field `json:"fields"` } diff --git a/index.go b/index.go index aa02abccb..fb6463997 100644 --- a/index.go +++ b/index.go @@ -31,7 +31,8 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "columnID" + DefaultColumnLabel = "columnID" + InputDefinitionRepo = ".input-definitions" ) // Index represents a container for frames. @@ -154,6 +155,44 @@ func (i *Index) Open() error { return err } + if err := i.openInputDefinition(); err != nil { + return err + } + + return nil +} + +// openInputDefinition opens and initializes the input definitions inside the index. +func (i *Index) openInputDefinition() error { + inputDefs, err := os.Open(i.path) + if err != nil { + return err + } + defer inputDefs.Close() + + fis, err := inputDefs.Readdir(0) + if err != nil { + return err + } + for _, fi := range fis { + if fi.Name() != InputDefinitionRepo { + continue + } + inputPath, err := os.Open(i.InputDefPath()) + if err != nil { + return err + } + inputFiles, err := inputPath.Readdir(0) + for _, file := range inputFiles { + input, err := i.newInputDefinition(i.InputDefPath(), file.Name()) + if err != nil { + return err + } + input.Open() + i.inputDefinitions[file.Name()] = input + } + + } return nil } @@ -171,13 +210,12 @@ func (i *Index) openFrames() error { } for _, fi := range fis { - if !fi.IsDir() || fi.Name() == ".input-definitions" { + if !fi.IsDir() || fi.Name() == InputDefinitionRepo { continue } fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { - fmt.Println(err) return ErrName } if err := fr.Open(); err != nil { @@ -334,7 +372,7 @@ func (i *Index) FramePath(name string) string { return filepath.Join(i.path, nam // InputDefPath returns the path to a inputdefinition in the index. func (i *Index) InputDefPath() string { - return filepath.Join(i.path, ".input-definitions") + return filepath.Join(i.path, InputDefinitionRepo) } // Frame returns a frame in the index by name. @@ -410,6 +448,7 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [ return nil, err } i.inputDefinitions[name] = inputDef + fmt.Printf("%+v\n", i.inputDefinitions[name]) return inputDef, nil } @@ -501,8 +540,6 @@ func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) if err != nil { return nil, err } - f.LogOutput = i.LogOutput - f.Stats = i.Stats.WithTags(fmt.Sprintf("input-definition:%s", name)) f.broadcaster = i.broadcaster return f, nil } diff --git a/input_definition.go b/input_definition.go index 8e1729dbe..d0651aede 100644 --- a/input_definition.go +++ b/input_definition.go @@ -3,7 +3,6 @@ package pilosa import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "io" "io/ioutil" "os" "path/filepath" @@ -14,8 +13,6 @@ type InputDefinition struct { path string index string broadcaster Broadcaster - Stats StatsClient - LogOutput io.Writer frames []InputFrame fields []Field } @@ -51,7 +48,6 @@ func (i *InputDefinition) Open() error { if err := i.loadMeta(); err != nil { return err } - return nil }(); err != nil { return err @@ -71,12 +67,43 @@ func (i *InputDefinition) loadMeta() error { } // Copy metadata fields. i.name = pb.Name - i.frames = pb.Frames - i.fields = pb.InputDefinitionFields + for _, fr := range pb.Frames { + frameMeta := fr.Meta + inputFrame := InputFrame{ + Name: fr.Name, + Options: FrameOptions{ + RowLabel: frameMeta.RowLabel, + InverseEnabled: frameMeta.InverseEnabled, + CacheSize: frameMeta.CacheSize, + CacheType: frameMeta.CacheType, + TimeQuantum: TimeQuantum(frameMeta.TimeQuantum), + }, + } + i.frames = append(i.frames, inputFrame) + } + + for _, field := range pb.InputDefinitionFields { + var actions []Action + for _, action := range field.Actions { + actions = append(actions, Action{ + Frame: action.Frame, + ValueDestination: action.ValueDestination, + ValueMap: action.ValueMap, + RowID: action.RowID, + }) + } + + inputField := Field{ + Name: field.Name, + PrimaryKey: field.PrimaryKey, + Actions: actions, + } + i.fields = append(i.fields, inputField) + } return nil } -//saveMeta writes meta data for the frame. +//saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { // Marshal metadata. var frames []*internal.Frame @@ -129,12 +156,6 @@ func (i *InputDefinition) saveMeta() error { return nil } -// FrameOptions represents options to set when initializing a frame. -type InputDefinitionMeta struct { - Frames []Frame `json:"frames,omitempty"` - Fields []Field `json:"fields,omitempty"` -} - type Field struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` @@ -147,11 +168,3 @@ type Action struct { ValueMap map[string]uint64 `json:"valueMap,omitempty"` RowID uint64 `json:"rowID,omitempty"` } - -// Encode converts o into its internal representation. -//func (o *InputDefinitionMeta) Encode() *internal.InputDefinitionMeta { -// return &internal.InputDefinitionMeta{ -// Frames: o.Frames, -// InputDefinitionFields: o.Fields, -// } -//} From 1c2adda8e9385bddd87f981c7059a45ac39af929 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 12 Jun 2017 16:23:40 -0500 Subject: [PATCH 06/99] create frames with input-definition's creation --- handler.go | 22 +++++++++------------- index.go | 17 +++++++++-------- input_definition.go | 3 +++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/handler.go b/handler.go index 38c5f9080..c183f70a4 100644 --- a/handler.go +++ b/handler.go @@ -1507,10 +1507,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { // Decode request. var req postInputDefinition err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the frame - // with default values. - } else if err != nil { + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -1532,15 +1529,14 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } - //err = h.Broadcaster.SendSync( - // &internal.CreateInputDefinitionMessage{ - // Index: indexName, - // InputDefinition: inputDefName, - // Meta: req.Options.Encode(), - // }) - //if err != nil { - // h.logger().Printf("problem sending CreateFrame message: %s", err) - //} + err = h.Broadcaster.SendSync( + &internal.CreateInputDefinitionMessage{ + Index: indexName, + InputDefinition: inputDefName, + }) + if err != nil { + h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + } } func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { diff --git a/index.go b/index.go index fb6463997..12bd5f30f 100644 --- a/index.go +++ b/index.go @@ -414,8 +414,6 @@ func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { // CreateInputDefinition creates a new input definition. func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { - i.mu.Lock() - defer i.mu.Unlock() // Ensure input definition doesn't already exist. if i.inputDefinitions[name] != nil { @@ -431,16 +429,20 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [ return nil, errors.New("frames and fields are required") } + for _, fr := range frames { + _, err := i.CreateFrame(fr.Name, fr.Options) + if err == ErrFrameExists { + continue + } else if err != nil { + return nil, err + } + } + // Initialize input definition. inputDef, err := i.newInputDefinition(i.InputDefPath(), name) - fmt.Println(inputDef) if err != nil { return nil, err } - // Open input definition. - if err := inputDef.Open(); err != nil { - return nil, err - } inputDef.frames = frames inputDef.fields = fields @@ -448,7 +450,6 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [ return nil, err } i.inputDefinitions[name] = inputDef - fmt.Printf("%+v\n", i.inputDefinitions[name]) return inputDef, nil } diff --git a/input_definition.go b/input_definition.go index d0651aede..f194a0745 100644 --- a/input_definition.go +++ b/input_definition.go @@ -105,6 +105,9 @@ func (i *InputDefinition) loadMeta() error { //saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { + if err := os.MkdirAll(i.path, 0777); err != nil { + return err + } // Marshal metadata. var frames []*internal.Frame for _, fr := range i.frames { From ea53216ff29738cdf9fd2f1c06f934d099e27f03 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 12 Jun 2017 23:43:03 -0500 Subject: [PATCH 07/99] expose input definition frames, fields --- index.go | 215 ++++++++++++++++++++++---------------------- index_test.go | 19 ++++ input_definition.go | 6 ++ 3 files changed, 130 insertions(+), 110 deletions(-) diff --git a/index.go b/index.go index 12bd5f30f..b0ebc1c40 100644 --- a/index.go +++ b/index.go @@ -162,40 +162,6 @@ func (i *Index) Open() error { return nil } -// openInputDefinition opens and initializes the input definitions inside the index. -func (i *Index) openInputDefinition() error { - inputDefs, err := os.Open(i.path) - if err != nil { - return err - } - defer inputDefs.Close() - - fis, err := inputDefs.Readdir(0) - if err != nil { - return err - } - for _, fi := range fis { - if fi.Name() != InputDefinitionRepo { - continue - } - inputPath, err := os.Open(i.InputDefPath()) - if err != nil { - return err - } - inputFiles, err := inputPath.Readdir(0) - for _, file := range inputFiles { - input, err := i.newInputDefinition(i.InputDefPath(), file.Name()) - if err != nil { - return err - } - input.Open() - i.inputDefinitions[file.Name()] = input - } - - } - return nil -} - // openFrames opens and initializes the frames inside the index. func (i *Index) openFrames() error { f, err := os.Open(i.path) @@ -412,47 +378,6 @@ func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { return i.createFrame(name, opt) } -// CreateInputDefinition creates a new input definition. -func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { - - // Ensure input definition doesn't already exist. - if i.inputDefinitions[name] != nil { - return nil, ErrInputDefinitionExists - } - return i.createInputDefinition(name, frames, field) -} - -func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) { - if name == "" { - return nil, errors.New("input-definition name required") - } else if len(frames) == 0 || len(fields) == 0 { - return nil, errors.New("frames and fields are required") - } - - for _, fr := range frames { - _, err := i.CreateFrame(fr.Name, fr.Options) - if err == ErrFrameExists { - continue - } else if err != nil { - return nil, err - } - } - - // Initialize input definition. - inputDef, err := i.newInputDefinition(i.InputDefPath(), name) - if err != nil { - return nil, err - } - - inputDef.frames = frames - inputDef.fields = fields - if err = inputDef.saveMeta(); err != nil { - return nil, err - } - i.inputDefinitions[name] = inputDef - return inputDef, nil -} - // CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { i.mu.Lock() @@ -536,15 +461,6 @@ func (i *Index) newFrame(path, name string) (*Frame, error) { return f, nil } -func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) { - f, err := NewInputDefinition(path, i.name, name) - if err != nil { - return nil, err - } - f.broadcaster = i.broadcaster - return f, nil -} - // DeleteFrame removes a frame from the index. func (i *Index) DeleteFrame(name string) error { i.mu.Lock() @@ -572,32 +488,6 @@ func (i *Index) DeleteFrame(name string) error { return nil } -// DeleteFrame removes a frame from the index. -func (i *Index) DeleteInputDefinition(name string) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Ignore if input definition doesn't exist. - f := i.inputDefinition(name) - if f == nil { - return nil - } - - //if err := f.Close(); err != nil { - // return err - //} - - // Delete input definition file. - if err := os.Remove(filepath.Join(i.InputDefPath(), name)); err != nil { - return err - } - - // Remove reference. - delete(i.inputDefinitions, name) - - return nil -} - type indexSlice []*Index func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -711,3 +601,108 @@ type importData struct { RowIDs []uint64 ColumnIDs []uint64 } + +// CreateInputDefinition creates a new input definition. +func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { + + // Ensure input definition doesn't already exist. + if i.inputDefinitions[name] != nil { + return nil, ErrInputDefinitionExists + } + return i.createInputDefinition(name, frames, field) +} + +func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) { + if name == "" { + return nil, errors.New("input-definition name required") + } else if len(frames) == 0 || len(fields) == 0 { + return nil, errors.New("frames and fields are required") + } + + for _, fr := range frames { + _, err := i.CreateFrame(fr.Name, fr.Options) + if err == ErrFrameExists { + continue + } else if err != nil { + return nil, err + } + } + + // Initialize input definition. + inputDef, err := i.newInputDefinition(i.InputDefPath(), name) + if err != nil { + return nil, err + } + + inputDef.frames = frames + inputDef.fields = fields + if err = inputDef.saveMeta(); err != nil { + return nil, err + } + i.inputDefinitions[name] = inputDef + return inputDef, nil +} + +func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) { + f, err := NewInputDefinition(path, i.name, name) + if err != nil { + return nil, err + } + f.broadcaster = i.broadcaster + return f, nil +} + +// DeleteFrame removes a frame from the index. +func (i *Index) DeleteInputDefinition(name string) error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ignore if input definition doesn't exist. + f := i.inputDefinition(name) + if f == nil { + return nil + } + + // Delete input definition file. + if err := os.Remove(filepath.Join(i.InputDefPath(), name)); err != nil { + return err + } + + // Remove reference. + delete(i.inputDefinitions, name) + return nil +} + +// openInputDefinition opens and initializes the input definitions inside the index. +func (i *Index) openInputDefinition() error { + inputDefs, err := os.Open(i.path) + if err != nil { + return err + } + defer inputDefs.Close() + + fis, err := inputDefs.Readdir(0) + if err != nil { + return err + } + for _, fi := range fis { + if fi.Name() != InputDefinitionRepo { + continue + } + inputPath, err := os.Open(i.InputDefPath()) + if err != nil { + return err + } + inputFiles, err := inputPath.Readdir(0) + for _, file := range inputFiles { + input, err := i.newInputDefinition(i.InputDefPath(), file.Name()) + if err != nil { + return err + } + input.Open() + i.inputDefinitions[file.Name()] = input + } + + } + return nil +} diff --git a/index_test.go b/index_test.go index bf59236c6..1e3df2b35 100644 --- a/index_test.go +++ b/index_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "reflect" ) // Ensure index can open and retrieve a frame. @@ -239,3 +240,21 @@ func TestIndex_InvalidName(t *testing.T) { t.Fatalf("unexpected index name %s", index) } } + +func TestIndex_CreateInputDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } else if inputDef.Frames()[0] != frames { + t.Fatalf("unexpected input definition frames", inputDef.Frames()) + } else if !reflect.DeepEqual(inputDef.Fields()[0], fields) { + t.Fatalf("unexpected input definition actions", inputDef.Fields()) + } +} diff --git a/input_definition.go b/input_definition.go index f194a0745..2cc5b9ab6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -39,6 +39,12 @@ func (i *InputDefinition) Index() string { return i.index } // Path returns the path of the input definition was initialized with. func (i *InputDefinition) Path() string { return i.path } +// Frames returns frames of the input definition was initialized with. +func (i *InputDefinition) Frames() []InputFrame { return i.frames } + +// Fields returns frames of the input definition was initialized with. +func (i *InputDefinition) Fields() []Field { return i.fields } + func (i *InputDefinition) Open() error { if err := func() error { if err := os.MkdirAll(i.path, 0777); err != nil { From 5ce6b5d684fe8c6f332c159f9dc3d199e7268b68 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 13 Jun 2017 10:14:14 -0500 Subject: [PATCH 08/99] CreateInputDefinition tests --- holder.go | 1 - index_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 22289e4a8..59fef1560 100644 --- a/holder.go +++ b/holder.go @@ -96,7 +96,6 @@ func (h *Holder) Open() error { index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { diff --git a/index_test.go b/index_test.go index 1e3df2b35..43a064967 100644 --- a/index_test.go +++ b/index_test.go @@ -258,3 +258,32 @@ func TestIndex_CreateInputDefinition(t *testing.T) { t.Fatalf("unexpected input definition actions", inputDef.Fields()) } } + +func TestIndex_CreateExistingInputDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + _ , err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != pilosa.ErrInputDefinitionExists { + t.Fatal(err) + } +} + +func TestIndex_CreateEmptyInputDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + _ , err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{}) + if err.Error() != "frames and fields are required" { + t.Fatal(err) + } +} From 2d3f244a7b3a2919bbc8614ef40c15218c9881fb Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 13 Jun 2017 13:56:08 -0500 Subject: [PATCH 09/99] couple more tests --- handler.go | 6 ++++++ handler_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ index.go | 10 ++++++++++ index_test.go | 28 ++++++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/handler.go b/handler.go index c183f70a4..e3049f3df 100644 --- a/handler.go +++ b/handler.go @@ -1537,6 +1537,10 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { @@ -1599,3 +1603,5 @@ type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` Fields []Field `json:"fields"` } + +type postInputDefinitionResponse struct{} diff --git a/handler_test.go b/handler_test.go index ab65894fb..8c7f3a7e2 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1066,3 +1066,51 @@ func MustReadAll(r io.Reader) []byte { } return buf } + +// Ensure handler can delete a frame. +func TestHandler_CreateInputDefinition(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + inputBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "id", + "primaryKey": true + }, + { + "name": "cabType", + "actions": [ + { + "frame": "cab-type", + "valueDestination": "mapping", + "valueMap": { + "Green": 1, + "Yellow": 2 + } + } + ] + } + ] + }`) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} diff --git a/index.go b/index.go index b0ebc1c40..58bb8adaf 100644 --- a/index.go +++ b/index.go @@ -348,6 +348,16 @@ func (i *Index) Frame(name string) *Frame { return i.frame(name) } +// InputDefinition returns an input definition in the index by name. +func (i *Index) InputDefinition(name string) *InputDefinition { + i.mu.Lock() + defer i.mu.Unlock() + if inputDef, ok := i.inputDefinitions[name]; ok { + return inputDef + } + return nil +} + func (i *Index) frame(name string) *Frame { return i.frames[name] } func (i *Index) inputDefinition(name string) *InputDefinition { return i.inputDefinitions[name] } diff --git a/index_test.go b/index_test.go index 43a064967..125be84c4 100644 --- a/index_test.go +++ b/index_test.go @@ -267,7 +267,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _ , err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) if err != nil { t.Fatal(err) } @@ -282,8 +282,32 @@ func TestIndex_CreateEmptyInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - _ , err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{}) + _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{}) if err.Error() != "frames and fields are required" { t.Fatal(err) } } + +func TestIndex_DeleteInputDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } else if index.InputDefinition("test") == nil { + t.Fatal("No input definition created") + } + + err = index.DeleteInputDefinition("test") + if err != nil { + t.Fatal(err) + } else if index.InputDefinition("test") != nil { + t.Fatal("input definition isn't deleted") + } + +} From 1db97b9312f04b35bca501a74fffe5def6f310ba Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 14 Jun 2017 14:12:07 -0500 Subject: [PATCH 10/99] CreateInputDefinitionMessage now contains the definition and not just the name. --- broadcast.go | 16 ++-- handler.go | 23 ++---- handler_test.go | 2 +- internal/private.pb.go | 178 +++++++++++++++++++++++++++-------------- internal/private.proto | 3 +- 5 files changed, 137 insertions(+), 85 deletions(-) diff --git a/broadcast.go b/broadcast.go index 95a6d58f5..491a48737 100644 --- a/broadcast.go +++ b/broadcast.go @@ -108,11 +108,13 @@ var NopBroadcastReceiver = &nopBroadcastReceiver{} // Broadcast message types. const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateIndex = 2 - MessageTypeDeleteIndex = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 + MessageTypeCreateSlice = 1 + MessageTypeCreateIndex = 2 + MessageTypeDeleteIndex = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 + MessageTypeCreateInputDefinition = 6 + MessageTypeDeleteInputDefinition = 7 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -129,6 +131,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateFrame case *internal.DeleteFrameMessage: typ = MessageTypeDeleteFrame + case *internal.CreateInputDefinitionMessage: + typ = MessageTypeCreateInputDefinition default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -155,6 +159,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateFrameMessage{} case MessageTypeDeleteFrame: m = &internal.DeleteFrameMessage{} + case MessageTypeCreateInputDefinition: + m = &internal.CreateInputDefinitionMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/handler.go b/handler.go index e3049f3df..4586755c2 100644 --- a/handler.go +++ b/handler.go @@ -1505,7 +1505,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { inputDefName := mux.Vars(r)["input-definition"] // Decode request. - var req postInputDefinition + var req internal.InputDefinition err := json.NewDecoder(r.Body).Decode(&req) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1519,20 +1519,14 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } - // Create InputDefinition. - _, err = index.CreateInputDefinition(inputDefName, req.Frames, req.Fields) - if err == ErrInputDefinitionExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + req.Name = inputDefName + fmt.Println(req.String()) err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ - Index: indexName, - InputDefinition: inputDefName, + Index: indexName, + Name: inputDefName, + Definition: &req, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) @@ -1585,11 +1579,6 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) } } -type postInputDefinition struct { - Frames []InputFrame `json:"frames,omitempty"` - Fields []Field `json:"fields,omitempty"` -} - type InputFrame struct { Name string `json:"name,omitempty"` Options FrameOptions `json:"options,omitempty"` diff --git a/handler_test.go b/handler_test.go index 8c7f3a7e2..dd1505036 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1087,7 +1087,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { "cacheType": "ranked" } }], - "fields": [ + "inputdefinitionfields": [ { "name": "id", "primaryKey": true diff --git a/internal/private.pb.go b/internal/private.pb.go index 6ef948b59..a98f25f5b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -309,8 +309,9 @@ func (m *Action) GetValueMap() map[string]uint64 { } type CreateInputDefinitionMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - InputDefinition string `protobuf:"bytes,2,opt,name=InputDefinition,proto3" json:"InputDefinition,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` + Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` } func (m *CreateInputDefinitionMessage) Reset() { *m = CreateInputDefinitionMessage{} } @@ -320,6 +321,13 @@ func (*CreateInputDefinitionMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (m *CreateInputDefinitionMessage) GetDefinition() *InputDefinition { + if m != nil { + return m.Definition + } + return nil +} + type NodeStatus struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -1095,11 +1103,21 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.InputDefinition) > 0 { + if len(m.Name) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.InputDefinition))) - i += copy(dAtA[i:], m.InputDefinition) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Definition != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Definition.Size())) + n13, err := m.Definition.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n13 } return i, nil } @@ -1517,10 +1535,14 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.InputDefinition) + l = len(m.Name) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.Definition != nil { + l = m.Definition.Size() + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -4038,7 +4060,7 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InputDefinition", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -4063,7 +4085,40 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.InputDefinition = string(dAtA[iNdEx:postIndex]) + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Definition", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Definition == nil { + m.Definition = &InputDefinition{} + } + if err := m.Definition.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex default: iNdEx = preIndex @@ -4414,57 +4469,58 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 823 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xed, 0xb5, 0x6b, 0xbf, 0x28, 0x89, 0x3b, 0x04, 0xb4, 0x8d, 0x22, 0x2b, 0x9a, 0x03, - 0x0d, 0x3e, 0xf8, 0x50, 0x2e, 0xa8, 0xe5, 0x00, 0xb5, 0x5d, 0xd5, 0x02, 0x17, 0x18, 0x47, 0x3d, - 0x22, 0x4d, 0xec, 0x07, 0xac, 0xb2, 0xde, 0x35, 0x3b, 0xb3, 0x4e, 0xcd, 0x81, 0xcf, 0x81, 0xc4, - 0x95, 0x2b, 0xdf, 0x83, 0x23, 0x57, 0x6e, 0x28, 0x5c, 0xf8, 0x18, 0x68, 0xfe, 0xed, 0xae, 0xd7, - 0x76, 0xaa, 0xf4, 0x36, 0xef, 0xf7, 0xfe, 0xfd, 0xe6, 0xcd, 0x7b, 0x6f, 0x17, 0x0e, 0x97, 0x69, - 0xb8, 0xe2, 0x12, 0xfb, 0xcb, 0x34, 0x91, 0x09, 0x69, 0x85, 0xb1, 0xc4, 0x34, 0xe6, 0x11, 0xfd, - 0x1a, 0xda, 0xe3, 0x78, 0x8e, 0x6f, 0x26, 0x28, 0x39, 0x39, 0x87, 0x83, 0x41, 0x12, 0x65, 0x8b, - 0xf8, 0x2b, 0x7e, 0x85, 0x51, 0xe0, 0x9d, 0x7b, 0x17, 0x6d, 0x56, 0x86, 0x94, 0xc5, 0x65, 0xb8, - 0xc0, 0x6f, 0x33, 0x1e, 0xcb, 0x6c, 0x11, 0xd4, 0x8c, 0x45, 0x09, 0xa2, 0x7f, 0x78, 0xd0, 0x7e, - 0x91, 0xf2, 0x05, 0xea, 0x88, 0xa7, 0xd0, 0x62, 0xc9, 0x4d, 0x39, 0x5c, 0x2e, 0x93, 0x8f, 0xe0, - 0x68, 0x1c, 0xaf, 0x30, 0x15, 0x38, 0x8a, 0xf9, 0x55, 0x84, 0x73, 0x1d, 0xae, 0xc5, 0x2a, 0x28, - 0x39, 0x83, 0xf6, 0x80, 0xcf, 0x7e, 0xc4, 0xcb, 0xf5, 0x12, 0x83, 0xba, 0x0e, 0x52, 0x00, 0xb9, - 0x76, 0x1a, 0xfe, 0x8c, 0x81, 0x7f, 0xee, 0x5d, 0x1c, 0xb2, 0x02, 0xa8, 0xf2, 0x6d, 0x6c, 0xf3, - 0xa5, 0x70, 0x34, 0x5e, 0x2c, 0x93, 0x54, 0x32, 0x14, 0xcb, 0x24, 0x16, 0x48, 0x3a, 0x50, 0x1f, - 0xa5, 0xa9, 0xa5, 0xab, 0x8e, 0xf4, 0x17, 0xe8, 0x3c, 0x8f, 0x92, 0xd9, 0xf5, 0x90, 0x4b, 0xce, - 0xf0, 0xa7, 0x0c, 0x85, 0x24, 0x27, 0xd0, 0xd0, 0x85, 0xb3, 0x76, 0x46, 0x50, 0xa8, 0xbe, 0xbc, - 0xad, 0x8c, 0x11, 0x14, 0xaa, 0xfd, 0x35, 0x7b, 0x9f, 0x19, 0x41, 0xa1, 0xd3, 0x28, 0x9c, 0x19, - 0xd6, 0x3e, 0x33, 0x02, 0x21, 0xe0, 0xbf, 0x0e, 0xf1, 0xc6, 0x52, 0xd5, 0x67, 0x3a, 0x86, 0x87, - 0xa5, 0xfc, 0x96, 0xe6, 0x87, 0xd0, 0x64, 0xc9, 0xcd, 0x78, 0x28, 0x02, 0xef, 0xbc, 0x7e, 0xe1, - 0x33, 0x2b, 0xe9, 0x82, 0xe8, 0x17, 0x53, 0xaa, 0x9a, 0x56, 0x15, 0x00, 0x7d, 0x04, 0x0d, 0x5d, - 0x1d, 0x75, 0xcb, 0xc2, 0x57, 0x1d, 0xe9, 0x6f, 0x1e, 0x3c, 0x9c, 0xf0, 0x37, 0x9a, 0x86, 0xc8, - 0xd3, 0xbc, 0x84, 0x76, 0x0e, 0x6a, 0xeb, 0x83, 0x27, 0xbd, 0xbe, 0x6b, 0x9f, 0xfe, 0x96, 0x7d, - 0x81, 0x8c, 0x62, 0x99, 0xae, 0x59, 0xe1, 0x7c, 0xfa, 0x19, 0x1c, 0x6d, 0x2a, 0x15, 0x87, 0x6b, - 0x5c, 0xbb, 0x4a, 0x5f, 0xe3, 0x5a, 0xd5, 0x64, 0xc5, 0xa3, 0xcc, 0xd4, 0xcf, 0x67, 0x46, 0x78, - 0x5a, 0xfb, 0xd4, 0xa3, 0xdf, 0x01, 0x19, 0xa4, 0xc8, 0x25, 0xea, 0x00, 0x13, 0x14, 0x82, 0xff, - 0x80, 0xfb, 0x5f, 0xc1, 0x54, 0xb6, 0x56, 0xae, 0xec, 0x19, 0xb4, 0xc7, 0xc2, 0xf6, 0x96, 0x7e, - 0x89, 0x16, 0x2b, 0x00, 0xda, 0x03, 0x32, 0xc4, 0x08, 0x25, 0xda, 0x71, 0xb8, 0x23, 0x3e, 0x9d, - 0x3a, 0x2e, 0x6f, 0xb7, 0x25, 0x8f, 0xc1, 0x57, 0x93, 0xa0, 0xa9, 0x1c, 0x3c, 0x79, 0xbf, 0x28, - 0x5d, 0x3e, 0x76, 0x4c, 0x1b, 0xd0, 0xd0, 0x05, 0xb5, 0xd3, 0xf3, 0x96, 0x0b, 0xee, 0x68, 0x33, - 0x97, 0xaa, 0x5e, 0x4d, 0x95, 0xcf, 0xa3, 0x4d, 0xf5, 0xb9, 0xbb, 0xeb, 0xbb, 0xa6, 0xa2, 0x43, - 0x8b, 0xaa, 0x76, 0x7d, 0xa5, 0xb4, 0xc6, 0x47, 0x9f, 0xf7, 0x5f, 0xb9, 0xca, 0xe3, 0x3f, 0xcf, - 0xa6, 0xbc, 0x5f, 0x98, 0x4a, 0xe5, 0xd4, 0x92, 0x71, 0x8d, 0x65, 0x27, 0x2c, 0x97, 0xc9, 0x63, - 0x68, 0xea, 0xac, 0x22, 0xf0, 0x75, 0xef, 0x1e, 0x57, 0xd8, 0x30, 0xab, 0x56, 0xe3, 0x64, 0x9b, - 0xbc, 0x61, 0xc6, 0xc9, 0x48, 0x64, 0x04, 0x9d, 0x71, 0xbc, 0xcc, 0xe4, 0x10, 0xbf, 0x0f, 0xe3, - 0x50, 0x86, 0x49, 0x2c, 0x82, 0xa6, 0x0e, 0xf5, 0xa8, 0xcc, 0x68, 0xc3, 0x82, 0x6d, 0xb9, 0xd0, - 0xdf, 0x3d, 0x38, 0xae, 0x80, 0x7b, 0x2e, 0xed, 0xf8, 0xd6, 0xee, 0xe6, 0x7b, 0x09, 0x1f, 0x54, - 0xe2, 0xbd, 0x08, 0x31, 0x9a, 0x8b, 0xa0, 0xae, 0xfd, 0xba, 0x7b, 0xc9, 0x69, 0x33, 0xb6, 0xdb, - 0x99, 0xae, 0xe0, 0x64, 0x97, 0x62, 0x27, 0xd5, 0x2e, 0xc0, 0x37, 0x69, 0xb8, 0xe0, 0xe9, 0xfa, - 0x4b, 0x5c, 0xdb, 0xdd, 0x5d, 0x42, 0x48, 0x0f, 0x1e, 0x7c, 0x31, 0x33, 0x05, 0x33, 0x9c, 0x3a, - 0x05, 0x27, 0xa3, 0x60, 0xce, 0x80, 0xfe, 0xed, 0x41, 0xd3, 0x9c, 0x8b, 0x86, 0xf3, 0xca, 0xbd, - 0xdd, 0x83, 0xce, 0x6b, 0xb5, 0x0b, 0x86, 0x28, 0x64, 0x18, 0x73, 0x65, 0x69, 0x3b, 0x72, 0x0b, - 0x27, 0x4f, 0xa1, 0xa5, 0xb1, 0x09, 0x5f, 0x6e, 0x57, 0xc3, 0x64, 0xe9, 0x3b, 0x03, 0xb3, 0xa5, - 0x72, 0x7b, 0x95, 0x5d, 0xef, 0x51, 0xb7, 0x94, 0xb5, 0x70, 0xfa, 0x0c, 0x0e, 0x37, 0x1c, 0xee, - 0xb9, 0xb9, 0xce, 0xdc, 0xb6, 0xd8, 0xa8, 0xec, 0xdd, 0x73, 0x77, 0xb1, 0xd5, 0x2f, 0xf6, 0xbe, - 0x55, 0x98, 0x72, 0x80, 0x57, 0xc9, 0x1c, 0xa7, 0x92, 0xcb, 0x4c, 0xa8, 0x97, 0x7a, 0x99, 0x08, - 0xe9, 0x5e, 0x4a, 0x9d, 0xf5, 0x3e, 0x94, 0x5c, 0xe6, 0x33, 0xac, 0x05, 0xf2, 0x31, 0x3c, 0xd0, - 0xa9, 0xd0, 0xbd, 0xcf, 0x71, 0x65, 0xc4, 0x98, 0xd3, 0xd3, 0x67, 0x70, 0x38, 0x88, 0x32, 0x21, - 0x31, 0xb5, 0x59, 0x7a, 0xd0, 0x50, 0x39, 0xdd, 0x17, 0xe1, 0xa4, 0xf0, 0x2c, 0xa8, 0x30, 0x63, - 0xf2, 0xbc, 0xf3, 0xe7, 0x6d, 0xd7, 0xfb, 0xeb, 0xb6, 0xeb, 0xfd, 0x73, 0xdb, 0xf5, 0x7e, 0xfd, - 0xb7, 0xfb, 0xde, 0x55, 0x53, 0xff, 0x85, 0x7c, 0xf2, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe2, - 0x37, 0x53, 0x60, 0x96, 0x08, 0x00, 0x00, + // 835 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x8e, 0x1b, 0x45, + 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x41, 0x93, 0xd5, 0xca, 0x5a, 0xf5, + 0x81, 0x2c, 0x3e, 0xec, 0x21, 0x5c, 0x20, 0xe1, 0x00, 0xb1, 0x1d, 0xc5, 0x02, 0x07, 0x68, 0xaf, + 0x72, 0x44, 0xea, 0xb5, 0x0b, 0x18, 0xed, 0x78, 0xc6, 0x4c, 0xf7, 0x78, 0x63, 0x0e, 0x5c, 0x78, + 0x09, 0x24, 0xae, 0x5c, 0x79, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, 0x9b, + 0x19, 0x8f, 0x7f, 0xa2, 0xe4, 0xd6, 0xf5, 0x75, 0xfd, 0x7c, 0xfd, 0x4d, 0x55, 0xd9, 0x70, 0xb8, + 0x48, 0xc3, 0x25, 0x97, 0x78, 0xb9, 0x48, 0x13, 0x99, 0x90, 0x56, 0x18, 0x4b, 0x4c, 0x63, 0x1e, + 0xd1, 0xaf, 0xa0, 0x3d, 0x8a, 0x67, 0xf8, 0x6a, 0x8c, 0x92, 0x93, 0x73, 0x38, 0xe8, 0x27, 0x51, + 0x36, 0x8f, 0xbf, 0xe4, 0xd7, 0x18, 0x05, 0xde, 0xb9, 0x77, 0xd1, 0x66, 0x65, 0x48, 0x79, 0x5c, + 0x85, 0x73, 0xfc, 0x26, 0xe3, 0xb1, 0xcc, 0xe6, 0x41, 0xcd, 0x78, 0x94, 0x20, 0xfa, 0x87, 0x07, + 0xed, 0x67, 0x29, 0x9f, 0xa3, 0xce, 0x78, 0x0a, 0x2d, 0x96, 0xdc, 0x96, 0xd3, 0xe5, 0x36, 0xf9, + 0x00, 0x8e, 0x46, 0xf1, 0x12, 0x53, 0x81, 0xc3, 0x98, 0x5f, 0x47, 0x38, 0xd3, 0xe9, 0x5a, 0xac, + 0x82, 0x92, 0x33, 0x68, 0xf7, 0xf9, 0xf4, 0x07, 0xbc, 0x5a, 0x2d, 0x30, 0xa8, 0xeb, 0x24, 0x05, + 0x90, 0xdf, 0x4e, 0xc2, 0x9f, 0x30, 0xf0, 0xcf, 0xbd, 0x8b, 0x43, 0x56, 0x00, 0x55, 0xbe, 0x8d, + 0x4d, 0xbe, 0x14, 0x8e, 0x46, 0xf3, 0x45, 0x92, 0x4a, 0x86, 0x62, 0x91, 0xc4, 0x02, 0x49, 0x07, + 0xea, 0xc3, 0x34, 0xb5, 0x74, 0xd5, 0x91, 0xfe, 0x0c, 0x9d, 0xa7, 0x51, 0x32, 0xbd, 0x19, 0x70, + 0xc9, 0x19, 0xfe, 0x98, 0xa1, 0x90, 0xe4, 0x04, 0x1a, 0x5a, 0x38, 0xeb, 0x67, 0x0c, 0x85, 0xea, + 0xc7, 0x5b, 0x65, 0x8c, 0xa1, 0x50, 0x1d, 0xaf, 0xd9, 0xfb, 0xcc, 0x18, 0x0a, 0x9d, 0x44, 0xe1, + 0xd4, 0xb0, 0xf6, 0x99, 0x31, 0x08, 0x01, 0xff, 0x65, 0x88, 0xb7, 0x96, 0xaa, 0x3e, 0xd3, 0x11, + 0xdc, 0x2f, 0xd5, 0xb7, 0x34, 0xdf, 0x87, 0x26, 0x4b, 0x6e, 0x47, 0x03, 0x11, 0x78, 0xe7, 0xf5, + 0x0b, 0x9f, 0x59, 0x4b, 0x0b, 0xa2, 0xbf, 0x98, 0xba, 0xaa, 0xe9, 0xab, 0x02, 0xa0, 0x0f, 0xa0, + 0xa1, 0xd5, 0x51, 0xaf, 0x2c, 0x62, 0xd5, 0x91, 0xfe, 0xe6, 0xc1, 0xfd, 0x31, 0x7f, 0xa5, 0x69, + 0x88, 0xbc, 0xcc, 0x73, 0x68, 0xe7, 0xa0, 0xf6, 0x3e, 0x78, 0xd4, 0xbb, 0x74, 0xed, 0x73, 0xb9, + 0xe1, 0x5f, 0x20, 0xc3, 0x58, 0xa6, 0x2b, 0x56, 0x04, 0x9f, 0x7e, 0x0a, 0x47, 0xeb, 0x97, 0x8a, + 0xc3, 0x0d, 0xae, 0x9c, 0xd2, 0x37, 0xb8, 0x52, 0x9a, 0x2c, 0x79, 0x94, 0x19, 0xfd, 0x7c, 0x66, + 0x8c, 0xc7, 0xb5, 0x8f, 0x3d, 0xfa, 0x2d, 0x90, 0x7e, 0x8a, 0x5c, 0xa2, 0x4e, 0x30, 0x46, 0x21, + 0xf8, 0xf7, 0xb8, 0xfb, 0x2b, 0x18, 0x65, 0x6b, 0x65, 0x65, 0xcf, 0xa0, 0x3d, 0x12, 0xb6, 0xb7, + 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x03, 0x8c, 0x50, 0xa2, 0x1d, 0x87, 0x3d, 0xf9, + 0xe9, 0xc4, 0x71, 0x79, 0xbd, 0x2f, 0x79, 0x08, 0xbe, 0x9a, 0x04, 0x4d, 0xe5, 0xe0, 0xd1, 0xbb, + 0x85, 0x74, 0xf9, 0xd8, 0x31, 0xed, 0x40, 0x43, 0x97, 0xd4, 0x4e, 0xcf, 0x6b, 0x1e, 0xb8, 0xa5, + 0xcd, 0x5c, 0xa9, 0x7a, 0xb5, 0x54, 0x3e, 0x8f, 0xb6, 0xd4, 0x67, 0xee, 0xad, 0x6f, 0x5b, 0x8a, + 0x0e, 0x2c, 0xaa, 0xda, 0xf5, 0x85, 0xba, 0x35, 0x31, 0xfa, 0xbc, 0xfb, 0xc9, 0x55, 0x1e, 0xff, + 0x79, 0xb6, 0xe4, 0x9b, 0xa5, 0xa9, 0x28, 0xa7, 0x96, 0x8c, 0x6b, 0x2c, 0x3b, 0x61, 0xb9, 0x4d, + 0x1e, 0x42, 0x53, 0x57, 0x15, 0x81, 0xaf, 0x7b, 0xf7, 0xb8, 0xc2, 0x86, 0xd9, 0x6b, 0x35, 0x4e, + 0xb6, 0xc9, 0x1b, 0x66, 0x9c, 0x8c, 0x45, 0x86, 0xd0, 0x19, 0xc5, 0x8b, 0x4c, 0x0e, 0xf0, 0xbb, + 0x30, 0x0e, 0x65, 0x98, 0xc4, 0x22, 0x68, 0xea, 0x54, 0x0f, 0xca, 0x8c, 0xd6, 0x3c, 0xd8, 0x46, + 0x08, 0xfd, 0xdd, 0x83, 0xe3, 0x0a, 0xb8, 0xe3, 0xd1, 0x8e, 0x6f, 0x6d, 0x3f, 0xdf, 0x2b, 0x78, + 0xaf, 0x92, 0xef, 0x59, 0x88, 0xd1, 0x4c, 0x04, 0x75, 0x1d, 0xd7, 0xdd, 0x49, 0x4e, 0xbb, 0xb1, + 0xed, 0xc1, 0x74, 0x09, 0x27, 0xdb, 0x2e, 0xb6, 0x52, 0xed, 0x02, 0x7c, 0x9d, 0x86, 0x73, 0x9e, + 0xae, 0xbe, 0xc0, 0x95, 0xdd, 0xdd, 0x25, 0x84, 0xf4, 0xe0, 0xde, 0xe7, 0x53, 0x23, 0x98, 0xe1, + 0xd4, 0x29, 0x38, 0x99, 0x0b, 0xe6, 0x1c, 0xe8, 0xdf, 0x1e, 0x34, 0xcd, 0xb9, 0x68, 0x38, 0xaf, + 0xdc, 0xdb, 0x3d, 0xe8, 0xbc, 0x54, 0xbb, 0x60, 0x80, 0x42, 0x86, 0x31, 0x57, 0x9e, 0xb6, 0x23, + 0x37, 0x70, 0xf2, 0x18, 0x5a, 0x1a, 0x1b, 0xf3, 0xc5, 0xa6, 0x1a, 0xa6, 0xca, 0xa5, 0x73, 0x30, + 0x5b, 0x2a, 0xf7, 0x57, 0xd5, 0xf5, 0x1e, 0x75, 0x4b, 0x59, 0x1b, 0xa7, 0x4f, 0xe0, 0x70, 0x2d, + 0xe0, 0x8d, 0x36, 0xd7, 0x2f, 0x1e, 0x9c, 0xb9, 0x75, 0xb1, 0x26, 0xed, 0xfe, 0xc1, 0x73, 0x92, + 0xd7, 0x4a, 0x92, 0x7f, 0x02, 0x50, 0x84, 0xdb, 0x39, 0xdf, 0xd3, 0x86, 0x25, 0x67, 0xca, 0x01, + 0x5e, 0x24, 0x33, 0x9c, 0x48, 0x2e, 0x33, 0xa1, 0x92, 0x3f, 0x4f, 0x84, 0x74, 0xdf, 0x53, 0x9d, + 0xf5, 0xd6, 0x94, 0x5c, 0xe6, 0x93, 0xae, 0x0d, 0xf2, 0x21, 0xdc, 0xd3, 0x7c, 0xd0, 0x7d, 0xc5, + 0xe3, 0xca, 0x20, 0x32, 0x77, 0x4f, 0x9f, 0xc0, 0x61, 0x3f, 0xca, 0x84, 0xc4, 0xd4, 0x56, 0xe9, + 0x41, 0x43, 0xd5, 0x74, 0xbf, 0x1b, 0x27, 0x45, 0x64, 0x41, 0x85, 0x19, 0x97, 0xa7, 0x9d, 0x3f, + 0xef, 0xba, 0xde, 0x5f, 0x77, 0x5d, 0xef, 0x9f, 0xbb, 0xae, 0xf7, 0xeb, 0xbf, 0xdd, 0x77, 0xae, + 0x9b, 0xfa, 0xbf, 0xca, 0x47, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0x2c, 0x40, 0x32, 0xbc, 0xbc, + 0x08, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 4329a33f7..d0b60d366 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -102,7 +102,8 @@ message Action { message CreateInputDefinitionMessage { string Index = 1; - string InputDefinition = 2; + string Name = 2; + InputDefinition Definition = 3; } message NodeStatus { From 00dec45e58afbe01e95546b50ba8b7542a9d3a0f Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 14 Jun 2017 14:23:25 -0500 Subject: [PATCH 11/99] Input definition "Fields" parameter name is consistent throughout now. --- handler.go | 2 +- handler_test.go | 2 +- input_definition.go | 13 +++-- internal/private.pb.go | 130 ++++++++++++++++++++--------------------- internal/private.proto | 2 +- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/handler.go b/handler.go index 4586755c2..299343dc5 100644 --- a/handler.go +++ b/handler.go @@ -1519,8 +1519,8 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } + // Add the name req.Name = inputDefName - fmt.Println(req.String()) err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ diff --git a/handler_test.go b/handler_test.go index dd1505036..8c7f3a7e2 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1087,7 +1087,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { "cacheType": "ranked" } }], - "inputdefinitionfields": [ + "fields": [ { "name": "id", "primaryKey": true diff --git a/input_definition.go b/input_definition.go index 2cc5b9ab6..31a354a3b 100644 --- a/input_definition.go +++ b/input_definition.go @@ -1,11 +1,12 @@ package pilosa import ( - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" "io/ioutil" "os" "path/filepath" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) type InputDefinition struct { @@ -88,7 +89,7 @@ func (i *InputDefinition) loadMeta() error { i.frames = append(i.frames, inputFrame) } - for _, field := range pb.InputDefinitionFields { + for _, field := range pb.Fields { var actions []Action for _, action := range field.Actions { actions = append(actions, Action{ @@ -149,9 +150,9 @@ func (i *InputDefinition) saveMeta() error { fields = append(fields, fieldMeta) } buf, err := proto.Marshal(&internal.InputDefinition{ - Name: i.name, - Frames: frames, - InputDefinitionFields: fields, + Name: i.name, + Frames: frames, + Fields: fields, }) if err != nil { return err diff --git a/internal/private.pb.go b/internal/private.pb.go index a98f25f5b..27ead21f1 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -247,9 +247,9 @@ func (m *Index) GetInputDefinitions() []*InputDefinition { } type InputDefinition struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Frames []*Frame `protobuf:"bytes,2,rep,name=Frames" json:"Frames,omitempty"` - InputDefinitionFields []*InputDefinitionField `protobuf:"bytes,3,rep,name=InputDefinitionFields" json:"InputDefinitionFields,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Frames []*Frame `protobuf:"bytes,2,rep,name=Frames" json:"Frames,omitempty"` + Fields []*InputDefinitionField `protobuf:"bytes,3,rep,name=Fields" json:"Fields,omitempty"` } func (m *InputDefinition) Reset() { *m = InputDefinition{} } @@ -264,9 +264,9 @@ func (m *InputDefinition) GetFrames() []*Frame { return nil } -func (m *InputDefinition) GetInputDefinitionFields() []*InputDefinitionField { +func (m *InputDefinition) GetFields() []*InputDefinitionField { if m != nil { - return m.InputDefinitionFields + return m.Fields } return nil } @@ -970,8 +970,8 @@ func (m *InputDefinition) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.InputDefinitionFields) > 0 { - for _, msg := range m.InputDefinitionFields { + if len(m.Fields) > 0 { + for _, msg := range m.Fields { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1475,8 +1475,8 @@ func (m *InputDefinition) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if len(m.InputDefinitionFields) > 0 { - for _, e := range m.InputDefinitionFields { + if len(m.Fields) > 0 { + for _, e := range m.Fields { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -3587,7 +3587,7 @@ func (m *InputDefinition) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionFields", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3611,8 +3611,8 @@ func (m *InputDefinition) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.InputDefinitionFields = append(m.InputDefinitionFields, &InputDefinitionField{}) - if err := m.InputDefinitionFields[len(m.InputDefinitionFields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Fields = append(m.Fields, &InputDefinitionField{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4469,58 +4469,58 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 835 bytes of a gzipped FileDescriptorProto + // 834 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x41, 0x93, 0xd5, 0xca, 0x5a, 0xf5, - 0x81, 0x2c, 0x3e, 0xec, 0x21, 0x5c, 0x20, 0xe1, 0x00, 0xb1, 0x1d, 0xc5, 0x02, 0x07, 0x68, 0xaf, - 0x72, 0x44, 0xea, 0xb5, 0x0b, 0x18, 0xed, 0x78, 0xc6, 0x4c, 0xf7, 0x78, 0x63, 0x0e, 0x5c, 0x78, - 0x09, 0x24, 0xae, 0x5c, 0x79, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, 0x9b, - 0x19, 0x8f, 0x7f, 0xa2, 0xe4, 0xd6, 0xf5, 0x75, 0xfd, 0x7c, 0xfd, 0x4d, 0x55, 0xd9, 0x70, 0xb8, - 0x48, 0xc3, 0x25, 0x97, 0x78, 0xb9, 0x48, 0x13, 0x99, 0x90, 0x56, 0x18, 0x4b, 0x4c, 0x63, 0x1e, - 0xd1, 0xaf, 0xa0, 0x3d, 0x8a, 0x67, 0xf8, 0x6a, 0x8c, 0x92, 0x93, 0x73, 0x38, 0xe8, 0x27, 0x51, - 0x36, 0x8f, 0xbf, 0xe4, 0xd7, 0x18, 0x05, 0xde, 0xb9, 0x77, 0xd1, 0x66, 0x65, 0x48, 0x79, 0x5c, - 0x85, 0x73, 0xfc, 0x26, 0xe3, 0xb1, 0xcc, 0xe6, 0x41, 0xcd, 0x78, 0x94, 0x20, 0xfa, 0x87, 0x07, - 0xed, 0x67, 0x29, 0x9f, 0xa3, 0xce, 0x78, 0x0a, 0x2d, 0x96, 0xdc, 0x96, 0xd3, 0xe5, 0x36, 0xf9, - 0x00, 0x8e, 0x46, 0xf1, 0x12, 0x53, 0x81, 0xc3, 0x98, 0x5f, 0x47, 0x38, 0xd3, 0xe9, 0x5a, 0xac, - 0x82, 0x92, 0x33, 0x68, 0xf7, 0xf9, 0xf4, 0x07, 0xbc, 0x5a, 0x2d, 0x30, 0xa8, 0xeb, 0x24, 0x05, - 0x90, 0xdf, 0x4e, 0xc2, 0x9f, 0x30, 0xf0, 0xcf, 0xbd, 0x8b, 0x43, 0x56, 0x00, 0x55, 0xbe, 0x8d, - 0x4d, 0xbe, 0x14, 0x8e, 0x46, 0xf3, 0x45, 0x92, 0x4a, 0x86, 0x62, 0x91, 0xc4, 0x02, 0x49, 0x07, - 0xea, 0xc3, 0x34, 0xb5, 0x74, 0xd5, 0x91, 0xfe, 0x0c, 0x9d, 0xa7, 0x51, 0x32, 0xbd, 0x19, 0x70, - 0xc9, 0x19, 0xfe, 0x98, 0xa1, 0x90, 0xe4, 0x04, 0x1a, 0x5a, 0x38, 0xeb, 0x67, 0x0c, 0x85, 0xea, - 0xc7, 0x5b, 0x65, 0x8c, 0xa1, 0x50, 0x1d, 0xaf, 0xd9, 0xfb, 0xcc, 0x18, 0x0a, 0x9d, 0x44, 0xe1, - 0xd4, 0xb0, 0xf6, 0x99, 0x31, 0x08, 0x01, 0xff, 0x65, 0x88, 0xb7, 0x96, 0xaa, 0x3e, 0xd3, 0x11, - 0xdc, 0x2f, 0xd5, 0xb7, 0x34, 0xdf, 0x87, 0x26, 0x4b, 0x6e, 0x47, 0x03, 0x11, 0x78, 0xe7, 0xf5, - 0x0b, 0x9f, 0x59, 0x4b, 0x0b, 0xa2, 0xbf, 0x98, 0xba, 0xaa, 0xe9, 0xab, 0x02, 0xa0, 0x0f, 0xa0, - 0xa1, 0xd5, 0x51, 0xaf, 0x2c, 0x62, 0xd5, 0x91, 0xfe, 0xe6, 0xc1, 0xfd, 0x31, 0x7f, 0xa5, 0x69, - 0x88, 0xbc, 0xcc, 0x73, 0x68, 0xe7, 0xa0, 0xf6, 0x3e, 0x78, 0xd4, 0xbb, 0x74, 0xed, 0x73, 0xb9, - 0xe1, 0x5f, 0x20, 0xc3, 0x58, 0xa6, 0x2b, 0x56, 0x04, 0x9f, 0x7e, 0x0a, 0x47, 0xeb, 0x97, 0x8a, - 0xc3, 0x0d, 0xae, 0x9c, 0xd2, 0x37, 0xb8, 0x52, 0x9a, 0x2c, 0x79, 0x94, 0x19, 0xfd, 0x7c, 0x66, - 0x8c, 0xc7, 0xb5, 0x8f, 0x3d, 0xfa, 0x2d, 0x90, 0x7e, 0x8a, 0x5c, 0xa2, 0x4e, 0x30, 0x46, 0x21, - 0xf8, 0xf7, 0xb8, 0xfb, 0x2b, 0x18, 0x65, 0x6b, 0x65, 0x65, 0xcf, 0xa0, 0x3d, 0x12, 0xb6, 0xb7, - 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x03, 0x8c, 0x50, 0xa2, 0x1d, 0x87, 0x3d, 0xf9, - 0xe9, 0xc4, 0x71, 0x79, 0xbd, 0x2f, 0x79, 0x08, 0xbe, 0x9a, 0x04, 0x4d, 0xe5, 0xe0, 0xd1, 0xbb, - 0x85, 0x74, 0xf9, 0xd8, 0x31, 0xed, 0x40, 0x43, 0x97, 0xd4, 0x4e, 0xcf, 0x6b, 0x1e, 0xb8, 0xa5, - 0xcd, 0x5c, 0xa9, 0x7a, 0xb5, 0x54, 0x3e, 0x8f, 0xb6, 0xd4, 0x67, 0xee, 0xad, 0x6f, 0x5b, 0x8a, - 0x0e, 0x2c, 0xaa, 0xda, 0xf5, 0x85, 0xba, 0x35, 0x31, 0xfa, 0xbc, 0xfb, 0xc9, 0x55, 0x1e, 0xff, - 0x79, 0xb6, 0xe4, 0x9b, 0xa5, 0xa9, 0x28, 0xa7, 0x96, 0x8c, 0x6b, 0x2c, 0x3b, 0x61, 0xb9, 0x4d, - 0x1e, 0x42, 0x53, 0x57, 0x15, 0x81, 0xaf, 0x7b, 0xf7, 0xb8, 0xc2, 0x86, 0xd9, 0x6b, 0x35, 0x4e, - 0xb6, 0xc9, 0x1b, 0x66, 0x9c, 0x8c, 0x45, 0x86, 0xd0, 0x19, 0xc5, 0x8b, 0x4c, 0x0e, 0xf0, 0xbb, - 0x30, 0x0e, 0x65, 0x98, 0xc4, 0x22, 0x68, 0xea, 0x54, 0x0f, 0xca, 0x8c, 0xd6, 0x3c, 0xd8, 0x46, - 0x08, 0xfd, 0xdd, 0x83, 0xe3, 0x0a, 0xb8, 0xe3, 0xd1, 0x8e, 0x6f, 0x6d, 0x3f, 0xdf, 0x2b, 0x78, - 0xaf, 0x92, 0xef, 0x59, 0x88, 0xd1, 0x4c, 0x04, 0x75, 0x1d, 0xd7, 0xdd, 0x49, 0x4e, 0xbb, 0xb1, - 0xed, 0xc1, 0x74, 0x09, 0x27, 0xdb, 0x2e, 0xb6, 0x52, 0xed, 0x02, 0x7c, 0x9d, 0x86, 0x73, 0x9e, - 0xae, 0xbe, 0xc0, 0x95, 0xdd, 0xdd, 0x25, 0x84, 0xf4, 0xe0, 0xde, 0xe7, 0x53, 0x23, 0x98, 0xe1, - 0xd4, 0x29, 0x38, 0x99, 0x0b, 0xe6, 0x1c, 0xe8, 0xdf, 0x1e, 0x34, 0xcd, 0xb9, 0x68, 0x38, 0xaf, - 0xdc, 0xdb, 0x3d, 0xe8, 0xbc, 0x54, 0xbb, 0x60, 0x80, 0x42, 0x86, 0x31, 0x57, 0x9e, 0xb6, 0x23, - 0x37, 0x70, 0xf2, 0x18, 0x5a, 0x1a, 0x1b, 0xf3, 0xc5, 0xa6, 0x1a, 0xa6, 0xca, 0xa5, 0x73, 0x30, - 0x5b, 0x2a, 0xf7, 0x57, 0xd5, 0xf5, 0x1e, 0x75, 0x4b, 0x59, 0x1b, 0xa7, 0x4f, 0xe0, 0x70, 0x2d, - 0xe0, 0x8d, 0x36, 0xd7, 0x2f, 0x1e, 0x9c, 0xb9, 0x75, 0xb1, 0x26, 0xed, 0xfe, 0xc1, 0x73, 0x92, - 0xd7, 0x4a, 0x92, 0x7f, 0x02, 0x50, 0x84, 0xdb, 0x39, 0xdf, 0xd3, 0x86, 0x25, 0x67, 0xca, 0x01, - 0x5e, 0x24, 0x33, 0x9c, 0x48, 0x2e, 0x33, 0xa1, 0x92, 0x3f, 0x4f, 0x84, 0x74, 0xdf, 0x53, 0x9d, - 0xf5, 0xd6, 0x94, 0x5c, 0xe6, 0x93, 0xae, 0x0d, 0xf2, 0x21, 0xdc, 0xd3, 0x7c, 0xd0, 0x7d, 0xc5, - 0xe3, 0xca, 0x20, 0x32, 0x77, 0x4f, 0x9f, 0xc0, 0x61, 0x3f, 0xca, 0x84, 0xc4, 0xd4, 0x56, 0xe9, - 0x41, 0x43, 0xd5, 0x74, 0xbf, 0x1b, 0x27, 0x45, 0x64, 0x41, 0x85, 0x19, 0x97, 0xa7, 0x9d, 0x3f, - 0xef, 0xba, 0xde, 0x5f, 0x77, 0x5d, 0xef, 0x9f, 0xbb, 0xae, 0xf7, 0xeb, 0xbf, 0xdd, 0x77, 0xae, - 0x9b, 0xfa, 0xbf, 0xca, 0x47, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0x2c, 0x40, 0x32, 0xbc, 0xbc, - 0x08, 0x00, 0x00, + 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x85, 0x26, 0xab, 0x95, 0xb5, 0xea, + 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0x51, 0x2c, 0x70, 0x80, 0x76, + 0x94, 0x23, 0x52, 0xaf, 0x5d, 0xc0, 0x68, 0xc7, 0x33, 0x66, 0xba, 0xc7, 0x1b, 0x73, 0xe0, 0x82, + 0xc4, 0x33, 0x20, 0xf1, 0x0c, 0xbc, 0x07, 0x47, 0xae, 0xdc, 0xd0, 0x72, 0xe1, 0x31, 0x50, 0xff, + 0xcd, 0x8c, 0xc7, 0x6b, 0x47, 0x9b, 0x5b, 0xd7, 0x57, 0x5f, 0x57, 0x7d, 0x5d, 0x53, 0x55, 0x36, + 0x1c, 0x2e, 0xd3, 0x70, 0xc5, 0x25, 0xf6, 0x97, 0x69, 0x22, 0x13, 0xd2, 0x0a, 0x63, 0x89, 0x69, + 0xcc, 0x23, 0xfa, 0x15, 0xb4, 0xc7, 0xf1, 0x1c, 0x5f, 0x4f, 0x50, 0x72, 0x72, 0x0e, 0x07, 0x83, + 0x24, 0xca, 0x16, 0xf1, 0x97, 0xfc, 0x12, 0xa3, 0xc0, 0x3b, 0xf7, 0x2e, 0xda, 0xac, 0x0c, 0x29, + 0xc6, 0xcb, 0x70, 0x81, 0xdf, 0x64, 0x3c, 0x96, 0xd9, 0x22, 0xa8, 0x19, 0x46, 0x09, 0xa2, 0x7f, + 0x78, 0xd0, 0x7e, 0x96, 0xf2, 0x05, 0xea, 0x88, 0xa7, 0xd0, 0x62, 0xc9, 0x75, 0x39, 0x5c, 0x6e, + 0x93, 0xf7, 0xe1, 0x68, 0x1c, 0xaf, 0x30, 0x15, 0x38, 0x8a, 0xf9, 0x65, 0x84, 0x73, 0x1d, 0xae, + 0xc5, 0x2a, 0x28, 0x39, 0x83, 0xf6, 0x80, 0xcf, 0x7e, 0xc0, 0x97, 0xeb, 0x25, 0x06, 0x75, 0x1d, + 0xa4, 0x00, 0x72, 0xef, 0x34, 0xfc, 0x09, 0x03, 0xff, 0xdc, 0xbb, 0x38, 0x64, 0x05, 0x50, 0xd5, + 0xdb, 0xd8, 0xd6, 0x4b, 0xe1, 0x68, 0xbc, 0x58, 0x26, 0xa9, 0x64, 0x28, 0x96, 0x49, 0x2c, 0x90, + 0x74, 0xa0, 0x3e, 0x4a, 0x53, 0x2b, 0x57, 0x1d, 0xe9, 0xcf, 0xd0, 0x79, 0x1a, 0x25, 0xb3, 0xab, + 0x21, 0x97, 0x9c, 0xe1, 0x8f, 0x19, 0x0a, 0x49, 0x4e, 0xa0, 0xa1, 0x0b, 0x67, 0x79, 0xc6, 0x50, + 0xa8, 0x7e, 0xbc, 0xad, 0x8c, 0x31, 0x14, 0xaa, 0xef, 0x6b, 0xf5, 0x3e, 0x33, 0x86, 0x42, 0xa7, + 0x51, 0x38, 0x33, 0xaa, 0x7d, 0x66, 0x0c, 0x42, 0xc0, 0x7f, 0x15, 0xe2, 0xb5, 0x95, 0xaa, 0xcf, + 0x74, 0x0c, 0xf7, 0x4b, 0xf9, 0xad, 0xcc, 0xf7, 0xa0, 0xc9, 0x92, 0xeb, 0xf1, 0x50, 0x04, 0xde, + 0x79, 0xfd, 0xc2, 0x67, 0xd6, 0xd2, 0x05, 0xd1, 0x5f, 0x4c, 0xb9, 0x6a, 0xda, 0x55, 0x00, 0xf4, + 0x01, 0x34, 0x74, 0x75, 0xd4, 0x2b, 0x8b, 0xbb, 0xea, 0x48, 0x7f, 0xf7, 0xe0, 0xfe, 0x84, 0xbf, + 0xd6, 0x32, 0x44, 0x9e, 0xe6, 0x39, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, + 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xa7, 0x70, 0xb4, 0xe9, + 0x54, 0x1a, 0xae, 0x70, 0xed, 0x2a, 0x7d, 0x85, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, + 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x63, 0x8f, 0x7e, 0x0b, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, + 0x50, 0x08, 0xfe, 0x3d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, + 0xb6, 0xb7, 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x43, 0x8c, 0x50, 0xa2, 0x1d, 0x87, + 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x33, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, + 0xe8, 0xdd, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x0d, 0x0f, + 0xbc, 0xa5, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x67, 0xee, 0xad, 0x6f, + 0x9b, 0x8a, 0x0e, 0x2d, 0xaa, 0xda, 0xf5, 0x85, 0xf2, 0x9a, 0x3b, 0xfa, 0xbc, 0xfb, 0xc9, 0x55, + 0x1d, 0xff, 0x79, 0x36, 0xe5, 0xdd, 0xc2, 0x54, 0x2a, 0xa7, 0x96, 0x8c, 0x6b, 0x2c, 0x3b, 0x61, + 0xb9, 0x4d, 0x1e, 0x42, 0x53, 0x67, 0x15, 0x81, 0xaf, 0x7b, 0xf7, 0xb8, 0xa2, 0x86, 0x59, 0xb7, + 0x1a, 0x27, 0xdb, 0xe4, 0x0d, 0x33, 0x4e, 0xc6, 0x22, 0x23, 0xe8, 0x8c, 0xe3, 0x65, 0x26, 0x87, + 0xf8, 0x5d, 0x18, 0x87, 0x32, 0x4c, 0x62, 0x11, 0x34, 0x75, 0xa8, 0x07, 0x65, 0x45, 0x1b, 0x0c, + 0xb6, 0x75, 0x85, 0xfe, 0xea, 0xc1, 0x71, 0x05, 0xdc, 0xf1, 0x68, 0xa7, 0xb7, 0xb6, 0x5f, 0xef, + 0x47, 0xd0, 0x7c, 0x16, 0x62, 0x34, 0x17, 0x41, 0x5d, 0x13, 0xbb, 0x3b, 0xd5, 0x68, 0x1a, 0xb3, + 0x6c, 0xba, 0x82, 0x93, 0xdb, 0xfc, 0xb7, 0x8a, 0xe9, 0x02, 0x7c, 0x9d, 0x86, 0x0b, 0x9e, 0xae, + 0xbf, 0xc0, 0xb5, 0xdd, 0xce, 0x25, 0x84, 0xf4, 0xe0, 0xde, 0xe7, 0x33, 0x53, 0x12, 0x23, 0xa2, + 0x53, 0x88, 0x30, 0x0e, 0xe6, 0x08, 0xf4, 0x6f, 0x0f, 0x9a, 0xe6, 0x5c, 0xb4, 0x94, 0x57, 0xee, + 0xde, 0x1e, 0x74, 0x5e, 0xa9, 0x69, 0x1f, 0xa2, 0x90, 0x61, 0xcc, 0x15, 0xd3, 0xf6, 0xdc, 0x16, + 0x4e, 0x1e, 0x43, 0x4b, 0x63, 0x13, 0xbe, 0xdc, 0x7e, 0xbe, 0xc9, 0xd2, 0x77, 0x04, 0xb3, 0x87, + 0x72, 0xbe, 0xca, 0xae, 0x37, 0xa5, 0x5b, 0xbb, 0xda, 0x38, 0x7d, 0x02, 0x87, 0x1b, 0x17, 0xee, + 0xb4, 0x9b, 0x7e, 0xf1, 0xe0, 0xcc, 0x2d, 0x84, 0x8d, 0xd2, 0xee, 0x1f, 0x2d, 0x57, 0xf2, 0x5a, + 0xa9, 0xe4, 0x9f, 0x00, 0x14, 0xd7, 0xed, 0x24, 0xef, 0x69, 0xb4, 0x12, 0x99, 0x72, 0x80, 0x17, + 0xc9, 0x1c, 0xa7, 0x92, 0xcb, 0x4c, 0xa8, 0xe0, 0xcf, 0x13, 0x21, 0xdd, 0xf7, 0x54, 0x67, 0xbd, + 0x17, 0x25, 0x97, 0xf9, 0x2c, 0x6b, 0x83, 0x7c, 0x00, 0xf7, 0xb4, 0x1e, 0x74, 0x5f, 0xf1, 0xb8, + 0x32, 0x6a, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, + 0xa8, 0x9c, 0xee, 0x97, 0xe1, 0xa4, 0xb8, 0x59, 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xe7, 0x4d, + 0xd7, 0xfb, 0xeb, 0xa6, 0xeb, 0xfd, 0x73, 0xd3, 0xf5, 0x7e, 0xfb, 0xb7, 0xfb, 0xce, 0x65, 0x53, + 0xff, 0x1b, 0xf9, 0xf0, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb0, 0xe6, 0xa2, 0xa7, 0x9e, 0x08, + 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index d0b60d366..9697f24ae 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -83,7 +83,7 @@ message Index { message InputDefinition { string Name = 1; repeated Frame Frames = 2; - repeated InputDefinitionField InputDefinitionFields = 3; + repeated InputDefinitionField Fields = 3; } message InputDefinitionField { From e72e3dc436d3f91441ef6f100b0c978d1a1b5660 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 15 Jun 2017 00:39:21 -0500 Subject: [PATCH 12/99] more handle tests --- handler.go | 15 +++++---- handler_test.go | 70 ++++++++++++++++++++++++++++++++++++---- index.go | 6 ++-- input_definition.go | 11 +------ input_definition_test.go | 39 +++++++++++++++++++++- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/handler.go b/handler.go index 299343dc5..464506ae8 100644 --- a/handler.go +++ b/handler.go @@ -1550,9 +1550,10 @@ func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { return } inputDef, _ := index.inputDefinitions[inputDefName] - inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} - if err := json.NewEncoder(w).Encode(getInputDefinitionResponse{ - InputDefinition: inputInfo, + //inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} + if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ + Frames: inputDef.frames, + Fields: inputDef.fields, }); err != nil { h.logger().Printf("write status response error: %s", err) } @@ -1577,6 +1578,10 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusInternalServerError) return } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } type InputFrame struct { @@ -1584,10 +1589,6 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -type getInputDefinitionResponse struct { - InputDefinition InputDefinitionInfo `json:"input-definition"` -} - type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` Fields []Field `json:"fields"` diff --git a/handler_test.go b/handler_test.go index 8c7f3a7e2..a89244d49 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1067,16 +1067,11 @@ func MustReadAll(r io.Reader) []byte { return buf } -// Ensure handler can delete a frame. +// Ensure handler can create a input definition. func TestHandler_CreateInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() inputBody := []byte(` { "frames":[{ @@ -1107,6 +1102,10 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } ] }`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -1114,3 +1113,62 @@ func TestHandler_CreateInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s", body) } } + +// Ensure handler can delete a input definition. +func TestHandler_DeleteInputDefinition(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } + + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if index.InputDefinition("test") != nil { + t.Fatalf("unexpected result: %s", index.InputDefinition("test")) + } +} + +// Return existing input definition +func TestHandler_GetInputDefinition(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } + + response := &pilosa.InputDefinitionInfo{Frames: inputDef.Frames(), Fields: inputDef.Fields()} + expect, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != string(expect)+"\n" { + t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) + } +} diff --git a/index.go b/index.go index 58bb8adaf..4dc40501b 100644 --- a/index.go +++ b/index.go @@ -662,14 +662,14 @@ func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) return f, nil } -// DeleteFrame removes a frame from the index. +// DeleteInputDefinition removes a input definition from the index. func (i *Index) DeleteInputDefinition(name string) error { i.mu.Lock() defer i.mu.Unlock() // Ignore if input definition doesn't exist. - f := i.inputDefinition(name) - if f == nil { + inputDef := i.inputDefinition(name) + if inputDef == nil { return nil } diff --git a/input_definition.go b/input_definition.go index 31a354a3b..baa8bf215 100644 --- a/input_definition.go +++ b/input_definition.go @@ -31,19 +31,10 @@ func NewInputDefinition(path, index, name string) (*InputDefinition, error) { }, nil } -// Name returns the name of input definition was initialized with. -func (i *InputDefinition) Name() string { return i.name } - -// Index returns the index name of the input definition was initialized with. -func (i *InputDefinition) Index() string { return i.index } - -// Path returns the path of the input definition was initialized with. -func (i *InputDefinition) Path() string { return i.path } - // Frames returns frames of the input definition was initialized with. func (i *InputDefinition) Frames() []InputFrame { return i.frames } -// Fields returns frames of the input definition was initialized with. +// Fields returns fields of the input definition was initialized with. func (i *InputDefinition) Fields() []Field { return i.fields } func (i *InputDefinition) Open() error { diff --git a/input_definition_test.go b/input_definition_test.go index 191b5d182..5fd78c60c 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -1 +1,38 @@ -package pilosa +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "github.com/pilosa/pilosa" + "testing" +) + +func TestInputDefinition_Open(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + if err != nil { + t.Fatal(err) + } + err = inputDef.Open() + if err != nil { + t.Fatal(err) + } +} From 64b51f3bc332bba93041a4332ab2c95516797013 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 15 Jun 2017 23:45:19 -0500 Subject: [PATCH 13/99] Use the internal.inputdefinition object for broadcasting to other nodes. Create encoding methods to translate the JSON request to create a definition. --- handler.go | 28 ++++++++-------- handler_test.go | 22 ++++++------ index.go | 31 +++++++++-------- index_test.go | 45 +++++++++++++------------ input_definition.go | 72 ++++++++++++++++++++++++++++++++++------ input_definition_test.go | 67 ++++++++++++++++++++++++++++++++++--- server.go | 20 ++++++++--- 7 files changed, 208 insertions(+), 77 deletions(-) diff --git a/handler.go b/handler.go index 464506ae8..bf8dbdd7e 100644 --- a/handler.go +++ b/handler.go @@ -1505,7 +1505,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { inputDefName := mux.Vars(r)["input-definition"] // Decode request. - var req internal.InputDefinition + var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1519,14 +1519,24 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { return } - // Add the name - req.Name = inputDefName + def := req.Encode() + def.Name = inputDefName + + // Create InputDefinition. + _, err = index.CreateInputDefinition(def) + if err == ErrInputDefinitionExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ Index: indexName, Name: inputDefName, - Definition: &req, + Definition: def, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) @@ -1584,14 +1594,4 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) } } -type InputFrame struct { - Name string `json:"name,omitempty"` - Options FrameOptions `json:"options,omitempty"` -} - -type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []Field `json:"fields"` -} - type postInputDefinitionResponse struct{} diff --git a/handler_test.go b/handler_test.go index a89244d49..45ca9f021 100644 --- a/handler_test.go +++ b/handler_test.go @@ -215,7 +215,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code, w.Body.String()) + t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) } @@ -1120,10 +1120,11 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -1138,7 +1139,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } else if index.InputDefinition("test") != nil { - t.Fatalf("unexpected result: %s", index.InputDefinition("test")) + t.Fatalf("unexpected result: %v", index.InputDefinition("test")) } } @@ -1148,10 +1149,11 @@ func TestHandler_GetInputDefinition(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } diff --git a/index.go b/index.go index 4dc40501b..581da85a4 100644 --- a/index.go +++ b/index.go @@ -613,24 +613,30 @@ type importData struct { } // CreateInputDefinition creates a new input definition. -func (i *Index) CreateInputDefinition(name string, frames []InputFrame, field []Field) (*InputDefinition, error) { - +func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { // Ensure input definition doesn't already exist. - if i.inputDefinitions[name] != nil { + if i.inputDefinitions[pb.Name] != nil { return nil, ErrInputDefinitionExists } - return i.createInputDefinition(name, frames, field) + return i.createInputDefinition(pb) } -func (i *Index) createInputDefinition(name string, frames []InputFrame, fields []Field) (*InputDefinition, error) { - if name == "" { +func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { + if pb.Name == "" { return nil, errors.New("input-definition name required") - } else if len(frames) == 0 || len(fields) == 0 { + } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { return nil, errors.New("frames and fields are required") } - for _, fr := range frames { - _, err := i.CreateFrame(fr.Name, fr.Options) + for _, fr := range pb.Frames { + opt := FrameOptions{ + RowLabel: fr.Meta.RowLabel, + InverseEnabled: fr.Meta.InverseEnabled, + CacheType: fr.Meta.CacheType, + CacheSize: fr.Meta.CacheSize, + TimeQuantum: TimeQuantum(fr.Meta.TimeQuantum), + } + _, err := i.CreateFrame(fr.Name, opt) if err == ErrFrameExists { continue } else if err != nil { @@ -639,17 +645,16 @@ func (i *Index) createInputDefinition(name string, frames []InputFrame, fields [ } // Initialize input definition. - inputDef, err := i.newInputDefinition(i.InputDefPath(), name) + inputDef, err := i.newInputDefinition(i.InputDefPath(), pb.Name) if err != nil { return nil, err } - inputDef.frames = frames - inputDef.fields = fields + inputDef.LoadDefinition(pb) if err = inputDef.saveMeta(); err != nil { return nil, err } - i.inputDefinitions[name] = inputDef + i.inputDefinitions[pb.Name] = inputDef return inputDef, nil } diff --git a/index_test.go b/index_test.go index 125be84c4..5d5e03ab3 100644 --- a/index_test.go +++ b/index_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/pilosa/pilosa" - "reflect" + "github.com/pilosa/pilosa/internal" ) // Ensure index can open and retrieve a frame. @@ -237,7 +237,7 @@ func TestIndex_InvalidName(t *testing.T) { } index, err := pilosa.NewIndex(path, "ABC") if index != nil { - t.Fatalf("unexpected index name %s", index) + t.Fatalf("unexpected index name %v", index) } } @@ -246,16 +246,17 @@ func TestIndex_CreateInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) - } else if inputDef.Frames()[0] != frames { - t.Fatalf("unexpected input definition frames", inputDef.Frames()) - } else if !reflect.DeepEqual(inputDef.Fields()[0], fields) { - t.Fatalf("unexpected input definition actions", inputDef.Fields()) + } else if inputDef.Frames()[0].Name != frames.Name { + t.Fatalf("unexpected input definition frames %v", inputDef.Frames()) + } else if inputDef.Fields()[0].Name != fields.Name { + t.Fatalf("unexpected input definition actions %v", inputDef.Fields()) } } @@ -264,14 +265,15 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } - _, err = index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + _, err = index.CreateInputDefinition(&def) if err != pilosa.ErrInputDefinitionExists { t.Fatal(err) } @@ -282,7 +284,8 @@ func TestIndex_CreateEmptyInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{}, []pilosa.Field{}) + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) if err.Error() != "frames and fields are required" { t.Fatal(err) } @@ -293,10 +296,11 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - _, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } else if index.InputDefinition("test") == nil { @@ -309,5 +313,4 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { } else if index.InputDefinition("test") != nil { t.Fatal("input definition isn't deleted") } - } diff --git a/input_definition.go b/input_definition.go index baa8bf215..d02d4813f 100644 --- a/input_definition.go +++ b/input_definition.go @@ -53,16 +53,8 @@ func (i *InputDefinition) Open() error { return nil } -func (i *InputDefinition) loadMeta() error { - var pb internal.InputDefinition - buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name)) - if err != nil { - return err - } else { - if err := proto.Unmarshal(buf, &pb); err != nil { - return err - } - } +// LoadDefinition loads the protobuf format of a defition +func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name for _, fr := range pb.Frames { @@ -98,9 +90,23 @@ func (i *InputDefinition) loadMeta() error { } i.fields = append(i.fields, inputField) } + return nil } +func (i *InputDefinition) loadMeta() error { + var pb internal.InputDefinition + buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name)) + if err != nil { + return err + } + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + + return i.LoadDefinition(&pb) +} + //saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { @@ -157,15 +163,61 @@ func (i *InputDefinition) saveMeta() error { return nil } +// Field descripes a single field mapping in the InputDefinition. type Field struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` Actions []Action `json:"actions,omitempty"` } +// Encode converts Field into its internal representation. +func (o *Field) Encode() *internal.InputDefinitionField { + field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { + field.Actions = append(field.Actions, action.Encode()) + } + return &field +} + +// Action descripes the mapping method for the field in the InputDefinition. type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` RowID uint64 `json:"rowID,omitempty"` } + +// Encode converts Action into its internal representation. +func (o *Action) Encode() *internal.Action { + return &internal.Action{ + Frame: o.Frame, + ValueDestination: o.ValueDestination, + ValueMap: o.ValueMap, + RowID: o.RowID, + } +} + +// InputFrame defines the frame used in the input definition. +type InputFrame struct { + Name string `json:"name,omitempty"` + Options FrameOptions `json:"options,omitempty"` +} + +// InputDefinitionInfo the json message format to create an InputDefinition. +type InputDefinitionInfo struct { + Frames []InputFrame `json:"frames"` + Fields []Field `json:"fields"` +} + +// Encode converts InputDefinitionInfo into its internal representation. +func (o *InputDefinitionInfo) Encode() *internal.InputDefinition { + var def internal.InputDefinition + for _, f := range o.Frames { + def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) + } + for _, f := range o.Fields { + def.Fields = append(def.Fields, f.Encode()) + } + + return &def +} diff --git a/input_definition_test.go b/input_definition_test.go index 5fd78c60c..037a97c28 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -15,8 +15,11 @@ package pilosa_test import ( - "github.com/pilosa/pilosa" + "encoding/json" "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" ) func TestInputDefinition_Open(t *testing.T) { @@ -24,10 +27,11 @@ func TestInputDefinition_Open(t *testing.T) { defer index.Close() // Create Input Definition. - frames := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} - action := pilosa.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} - fields := pilosa.Field{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} - inputDef, err := index.CreateInputDefinition("test", []pilosa.InputFrame{frames}, []pilosa.Field{fields}) + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -36,3 +40,56 @@ func TestInputDefinition_Open(t *testing.T) { t.Fatal(err) } } + +// Verify the InputDefinition Encoding to the internal format +func TestInputDefinition_Encoding(t *testing.T) { + inputBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": true, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "id", + "primaryKey": true + }, + { + "name": "cabType", + "actions": [ + { + "frame": "cab-type", + "valueDestination": "mapping", + "valueMap": { + "Green": 1, + "Yellow": 2 + } + } + ] + } + ] + }`) + var def pilosa.InputDefinitionInfo + err := json.Unmarshal(inputBody, &def) + if err != nil { + t.Fatal(err) + } + + internalDef := def.Encode() + + if internalDef.Frames[0].Name != "event-time" { + t.Fatalf("unexpected frame: %v", internalDef) + } else if internalDef.Frames[0].Meta.CacheType != "ranked" { + t.Fatalf("unexpected frame meta data: %v", internalDef) + } else if len(internalDef.Fields) != 2 { + t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) + } else if len(internalDef.Fields[1].Actions) != 1 { + t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions) + } else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" { + t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) + } +} diff --git a/server.go b/server.go index d9b262088..6903c216b 100644 --- a/server.go +++ b/server.go @@ -311,15 +311,27 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { CacheSize: obj.Meta.CacheSize, TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), } - _, err := idx.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } + err := s.createFrame(idx, obj.Frame, opt) + return err case *internal.DeleteFrameMessage: idx := s.Holder.Index(obj.Index) if err := idx.DeleteFrame(obj.Frame); err != nil { return err } + case *internal.CreateInputDefinitionMessage: + idx := s.Holder.Index(obj.Index) + if idx == nil { + return fmt.Errorf("Local Index not found: %s", obj.Index) + } + idx.CreateInputDefinition(obj.Definition) + } + return nil +} + +func (s *Server) createFrame(idx *Index, frame string, opt FrameOptions) error { + _, err := idx.CreateFrame(frame, opt) + if err != nil { + return err } return nil } From 8c8f863943fb8d27688d16c1dabe3433503a2524 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 16 Jun 2017 13:25:45 -0500 Subject: [PATCH 14/99] reverted the broadcaster handler for internal.CreateFrameMessage --- server.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/server.go b/server.go index 6903c216b..fe69a1457 100644 --- a/server.go +++ b/server.go @@ -311,8 +311,10 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { CacheSize: obj.Meta.CacheSize, TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), } - err := s.createFrame(idx, obj.Frame, opt) - return err + _, err := idx.CreateFrame(obj.Frame, opt) + if err != nil { + return err + } case *internal.DeleteFrameMessage: idx := s.Holder.Index(obj.Index) if err := idx.DeleteFrame(obj.Frame); err != nil { @@ -328,14 +330,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return nil } -func (s *Server) createFrame(idx *Index, frame string, opt FrameOptions) error { - _, err := idx.CreateFrame(frame, opt) - if err != nil { - return err - } - return nil -} - // LocalStatus returns the state of the local node as well as the // holder (indexes/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. From 428b94dd0575445493263b37b94963b0dabf308f Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 16 Jun 2017 14:06:46 -0500 Subject: [PATCH 15/99] Broadcast Delete InputDefinition Messages and handle on remote nodes. --- broadcast.go | 4 + handler.go | 11 +- internal/private.pb.go | 274 +++++++++++++++++++++++++++++++++-------- internal/private.proto | 5 + server.go | 6 + 5 files changed, 245 insertions(+), 55 deletions(-) diff --git a/broadcast.go b/broadcast.go index 491a48737..866b03d02 100644 --- a/broadcast.go +++ b/broadcast.go @@ -133,6 +133,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeDeleteFrame case *internal.CreateInputDefinitionMessage: typ = MessageTypeCreateInputDefinition + case *internal.DeleteInputDefinitionMessage: + typ = MessageTypeDeleteInputDefinition default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -161,6 +163,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.DeleteFrameMessage{} case MessageTypeCreateInputDefinition: m = &internal.CreateInputDefinitionMessage{} + case MessageTypeDeleteInputDefinition: + m = &internal.DeleteInputDefinitionMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/handler.go b/handler.go index bf8dbdd7e..230635632 100644 --- a/handler.go +++ b/handler.go @@ -894,7 +894,7 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() validQuery := validOptions(QueryRequest{}) - for key, _ := range q { + for key := range q { if _, ok := validQuery[key]; !ok { return nil, errors.New("invalid query params") } @@ -1589,6 +1589,15 @@ func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) return } + err := h.Broadcaster.SendSync( + &internal.DeleteInputDefinitionMessage{ + Index: indexName, + Name: inputDefName, + }) + if err != nil { + h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + } + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } diff --git a/internal/private.pb.go b/internal/private.pb.go index 27ead21f1..a59d44430 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -27,6 +27,7 @@ InputDefinitionField Action CreateInputDefinitionMessage + DeleteInputDefinitionMessage NodeStatus ClusterStatus */ @@ -328,6 +329,18 @@ func (m *CreateInputDefinitionMessage) GetDefinition() *InputDefinition { return nil } +type DeleteInputDefinitionMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` +} + +func (m *DeleteInputDefinitionMessage) Reset() { *m = DeleteInputDefinitionMessage{} } +func (m *DeleteInputDefinitionMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteInputDefinitionMessage) ProtoMessage() {} +func (*DeleteInputDefinitionMessage) Descriptor() ([]byte, []int) { + return fileDescriptorPrivate, []int{18} +} + type NodeStatus struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` @@ -337,7 +350,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeStatus) GetIndexes() []*Index { if m != nil { @@ -353,7 +366,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *ClusterStatus) GetNodes() []*NodeStatus { if m != nil { @@ -381,6 +394,7 @@ func init() { proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") proto.RegisterType((*Action)(nil), "internal.Action") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") + proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") } @@ -1122,6 +1136,36 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *DeleteInputDefinitionMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Name) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + return i, nil +} + func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1546,6 +1590,20 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { return n } +func (m *DeleteInputDefinitionMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func (m *NodeStatus) Size() (n int) { var l int _ = l @@ -4141,6 +4199,114 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { } return nil } +func (m *DeleteInputDefinitionMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteInputDefinitionMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteInputDefinitionMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4469,58 +4635,58 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 834 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xcd, 0x8e, 0x1b, 0x45, + // 842 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x8e, 0x1b, 0x45, 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x85, 0x26, 0xab, 0x95, 0xb5, 0xea, - 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0x51, 0x2c, 0x70, 0x80, 0x76, - 0x94, 0x23, 0x52, 0xaf, 0x5d, 0xc0, 0x68, 0xc7, 0x33, 0x66, 0xba, 0xc7, 0x1b, 0x73, 0xe0, 0x82, - 0xc4, 0x33, 0x20, 0xf1, 0x0c, 0xbc, 0x07, 0x47, 0xae, 0xdc, 0xd0, 0x72, 0xe1, 0x31, 0x50, 0xff, - 0xcd, 0x8c, 0xc7, 0x6b, 0x47, 0x9b, 0x5b, 0xd7, 0x57, 0x5f, 0x57, 0x7d, 0x5d, 0x53, 0x55, 0x36, - 0x1c, 0x2e, 0xd3, 0x70, 0xc5, 0x25, 0xf6, 0x97, 0x69, 0x22, 0x13, 0xd2, 0x0a, 0x63, 0x89, 0x69, - 0xcc, 0x23, 0xfa, 0x15, 0xb4, 0xc7, 0xf1, 0x1c, 0x5f, 0x4f, 0x50, 0x72, 0x72, 0x0e, 0x07, 0x83, - 0x24, 0xca, 0x16, 0xf1, 0x97, 0xfc, 0x12, 0xa3, 0xc0, 0x3b, 0xf7, 0x2e, 0xda, 0xac, 0x0c, 0x29, - 0xc6, 0xcb, 0x70, 0x81, 0xdf, 0x64, 0x3c, 0x96, 0xd9, 0x22, 0xa8, 0x19, 0x46, 0x09, 0xa2, 0x7f, - 0x78, 0xd0, 0x7e, 0x96, 0xf2, 0x05, 0xea, 0x88, 0xa7, 0xd0, 0x62, 0xc9, 0x75, 0x39, 0x5c, 0x6e, - 0x93, 0xf7, 0xe1, 0x68, 0x1c, 0xaf, 0x30, 0x15, 0x38, 0x8a, 0xf9, 0x65, 0x84, 0x73, 0x1d, 0xae, - 0xc5, 0x2a, 0x28, 0x39, 0x83, 0xf6, 0x80, 0xcf, 0x7e, 0xc0, 0x97, 0xeb, 0x25, 0x06, 0x75, 0x1d, - 0xa4, 0x00, 0x72, 0xef, 0x34, 0xfc, 0x09, 0x03, 0xff, 0xdc, 0xbb, 0x38, 0x64, 0x05, 0x50, 0xd5, - 0xdb, 0xd8, 0xd6, 0x4b, 0xe1, 0x68, 0xbc, 0x58, 0x26, 0xa9, 0x64, 0x28, 0x96, 0x49, 0x2c, 0x90, - 0x74, 0xa0, 0x3e, 0x4a, 0x53, 0x2b, 0x57, 0x1d, 0xe9, 0xcf, 0xd0, 0x79, 0x1a, 0x25, 0xb3, 0xab, - 0x21, 0x97, 0x9c, 0xe1, 0x8f, 0x19, 0x0a, 0x49, 0x4e, 0xa0, 0xa1, 0x0b, 0x67, 0x79, 0xc6, 0x50, - 0xa8, 0x7e, 0xbc, 0xad, 0x8c, 0x31, 0x14, 0xaa, 0xef, 0x6b, 0xf5, 0x3e, 0x33, 0x86, 0x42, 0xa7, - 0x51, 0x38, 0x33, 0xaa, 0x7d, 0x66, 0x0c, 0x42, 0xc0, 0x7f, 0x15, 0xe2, 0xb5, 0x95, 0xaa, 0xcf, - 0x74, 0x0c, 0xf7, 0x4b, 0xf9, 0xad, 0xcc, 0xf7, 0xa0, 0xc9, 0x92, 0xeb, 0xf1, 0x50, 0x04, 0xde, - 0x79, 0xfd, 0xc2, 0x67, 0xd6, 0xd2, 0x05, 0xd1, 0x5f, 0x4c, 0xb9, 0x6a, 0xda, 0x55, 0x00, 0xf4, - 0x01, 0x34, 0x74, 0x75, 0xd4, 0x2b, 0x8b, 0xbb, 0xea, 0x48, 0x7f, 0xf7, 0xe0, 0xfe, 0x84, 0xbf, - 0xd6, 0x32, 0x44, 0x9e, 0xe6, 0x39, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, - 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xa7, 0x70, 0xb4, 0xe9, - 0x54, 0x1a, 0xae, 0x70, 0xed, 0x2a, 0x7d, 0x85, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, - 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x63, 0x8f, 0x7e, 0x0b, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, - 0x50, 0x08, 0xfe, 0x3d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, + 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0xd1, 0x5a, 0xe0, 0x00, 0xed, + 0x28, 0x47, 0xa4, 0x5e, 0xbb, 0x80, 0xd1, 0x8e, 0x67, 0xcc, 0x74, 0x8f, 0x37, 0xe6, 0xc0, 0x05, + 0x89, 0x67, 0x40, 0xe2, 0x19, 0x78, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, + 0x9b, 0x19, 0x8f, 0xd7, 0x8e, 0x02, 0xb7, 0xae, 0xaf, 0xbe, 0xae, 0xfa, 0xba, 0xa6, 0xaa, 0x6c, + 0x38, 0x5c, 0xa6, 0xe1, 0x8a, 0x4b, 0xec, 0x2f, 0xd3, 0x44, 0x26, 0xa4, 0x15, 0xc6, 0x12, 0xd3, + 0x98, 0x47, 0xf4, 0x0b, 0x68, 0x8f, 0xe3, 0x39, 0xbe, 0x9a, 0xa0, 0xe4, 0xe4, 0x1c, 0x0e, 0x06, + 0x49, 0x94, 0x2d, 0xe2, 0xcf, 0xf9, 0x15, 0x46, 0x81, 0x77, 0xee, 0x5d, 0xb4, 0x59, 0x19, 0x52, + 0x8c, 0x17, 0xe1, 0x02, 0xbf, 0xca, 0x78, 0x2c, 0xb3, 0x45, 0x50, 0x33, 0x8c, 0x12, 0x44, 0x7f, + 0xf3, 0xa0, 0xfd, 0x2c, 0xe5, 0x0b, 0xd4, 0x11, 0x4f, 0xa1, 0xc5, 0x92, 0x9b, 0x72, 0xb8, 0xdc, + 0x26, 0xef, 0xc2, 0xd1, 0x38, 0x5e, 0x61, 0x2a, 0x70, 0x14, 0xf3, 0xab, 0x08, 0xe7, 0x3a, 0x5c, + 0x8b, 0x55, 0x50, 0x72, 0x06, 0xed, 0x01, 0x9f, 0x7d, 0x87, 0x2f, 0xd6, 0x4b, 0x0c, 0xea, 0x3a, + 0x48, 0x01, 0xe4, 0xde, 0x69, 0xf8, 0x03, 0x06, 0xfe, 0xb9, 0x77, 0x71, 0xc8, 0x0a, 0xa0, 0xaa, + 0xb7, 0xb1, 0xad, 0x97, 0xc2, 0xd1, 0x78, 0xb1, 0x4c, 0x52, 0xc9, 0x50, 0x2c, 0x93, 0x58, 0x20, + 0xe9, 0x40, 0x7d, 0x94, 0xa6, 0x56, 0xae, 0x3a, 0xd2, 0x1f, 0xa1, 0xf3, 0x34, 0x4a, 0x66, 0xd7, + 0x43, 0x2e, 0x39, 0xc3, 0xef, 0x33, 0x14, 0x92, 0x9c, 0x40, 0x43, 0x17, 0xce, 0xf2, 0x8c, 0xa1, + 0x50, 0xfd, 0x78, 0x5b, 0x19, 0x63, 0x28, 0x54, 0xdf, 0xd7, 0xea, 0x7d, 0x66, 0x0c, 0x85, 0x4e, + 0xa3, 0x70, 0x66, 0x54, 0xfb, 0xcc, 0x18, 0x84, 0x80, 0xff, 0x32, 0xc4, 0x1b, 0x2b, 0x55, 0x9f, + 0xe9, 0x18, 0xee, 0x97, 0xf2, 0x5b, 0x99, 0xef, 0x40, 0x93, 0x25, 0x37, 0xe3, 0xa1, 0x08, 0xbc, + 0xf3, 0xfa, 0x85, 0xcf, 0xac, 0xa5, 0x0b, 0xa2, 0xbf, 0x98, 0x72, 0xd5, 0xb4, 0xab, 0x00, 0xe8, + 0x03, 0x68, 0xe8, 0xea, 0xa8, 0x57, 0x16, 0x77, 0xd5, 0x91, 0xfe, 0xea, 0xc1, 0xfd, 0x09, 0x7f, + 0xa5, 0x65, 0x88, 0x3c, 0xcd, 0x25, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, + 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xc7, 0x70, 0xb4, 0xe9, + 0x54, 0x1a, 0xae, 0x71, 0xed, 0x2a, 0x7d, 0x8d, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, + 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x43, 0x8f, 0x7e, 0x0d, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, + 0x50, 0x08, 0xfe, 0x2d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, 0xb6, 0xb7, 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x43, 0x8c, 0x50, 0xa2, 0x1d, 0x87, - 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x33, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, - 0xe8, 0xdd, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x0d, 0x0f, - 0xbc, 0xa5, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x67, 0xee, 0xad, 0x6f, - 0x9b, 0x8a, 0x0e, 0x2d, 0xaa, 0xda, 0xf5, 0x85, 0xf2, 0x9a, 0x3b, 0xfa, 0xbc, 0xfb, 0xc9, 0x55, - 0x1d, 0xff, 0x79, 0x36, 0xe5, 0xdd, 0xc2, 0x54, 0x2a, 0xa7, 0x96, 0x8c, 0x6b, 0x2c, 0x3b, 0x61, - 0xb9, 0x4d, 0x1e, 0x42, 0x53, 0x67, 0x15, 0x81, 0xaf, 0x7b, 0xf7, 0xb8, 0xa2, 0x86, 0x59, 0xb7, - 0x1a, 0x27, 0xdb, 0xe4, 0x0d, 0x33, 0x4e, 0xc6, 0x22, 0x23, 0xe8, 0x8c, 0xe3, 0x65, 0x26, 0x87, - 0xf8, 0x5d, 0x18, 0x87, 0x32, 0x4c, 0x62, 0x11, 0x34, 0x75, 0xa8, 0x07, 0x65, 0x45, 0x1b, 0x0c, - 0xb6, 0x75, 0x85, 0xfe, 0xea, 0xc1, 0x71, 0x05, 0xdc, 0xf1, 0x68, 0xa7, 0xb7, 0xb6, 0x5f, 0xef, - 0x47, 0xd0, 0x7c, 0x16, 0x62, 0x34, 0x17, 0x41, 0x5d, 0x13, 0xbb, 0x3b, 0xd5, 0x68, 0x1a, 0xb3, - 0x6c, 0xba, 0x82, 0x93, 0xdb, 0xfc, 0xb7, 0x8a, 0xe9, 0x02, 0x7c, 0x9d, 0x86, 0x0b, 0x9e, 0xae, - 0xbf, 0xc0, 0xb5, 0xdd, 0xce, 0x25, 0x84, 0xf4, 0xe0, 0xde, 0xe7, 0x33, 0x53, 0x12, 0x23, 0xa2, - 0x53, 0x88, 0x30, 0x0e, 0xe6, 0x08, 0xf4, 0x6f, 0x0f, 0x9a, 0xe6, 0x5c, 0xb4, 0x94, 0x57, 0xee, - 0xde, 0x1e, 0x74, 0x5e, 0xa9, 0x69, 0x1f, 0xa2, 0x90, 0x61, 0xcc, 0x15, 0xd3, 0xf6, 0xdc, 0x16, - 0x4e, 0x1e, 0x43, 0x4b, 0x63, 0x13, 0xbe, 0xdc, 0x7e, 0xbe, 0xc9, 0xd2, 0x77, 0x04, 0xb3, 0x87, - 0x72, 0xbe, 0xca, 0xae, 0x37, 0xa5, 0x5b, 0xbb, 0xda, 0x38, 0x7d, 0x02, 0x87, 0x1b, 0x17, 0xee, - 0xb4, 0x9b, 0x7e, 0xf1, 0xe0, 0xcc, 0x2d, 0x84, 0x8d, 0xd2, 0xee, 0x1f, 0x2d, 0x57, 0xf2, 0x5a, - 0xa9, 0xe4, 0x9f, 0x00, 0x14, 0xd7, 0xed, 0x24, 0xef, 0x69, 0xb4, 0x12, 0x99, 0x72, 0x80, 0x17, - 0xc9, 0x1c, 0xa7, 0x92, 0xcb, 0x4c, 0xa8, 0xe0, 0xcf, 0x13, 0x21, 0xdd, 0xf7, 0x54, 0x67, 0xbd, - 0x17, 0x25, 0x97, 0xf9, 0x2c, 0x6b, 0x83, 0x7c, 0x00, 0xf7, 0xb4, 0x1e, 0x74, 0x5f, 0xf1, 0xb8, - 0x32, 0x6a, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, - 0xa8, 0x9c, 0xee, 0x97, 0xe1, 0xa4, 0xb8, 0x59, 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xe7, 0x4d, - 0xd7, 0xfb, 0xeb, 0xa6, 0xeb, 0xfd, 0x73, 0xd3, 0xf5, 0x7e, 0xfb, 0xb7, 0xfb, 0xce, 0x65, 0x53, - 0xff, 0x1b, 0xf9, 0xf0, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb0, 0xe6, 0xa2, 0xa7, 0x9e, 0x08, - 0x00, 0x00, + 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x3d, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, + 0xe8, 0xed, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x35, 0x0f, + 0xbc, 0xa3, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x27, 0xee, 0xad, 0xff, + 0x35, 0x15, 0x1d, 0x5a, 0x54, 0xb5, 0xeb, 0x73, 0xe5, 0x35, 0x77, 0xf4, 0x79, 0xf7, 0x93, 0xab, + 0x3a, 0xfe, 0xf1, 0x6c, 0xca, 0x37, 0x0b, 0x53, 0xa9, 0x9c, 0x5a, 0x32, 0xae, 0xb1, 0xec, 0x84, + 0xe5, 0x36, 0x79, 0x08, 0x4d, 0x9d, 0x55, 0x04, 0xbe, 0xee, 0xdd, 0xe3, 0x8a, 0x1a, 0x66, 0xdd, + 0x6a, 0x9c, 0x6c, 0x93, 0x37, 0xcc, 0x38, 0x19, 0x8b, 0x8c, 0xa0, 0x33, 0x8e, 0x97, 0x99, 0x1c, + 0xe2, 0x37, 0x61, 0x1c, 0xca, 0x30, 0x89, 0x45, 0xd0, 0xd4, 0xa1, 0x1e, 0x94, 0x15, 0x6d, 0x30, + 0xd8, 0xd6, 0x15, 0xfa, 0xb3, 0x07, 0xc7, 0x15, 0x70, 0xc7, 0xa3, 0x9d, 0xde, 0xda, 0x7e, 0xbd, + 0x1f, 0x40, 0xf3, 0x59, 0x88, 0xd1, 0x5c, 0x04, 0x75, 0x4d, 0xec, 0xee, 0x54, 0xa3, 0x69, 0xcc, + 0xb2, 0xe9, 0x0a, 0x4e, 0xee, 0xf2, 0xdf, 0x29, 0xa6, 0x0b, 0xf0, 0x65, 0x1a, 0x2e, 0x78, 0xba, + 0xfe, 0x0c, 0xd7, 0x76, 0x3b, 0x97, 0x10, 0xd2, 0x83, 0x7b, 0x9f, 0xce, 0x4c, 0x49, 0x8c, 0x88, + 0x4e, 0x21, 0xc2, 0x38, 0x98, 0x23, 0xd0, 0x3f, 0x3d, 0x68, 0x9a, 0x73, 0xd1, 0x52, 0x5e, 0xb9, + 0x7b, 0x7b, 0xd0, 0x79, 0xa9, 0xa6, 0x7d, 0x88, 0x42, 0x86, 0x31, 0x57, 0x4c, 0xdb, 0x73, 0x5b, + 0x38, 0x79, 0x0c, 0x2d, 0x8d, 0x4d, 0xf8, 0x72, 0xfb, 0xf9, 0x26, 0x4b, 0xdf, 0x11, 0xcc, 0x1e, + 0xca, 0xf9, 0x2a, 0xbb, 0xde, 0x94, 0x6e, 0xed, 0x6a, 0xe3, 0xf4, 0x09, 0x1c, 0x6e, 0x5c, 0x78, + 0xa3, 0xdd, 0xf4, 0x93, 0x07, 0x67, 0x6e, 0x21, 0x6c, 0x94, 0x76, 0xff, 0x68, 0xb9, 0x92, 0xd7, + 0x4a, 0x25, 0xff, 0x08, 0xa0, 0xb8, 0x6e, 0x27, 0x79, 0x4f, 0xa3, 0x95, 0xc8, 0xf4, 0x12, 0xce, + 0xdc, 0x06, 0xfb, 0x7f, 0x22, 0x28, 0x07, 0x78, 0x9e, 0xcc, 0x71, 0x2a, 0xb9, 0xcc, 0x84, 0x62, + 0x5c, 0x26, 0x42, 0xba, 0xce, 0x50, 0x67, 0xbd, 0x61, 0x25, 0x97, 0xf9, 0x56, 0xd0, 0x06, 0x79, + 0x0f, 0xee, 0xe9, 0xa0, 0xe8, 0xfa, 0xe1, 0xb8, 0x32, 0xb4, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, + 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, 0xa8, 0x9c, 0xee, 0x37, 0xe6, 0xa4, 0xb8, 0x59, + 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xfb, 0x6d, 0xd7, 0xfb, 0xe3, 0xb6, 0xeb, 0xfd, 0x75, 0xdb, + 0xf5, 0x7e, 0xf9, 0xbb, 0xfb, 0xd6, 0x55, 0x53, 0xff, 0xaf, 0x79, 0xff, 0xdf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x41, 0x4c, 0x6a, 0xbc, 0xe8, 0x08, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 9697f24ae..3ad12df7c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -106,6 +106,11 @@ message CreateInputDefinitionMessage { InputDefinition Definition = 3; } +message DeleteInputDefinitionMessage { + string Index = 1; + string Name = 2; +} + message NodeStatus { string Host = 1; string State = 2; diff --git a/server.go b/server.go index fe69a1457..9a4c5d4f1 100644 --- a/server.go +++ b/server.go @@ -326,6 +326,12 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return fmt.Errorf("Local Index not found: %s", obj.Index) } idx.CreateInputDefinition(obj.Definition) + case *internal.DeleteInputDefinitionMessage: + idx := s.Holder.Index(obj.Index) + err := idx.DeleteInputDefinition(obj.Name) + if err != nil { + return err + } } return nil } From 431fcbbe4b92b125937af6827b9130f55de4c97a Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 16 Jun 2017 14:13:36 -0500 Subject: [PATCH 16/99] Put the handler endpoints in alphabetical order. Made the InputDefinition handler function names more descriptive. --- handler.go | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/handler.go b/handler.go index bf8dbdd7e..539f06e33 100644 --- a/handler.go +++ b/handler.go @@ -94,20 +94,6 @@ func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") - router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") - router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET") @@ -117,6 +103,23 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST") router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") + router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") + router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") + router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") + router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") @@ -129,10 +132,6 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetDefinition).Methods("GET") - router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostDefinition).Methods("POST") - router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteDefinition).Methods("DELETE") - return router } @@ -1500,7 +1499,7 @@ func errorString(err error) string { return err.Error() } -func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { +func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1547,7 +1546,7 @@ func (h *Handler) handlePostDefinition(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { +func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1570,7 +1569,7 @@ func (h *Handler) handleGetDefinition(w http.ResponseWriter, r *http.Request) { } -func (h *Handler) handleDeleteDefinition(w http.ResponseWriter, r *http.Request) { +func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] From b7ac8eda71b95219763323493e20afbb96c2b465 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 16 Jun 2017 14:33:33 -0500 Subject: [PATCH 17/99] add comments --- input_definition.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/input_definition.go b/input_definition.go index d02d4813f..20c2a6f0b 100644 --- a/input_definition.go +++ b/input_definition.go @@ -9,6 +9,7 @@ import ( "github.com/pilosa/pilosa/internal" ) +// InputDefinition represents a container for the data input definition. type InputDefinition struct { name string path string @@ -18,6 +19,7 @@ type InputDefinition struct { fields []Field } +// NewInputDefinition returns a new instance of InputDefinition. func NewInputDefinition(path, index, name string) (*InputDefinition, error) { err := ValidateName(name) if err != nil { @@ -37,6 +39,7 @@ func (i *InputDefinition) Frames() []InputFrame { return i.frames } // Fields returns fields of the input definition was initialized with. func (i *InputDefinition) Fields() []Field { return i.fields } +// Open opens and initializes the InputDefinition from file. func (i *InputDefinition) Open() error { if err := func() error { if err := os.MkdirAll(i.path, 0777); err != nil { From 568320ec6deaccc6e628e55bfecbc6ace8062da6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 19 Jun 2017 11:37:01 -0500 Subject: [PATCH 18/99] fix reviews and add more test --- frame.go | 2 +- index.go | 63 ++++++++++++++++++++++----------------------- index_test.go | 22 ++++++++++++++++ input_definition.go | 28 +++++++++++++++++--- 4 files changed, 79 insertions(+), 36 deletions(-) diff --git a/frame.go b/frame.go index dbc26541e..f694c9e97 100644 --- a/frame.go +++ b/frame.go @@ -653,7 +653,7 @@ type FrameOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } -// Encode converts o into its internal representation. +// Encode converts i into its internal representation. func (o *FrameOptions) Encode() *internal.FrameMeta { return &internal.FrameMeta{ RowLabel: o.RowLabel, diff --git a/index.go b/index.go index 581da85a4..c0877f0d7 100644 --- a/index.go +++ b/index.go @@ -31,8 +31,8 @@ import ( // Default index settings. const ( - DefaultColumnLabel = "columnID" - InputDefinitionRepo = ".input-definitions" + DefaultColumnLabel = "columnID" + InputDefinitionDir = ".input-definitions" ) // Index represents a container for frames. @@ -176,7 +176,7 @@ func (i *Index) openFrames() error { } for _, fi := range fis { - if !fi.IsDir() || fi.Name() == InputDefinitionRepo { + if !fi.IsDir() || fi.Name() == InputDefinitionDir { continue } @@ -336,9 +336,9 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } -// InputDefPath returns the path to a inputdefinition in the index. -func (i *Index) InputDefPath() string { - return filepath.Join(i.path, InputDefinitionRepo) +// InputDefinitionPath returns the path to a inputdefinition in the index. +func (i *Index) InputDefinitionPath() string { + return filepath.Join(i.path, InputDefinitionDir) } // Frame returns a frame in the index by name. @@ -584,7 +584,7 @@ type IndexOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } -// Encode converts o into its internal representation. +// Encode converts i into its internal representation. func (o *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ ColumnLabel: o.ColumnLabel, @@ -645,7 +645,7 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin } // Initialize input definition. - inputDef, err := i.newInputDefinition(i.InputDefPath(), pb.Name) + inputDef, err := i.newInputDefinition(pb.Name) if err != nil { return nil, err } @@ -658,13 +658,13 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin return inputDef, nil } -func (i *Index) newInputDefinition(path, name string) (*InputDefinition, error) { - f, err := NewInputDefinition(path, i.name, name) +func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { + inputDef, err := NewInputDefinition(i.InputDefinitionPath(), i.name, name) if err != nil { return nil, err } - f.broadcaster = i.broadcaster - return f, nil + inputDef.broadcaster = i.broadcaster + return inputDef, nil } // DeleteInputDefinition removes a input definition from the index. @@ -679,7 +679,7 @@ func (i *Index) DeleteInputDefinition(name string) error { } // Delete input definition file. - if err := os.Remove(filepath.Join(i.InputDefPath(), name)); err != nil { + if err := os.Remove(filepath.Join(i.InputDefinitionPath(), name)); err != nil { return err } @@ -690,32 +690,31 @@ func (i *Index) DeleteInputDefinition(name string) error { // openInputDefinition opens and initializes the input definitions inside the index. func (i *Index) openInputDefinition() error { - inputDefs, err := os.Open(i.path) - if err != nil { + inputDef, err := os.Open(i.InputDefinitionPath()) + if os.IsNotExist(err) { + return nil + } else if err != nil { return err } - defer inputDefs.Close() + defer inputDef.Close() - fis, err := inputDefs.Readdir(0) - if err != nil { - return err - } - for _, fi := range fis { - if fi.Name() != InputDefinitionRepo { - continue - } - inputPath, err := os.Open(i.InputDefPath()) + inputFiles, err := inputDef.Readdir(0) + for _, file := range inputFiles { + input, err := i.newInputDefinition(file.Name()) if err != nil { return err } - inputFiles, err := inputPath.Readdir(0) - for _, file := range inputFiles { - input, err := i.newInputDefinition(i.InputDefPath(), file.Name()) - if err != nil { - return err + input.Open() + i.inputDefinitions[file.Name()] = input + + // Create frame if it doesn't exist + for _, fr := range input.frames { + _, err := i.CreateFrame(fr.Name, fr.Options) + if err == ErrFrameExists { + continue + } else if err != nil { + return nil } - input.Open() - i.inputDefinitions[file.Name()] = input } } diff --git a/index_test.go b/index_test.go index 5d5e03ab3..f948310ca 100644 --- a/index_test.go +++ b/index_test.go @@ -314,3 +314,25 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { t.Fatal("input definition isn't deleted") } } + +func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + input, err := index.CreateInputDefinition(&def) + if err != nil { + t.Fatal(err) + } + + input.AddFrame(pilosa.InputFrame{Name: "f1"}) + index.Reopen() + if index.Frame("f1") == nil { + t.Fatal("Frame does not created when open index") + } + +} diff --git a/input_definition.go b/input_definition.go index 20c2a6f0b..e6bb5e972 100644 --- a/input_definition.go +++ b/input_definition.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( @@ -213,14 +227,22 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (o *InputDefinitionInfo) Encode() *internal.InputDefinition { +func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { var def internal.InputDefinition - for _, f := range o.Frames { + for _, f := range i.Frames { def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) } - for _, f := range o.Fields { + for _, f := range i.Fields { def.Fields = append(def.Fields, f.Encode()) } return &def } + +func (i *InputDefinition) AddFrame(frame InputFrame) error { + i.frames = append(i.frames, frame) + if err := i.saveMeta(); err != nil { + return err + } + return nil +} From 3cecd74e2e79e82c1da322a96d4bbd3df512e65b Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 19 Jun 2017 11:42:42 -0500 Subject: [PATCH 19/99] revert comment line of frame.go --- frame.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame.go b/frame.go index f694c9e97..dbc26541e 100644 --- a/frame.go +++ b/frame.go @@ -653,7 +653,7 @@ type FrameOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } -// Encode converts i into its internal representation. +// Encode converts o into its internal representation. func (o *FrameOptions) Encode() *internal.FrameMeta { return &internal.FrameMeta{ RowLabel: o.RowLabel, From 23e7e42d99354e1b310673de00f167228d7572bd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 19 Jun 2017 13:18:20 -0500 Subject: [PATCH 20/99] Test create input definition across 2 servers --- server/server_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..a92da70a8 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -494,6 +494,29 @@ func TestMain_SendReceiveMessage(t *testing.T) { if maxSlices1["i"] != 2 { t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "id", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } } // availablePorts returns a slice of ports that can be used for testing. @@ -615,6 +638,15 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 From 6d12478e6bc9409eabb32c34151a03872407824e Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 20 Jun 2017 12:12:49 -0500 Subject: [PATCH 21/99] add validation for input definition fields --- input_definition.go | 50 ++++++++++++++++++++++++++++++++++++++++ input_definition_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/input_definition.go b/input_definition.go index e6bb5e972..cb15732ca 100644 --- a/input_definition.go +++ b/input_definition.go @@ -19,10 +19,14 @@ import ( "os" "path/filepath" + "errors" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) +var ValidValueDestination = []string{"map", "valueToRow", "stringToBool"} + // InputDefinition represents a container for the data input definition. type InputDefinition struct { name string @@ -89,9 +93,22 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } + numPrimaryKey := 0 + countRowID := make(map[uint64]bool) for _, field := range pb.Fields { var actions []Action for _, action := range field.Actions { + if err := i.ValidateAction(action); err != nil { + return err + } + if action.RowID != 0 { + _, ok := countRowID[action.RowID] + if !ok { + countRowID[action.RowID] = true + } else { + return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + } + } actions = append(actions, Action{ Frame: action.Frame, ValueDestination: action.ValueDestination, @@ -100,6 +117,14 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { }) } + if field.PrimaryKey { + numPrimaryKey += 1 + } + + if numPrimaryKey > 1 { + return errors.New("duplicate primaryKey with other field") + } + inputField := Field{ Name: field.Name, PrimaryKey: field.PrimaryKey, @@ -246,3 +271,28 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } return nil } + +func (i *InputDefinition) ValidateAction(action *internal.Action) error { + validValues := make(map[string]bool) + for _, val := range ValidValueDestination { + validValues[val] = true + } + if _, ok := validValues[action.ValueDestination]; !ok { + return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) + } + + switch action.ValueDestination { + case "map": + if len(action.ValueMap) == 0 { + return errors.New("valueMap required for map") + } + case "stringToBool": + if action.RowID == 0 { + return errors.New("rowID required for stringToBool") + } + default: + return nil + } + + return nil +} diff --git a/input_definition_test.go b/input_definition_test.go index 037a97c28..178b30ea7 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "strings" ) func TestInputDefinition_Open(t *testing.T) { @@ -93,3 +94,50 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) } } + +func TestInputDefinition_LoadDefinition(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create Input Definition. + input := pilosa.InputDefinition{} + frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} + action := internal.Action{Frame: "f", ValueDestination: "ValueToRow", ValueMap: map[string]uint64{"Green": 1}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err := input.LoadDefinition(def) + if !strings.Contains(err.Error(), "invalid ValueDestination") { + t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) + } + + action = internal.Action{Frame: "f", ValueDestination: "stringToBool", ValueMap: map[string]uint64{"Green": 1}} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "rowID required for stringToBool") { + t.Fatalf("Expected rowID required for stringToBool error, actual error: %s", err) + } + + action = internal.Action{Frame: "f", ValueDestination: "map", RowID: 100} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "valueMap required for map") { + t.Fatalf("Expected valueMap required for map error, actual error: %s", err) + } + + action = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 101} + field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { + t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) + } + + action1 = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "duplicate rowID with other field") { + t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) + } +} From e0bf37da40e7f08921a7cd61e814f44020cfbc86 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 20 Jun 2017 21:37:27 -0500 Subject: [PATCH 22/99] check same rowID for same frame in actions --- input_definition.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/input_definition.go b/input_definition.go index cb15732ca..f05ccb64a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -94,19 +94,19 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } numPrimaryKey := 0 - countRowID := make(map[uint64]bool) + countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 { - _, ok := countRowID[action.RowID] - if !ok { - countRowID[action.RowID] = true - } else { + if action.RowID != 0 && action.Frame != ""{ + val, ok := countRowID[action.Frame] + if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + } else { + countRowID[action.Frame] = action.RowID } } actions = append(actions, Action{ From d416d2c5d2f13c30e9089d3241e5a46e1c042ac5 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 12:41:52 -0500 Subject: [PATCH 23/99] JSON parser --- handler.go | 33 +++++++++++++++++++++++++++++++ index.go | 47 +++++++++++++++++++++++++++++++++++++++++++++ input_definition.go | 21 ++++++++++---------- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/handler.go b/handler.go index cefb60ec1..3eeb1ad12 100644 --- a/handler.go +++ b/handler.go @@ -115,6 +115,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") @@ -1510,6 +1511,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } + fmt.Println() // Find index. index := h.Holder.Index(indexName) @@ -1603,3 +1605,34 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req } type postInputDefinitionResponse struct{} + +func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + + + // Decode request. + var reqs []interface{} + err := json.NewDecoder(r.Body).Decode(&reqs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, req := range reqs { + definition := index.inputDefinition(inputDefName) + err = index.JSONParser(req.(map[string]interface{}), definition) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } +} + + diff --git a/index.go b/index.go index c0877f0d7..3c1cea61a 100644 --- a/index.go +++ b/index.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "reflect" ) // Default index settings. @@ -720,3 +721,49 @@ func (i *Index) openInputDefinition() error { } return nil } + + +func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { + fmt.Println(i.Frames) + for _, field := range inputDef.fields{ + if _, ok := req[field.Name]; !ok { + return fmt.Errorf("field not found") + } + for _, action := range field.Actions { + switch action.ValueDestination { + case "map": + err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "stringToBool": + fmt.Println("HERE") + err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "valueToRow": + err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + } + } + } + val := reflect.ValueOf(Action{}) + for i := 0; i < val.Type().NumField(); i++{ + fmt.Println(val.Type().Field(i).Type) + } + return nil +} +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ + return nil +} + +func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ + return nil +} + +func (i *Index) ValueToRow(frame, name string, value uint64) error{ + return nil +} diff --git a/input_definition.go b/input_definition.go index cb15732ca..c7d845903 100644 --- a/input_definition.go +++ b/input_definition.go @@ -94,19 +94,19 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { } numPrimaryKey := 0 - countRowID := make(map[uint64]bool) + countRowID := make(map[string]uint64) + var actions []Action for _, field := range pb.Fields { - var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 { - _, ok := countRowID[action.RowID] - if !ok { - countRowID[action.RowID] = true - } else { + if action.RowID != 0 && action.Frame != ""{ + val, ok := countRowID[action.Frame] + if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + } else { + countRowID[action.Frame] = action.RowID } } actions = append(actions, Action{ @@ -116,7 +116,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: action.RowID, }) } - + fmt.Println(actions) if field.PrimaryKey { numPrimaryKey += 1 } @@ -280,18 +280,17 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - + fmt.Println("ACTION", action.ValueDestination) switch action.ValueDestination { case "map": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } case "stringToBool": + fmt.Println("HERR", action.RowID) if action.RowID == 0 { return errors.New("rowID required for stringToBool") } - default: - return nil } return nil From 9fd1de8f2966f3ffad42416f5b35ac2dfda1439a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 14:38:04 -0500 Subject: [PATCH 24/99] handle nil rowID --- handler_test.go | 4 ++-- index.go | 4 +++- index_test.go | 8 ++++---- input_definition.go | 34 ++++++++++++++++++++-------------- input_definition_test.go | 21 ++++++++++++--------- 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/handler_test.go b/handler_test.go index 45ca9f021..1b8c914ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,7 +1121,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -1150,7 +1150,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) diff --git a/index.go b/index.go index c0877f0d7..c70c9eec4 100644 --- a/index.go +++ b/index.go @@ -650,7 +650,9 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin return nil, err } - inputDef.LoadDefinition(pb) + if err = inputDef.LoadDefinition(pb); err != nil { + return nil, err + } if err = inputDef.saveMeta(); err != nil { return nil, err } diff --git a/index_test.go b/index_test.go index f948310ca..f4cb534af 100644 --- a/index_test.go +++ b/index_test.go @@ -247,7 +247,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -266,7 +266,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -297,7 +297,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -321,7 +321,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) diff --git a/input_definition.go b/input_definition.go index f05ccb64a..10cfafd68 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"map", "valueToRow", "stringToBool"} +var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -101,10 +101,10 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != ""{ + if action.RowID != 0 && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { - return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) } else { countRowID[action.Frame] = action.RowID } @@ -113,7 +113,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: &action.RowID, }) } @@ -176,7 +176,7 @@ func (i *InputDefinition) saveMeta() error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: convert(action.RowID), } actions = append(actions, actionMeta) } @@ -226,7 +226,7 @@ type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` - RowID uint64 `json:"rowID,omitempty"` + RowID *uint64 `json:"rowID,omitempty"` } // Encode converts Action into its internal representation. @@ -235,10 +235,19 @@ func (o *Action) Encode() *internal.Action { Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, - RowID: o.RowID, + RowID: convert(o.RowID), } } +func convert(x *uint64) uint64 { + if x != nil { + return *x + } + var v int64 = -1 + var v2 uint64 = uint64(v) + return v2 +} + // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` @@ -282,17 +291,14 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { } switch action.ValueDestination { - case "map": + case "mapping": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "stringToBool": - if action.RowID == 0 { - return errors.New("rowID required for stringToBool") + case "single-row-boolean": + if int64(action.RowID) == -1 { + return errors.New("rowID required for single-row-boolean") } - default: - return nil } - return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 178b30ea7..d8b95d5ba 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,7 +29,7 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -102,7 +102,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "ValueToRow", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) @@ -110,22 +110,25 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} + encodeAction := act.Encode() + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "rowID required for stringToBool") { - t.Fatalf("Expected rowID required for stringToBool error, actual error: %s", err) + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "map", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 101} + action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -133,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) From 56c16e923ac3e0c7cb242fc5dd4e53c44edd726a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 14:53:01 -0500 Subject: [PATCH 25/99] add frame required in action --- input_definition.go | 3 +++ input_definition_test.go | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/input_definition.go b/input_definition.go index 10cfafd68..742139a4a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -282,6 +282,9 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } func (i *InputDefinition) ValidateAction(action *internal.Action) error { + if action.Frame == "" { + return ErrFrameRequired + } validValues := make(map[string]bool) for _, val := range ValidValueDestination { validValues[val] = true diff --git a/input_definition_test.go b/input_definition_test.go index d8b95d5ba..9a1c4a5a4 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -143,4 +143,11 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } + + action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "frame required") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } } From ed1136c466d58f796ccaba02455d42dad486c9e6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 15:10:05 -0500 Subject: [PATCH 26/99] update definition validation --- handler_test.go | 4 ++-- index.go | 4 +++- index_test.go | 8 ++++---- input_definition.go | 40 ++++++++++++++++++++++++---------------- input_definition_test.go | 28 +++++++++++++++++++--------- 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/handler_test.go b/handler_test.go index 45ca9f021..1b8c914ad 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,7 +1121,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -1150,7 +1150,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) diff --git a/index.go b/index.go index 3c1cea61a..83ca3762f 100644 --- a/index.go +++ b/index.go @@ -651,7 +651,9 @@ func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefin return nil, err } - inputDef.LoadDefinition(pb) + if err = inputDef.LoadDefinition(pb); err != nil { + return nil, err + } if err = inputDef.saveMeta(); err != nil { return nil, err } diff --git a/index_test.go b/index_test.go index f948310ca..f4cb534af 100644 --- a/index_test.go +++ b/index_test.go @@ -247,7 +247,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -266,7 +266,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -297,7 +297,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) @@ -321,7 +321,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) diff --git a/input_definition.go b/input_definition.go index c7d845903..9d0304125 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"map", "valueToRow", "stringToBool"} +var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -95,16 +95,16 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { numPrimaryKey := 0 countRowID := make(map[string]uint64) - var actions []Action for _, field := range pb.Fields { + var actions []Action for _, action := range field.Actions { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != ""{ + if action.RowID != 0 && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { - return fmt.Errorf("duplicate rowID with other field: %s", action.RowID) + return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) } else { countRowID[action.Frame] = action.RowID } @@ -113,10 +113,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: &action.RowID, }) } - fmt.Println(actions) if field.PrimaryKey { numPrimaryKey += 1 } @@ -176,7 +175,7 @@ func (i *InputDefinition) saveMeta() error { Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, - RowID: action.RowID, + RowID: convert(action.RowID), } actions = append(actions, actionMeta) } @@ -226,7 +225,7 @@ type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` ValueMap map[string]uint64 `json:"valueMap,omitempty"` - RowID uint64 `json:"rowID,omitempty"` + RowID *uint64 `json:"rowID,omitempty"` } // Encode converts Action into its internal representation. @@ -235,10 +234,19 @@ func (o *Action) Encode() *internal.Action { Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, - RowID: o.RowID, + RowID: convert(o.RowID), } } +func convert(x *uint64) uint64 { + if x != nil { + return *x + } + var v int64 = -1 + var v2 uint64 = uint64(v) + return v2 +} + // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` @@ -273,6 +281,9 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } func (i *InputDefinition) ValidateAction(action *internal.Action) error { + if action.Frame == "" { + return ErrFrameRequired + } validValues := make(map[string]bool) for _, val := range ValidValueDestination { validValues[val] = true @@ -280,18 +291,15 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - fmt.Println("ACTION", action.ValueDestination) switch action.ValueDestination { - case "map": + case "mapping": if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "stringToBool": - fmt.Println("HERR", action.RowID) - if action.RowID == 0 { - return errors.New("rowID required for stringToBool") + case "single-row-boolean": + if int64(action.RowID) == -1 { + return errors.New("rowID required for single-row-boolean") } } - return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 178b30ea7..9a1c4a5a4 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,7 +29,7 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "map", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) @@ -102,7 +102,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "ValueToRow", ValueMap: map[string]uint64{"Green": 1}} + action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) @@ -110,22 +110,25 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} + encodeAction := act.Encode() + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "rowID required for stringToBool") { - t.Fatalf("Expected rowID required for stringToBool error, actual error: %s", err) + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "map", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 101} + action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -133,11 +136,18 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "stringToBool", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } + + action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} + err = input.LoadDefinition(def) + if !strings.Contains(err.Error(), "frame required") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } } From d1d8c5b696ddb02e0565f6c12ba4038e24a0a9c9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 11:05:33 -0500 Subject: [PATCH 27/99] handle actions in JSONparser --- handler.go | 17 ++++++---- index.go | 71 ++++++++++++++++++++++++++-------------- input_definition.go | 46 ++++++++++++++++---------- input_definition_test.go | 22 ++++++------- pilosa.go | 14 ++++---- 5 files changed, 105 insertions(+), 65 deletions(-) diff --git a/handler.go b/handler.go index 3eeb1ad12..5dd2347e8 100644 --- a/handler.go +++ b/handler.go @@ -1520,7 +1520,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - def := req.Encode() + def, err := req.Encode() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } def.Name = inputDefName // Create InputDefinition. @@ -1617,7 +1621,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1626,13 +1629,13 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - definition := index.inputDefinition(inputDefName) - err = index.JSONParser(req.(map[string]interface{}), definition) - if err != nil { + err = index.JSONParser(req.(map[string]interface{}), inputDefName) + if err == ErrInputDefinitionNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } } - - diff --git a/index.go b/index.go index 83ca3762f..1114be5d0 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "reflect" ) // Default index settings. @@ -724,48 +723,72 @@ func (i *Index) openInputDefinition() error { return nil } - -func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { - fmt.Println(i.Frames) - for _, field := range inputDef.fields{ - if _, ok := req[field.Name]; !ok { - return fmt.Errorf("field not found") +func (i *Index) JSONParser(req map[string]interface{}, name string) error { + inputDef := i.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound + } + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + } + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } + } + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + for _, action := range field.Actions { switch action.ValueDestination { - case "map": + case "mapping": + value, ok := req[field.Name].(string) + if !ok { + return fmt.Errorf("String type required, got %s:%s", field.Name, value) + } err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action mapping : %s", err) } - case "stringToBool": - fmt.Println("HERE") - err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) - if err != nil { - return fmt.Errorf("Error map value: %s", err) + case "single-row-boolean": + value, ok := req[field.Name].(bool) + if !ok { + return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) } - case "valueToRow": - err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + err := i.StringRowBoolean(action.Frame, action.RowID, value) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action single-row-boolean : %s", err) + } + case "value-to-row": + value, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) + } + err := i.ValueToRow(action.Frame, field.Name, uint64(value)) + if err != nil { + return fmt.Errorf("Error action value-to-row: %s", err) } } } } - val := reflect.ValueOf(Action{}) - for i := 0; i < val.Type().NumField(); i++{ - fmt.Println(val.Type().Field(i).Type) - } + return nil } -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { return nil } -func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ +func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { return nil } -func (i *Index) ValueToRow(frame, name string, value uint64) error{ +func (i *Index) ValueToRow(frame, name string, value uint64) error { return nil } diff --git a/input_definition.go b/input_definition.go index 9d0304125..a24163ca6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,13 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} +const ( + Mapping = "mapping" + ValueToRow = "value-to-row" + SingleRowBool = "single-row-boolean" +) + +var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -212,12 +218,17 @@ type Field struct { } // Encode converts Field into its internal representation. -func (o *Field) Encode() *internal.InputDefinitionField { +func (o *Field) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { - field.Actions = append(field.Actions, action.Encode()) + actionEncode, err := action.Encode() + if err != nil { + return nil, err + } + field.Actions = append(field.Actions, actionEncode) } - return &field + return &field, nil } // Action descripes the mapping method for the field in the InputDefinition. @@ -229,22 +240,23 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() *internal.Action { +func (o *Action) Encode() (*internal.Action, error) { + if o.RowID == nil && o.ValueDestination == "single-row-boolean" { + return nil, errors.New("rowID required for single-row-boolean") + } return &internal.Action{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, RowID: convert(o.RowID), - } + }, nil } func convert(x *uint64) uint64 { if x != nil { return *x } - var v int64 = -1 - var v2 uint64 = uint64(v) - return v2 + return 0 } // InputFrame defines the frame used in the input definition. @@ -260,16 +272,20 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { +func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { var def internal.InputDefinition for _, f := range i.Frames { def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) } for _, f := range i.Fields { - def.Fields = append(def.Fields, f.Encode()) + fEncode, err := f.Encode() + if err != nil { + return nil, err + } + def.Fields = append(def.Fields, fEncode) } - return &def + return &def, nil } func (i *InputDefinition) AddFrame(frame InputFrame) error { @@ -292,14 +308,10 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } switch action.ValueDestination { - case "mapping": + case Mapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "single-row-boolean": - if int64(action.RowID) == -1 { - return errors.New("rowID required for single-row-boolean") - } } return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 9a1c4a5a4..160064e90 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -80,7 +80,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatal(err) } - internalDef := def.Encode() + internalDef, err := def.Encode() + if err != nil { + t.Fatal(err) + } if internalDef.Frames[0].Name != "event-time" { t.Fatalf("unexpected frame: %v", internalDef) @@ -110,16 +113,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} - encodeAction := act.Encode() - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -127,8 +127,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} + action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -136,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -144,7 +144,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { diff --git a/pilosa.go b/pilosa.go index 504df3b65..ff1ef148b 100644 --- a/pilosa.go +++ b/pilosa.go @@ -30,12 +30,14 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") + ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From 8a0609f1fd24bf22f648b0ff13625e4ea5f8b6e9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 11:11:36 -0500 Subject: [PATCH 28/99] handle nil rowID --- handler.go | 6 ++++- input_definition.go | 48 ++++++++++++++++++++++++---------------- input_definition_test.go | 22 +++++++++--------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/handler.go b/handler.go index cefb60ec1..9a9617994 100644 --- a/handler.go +++ b/handler.go @@ -1518,7 +1518,11 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - def := req.Encode() + def, err := req.Encode() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } def.Name = inputDefName // Create InputDefinition. diff --git a/input_definition.go b/input_definition.go index 742139a4a..a24163ca6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -25,7 +25,13 @@ import ( "github.com/pilosa/pilosa/internal" ) -var ValidValueDestination = []string{"mapping", "value-to-row", "single-row-boolean"} +const ( + Mapping = "mapping" + ValueToRow = "value-to-row" + SingleRowBool = "single-row-boolean" +) + +var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -116,7 +122,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: &action.RowID, }) } - if field.PrimaryKey { numPrimaryKey += 1 } @@ -213,12 +218,17 @@ type Field struct { } // Encode converts Field into its internal representation. -func (o *Field) Encode() *internal.InputDefinitionField { +func (o *Field) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} + for _, action := range o.Actions { - field.Actions = append(field.Actions, action.Encode()) + actionEncode, err := action.Encode() + if err != nil { + return nil, err + } + field.Actions = append(field.Actions, actionEncode) } - return &field + return &field, nil } // Action descripes the mapping method for the field in the InputDefinition. @@ -230,22 +240,23 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() *internal.Action { +func (o *Action) Encode() (*internal.Action, error) { + if o.RowID == nil && o.ValueDestination == "single-row-boolean" { + return nil, errors.New("rowID required for single-row-boolean") + } return &internal.Action{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, RowID: convert(o.RowID), - } + }, nil } func convert(x *uint64) uint64 { if x != nil { return *x } - var v int64 = -1 - var v2 uint64 = uint64(v) - return v2 + return 0 } // InputFrame defines the frame used in the input definition. @@ -261,16 +272,20 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { +func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { var def internal.InputDefinition for _, f := range i.Frames { def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) } for _, f := range i.Fields { - def.Fields = append(def.Fields, f.Encode()) + fEncode, err := f.Encode() + if err != nil { + return nil, err + } + def.Fields = append(def.Fields, fEncode) } - return &def + return &def, nil } func (i *InputDefinition) AddFrame(frame InputFrame) error { @@ -292,16 +307,11 @@ func (i *InputDefinition) ValidateAction(action *internal.Action) error { if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } - switch action.ValueDestination { - case "mapping": + case Mapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } - case "single-row-boolean": - if int64(action.RowID) == -1 { - return errors.New("rowID required for single-row-boolean") - } } return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 9a1c4a5a4..160064e90 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -80,7 +80,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatal(err) } - internalDef := def.Encode() + internalDef, err := def.Encode() + if err != nil { + t.Fatal(err) + } if internalDef.Frames[0].Name != "event-time" { t.Fatalf("unexpected frame: %v", internalDef) @@ -110,16 +113,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: "single-row-boolean", ValueMap: map[string]uint64{"Green": 1}} - encodeAction := act.Encode() - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{encodeAction}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "mapping", RowID: 100} + action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -127,8 +127,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 0} + action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -136,7 +136,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: "single-row-boolean", RowID: 100} + action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -144,7 +144,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: "single-row-boolean", RowID: 100} + action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { From 13b32b0b109e66bea051d3d368871375fcf68874 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 14:12:16 -0500 Subject: [PATCH 29/99] check SingleRowBool for duplicate rowID --- input_definition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_definition.go b/input_definition.go index a24163ca6..10179ca31 100644 --- a/input_definition.go +++ b/input_definition.go @@ -107,7 +107,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.RowID != 0 && action.Frame != "" { + if action.ValueDestination == SingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) From aa2cc13a9cb478b0a15b8376ad99627dc91e2766 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 14:55:29 -0500 Subject: [PATCH 30/99] change Action to InputDefinitionAction in proto --- handler_test.go | 8 +- index_test.go | 16 ++-- input_definition.go | 20 ++--- input_definition_test.go | 32 ++++---- internal/private.pb.go | 161 ++++++++++++++++++++------------------- internal/private.proto | 4 +- 6 files changed, 121 insertions(+), 120 deletions(-) diff --git a/handler_test.go b/handler_test.go index 1b8c914ad..35d57e912 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,8 +1121,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -1150,8 +1150,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/index_test.go b/index_test.go index f4cb534af..a1c265ee4 100644 --- a/index_test.go +++ b/index_test.go @@ -247,8 +247,8 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -266,8 +266,8 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -297,8 +297,8 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} _, err := index.CreateInputDefinition(&def) if err != nil { @@ -321,8 +321,8 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} input, err := index.CreateInputDefinition(&def) if err != nil { diff --git a/input_definition.go b/input_definition.go index 10179ca31..53f3caaa2 100644 --- a/input_definition.go +++ b/input_definition.go @@ -103,7 +103,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action - for _, action := range field.Actions { + for _, action := range field.InputDefinitionActions { if err := i.ValidateAction(action); err != nil { return err } @@ -175,9 +175,9 @@ func (i *InputDefinition) saveMeta() error { var fields []*internal.InputDefinitionField for _, field := range i.fields { - var actions []*internal.Action + var actions []*internal.InputDefinitionAction for _, action := range field.Actions { - actionMeta := &internal.Action{ + actionMeta := &internal.InputDefinitionAction{ Frame: action.Frame, ValueDestination: action.ValueDestination, ValueMap: action.ValueMap, @@ -187,9 +187,9 @@ func (i *InputDefinition) saveMeta() error { } fieldMeta := &internal.InputDefinitionField{ - Name: field.Name, - PrimaryKey: field.PrimaryKey, - Actions: actions, + Name: field.Name, + PrimaryKey: field.PrimaryKey, + InputDefinitionActions: actions, } fields = append(fields, fieldMeta) } @@ -226,7 +226,7 @@ func (o *Field) Encode() (*internal.InputDefinitionField, error) { if err != nil { return nil, err } - field.Actions = append(field.Actions, actionEncode) + field.InputDefinitionActions = append(field.InputDefinitionActions, actionEncode) } return &field, nil } @@ -240,11 +240,11 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() (*internal.Action, error) { +func (o *Action) Encode() (*internal.InputDefinitionAction, error) { if o.RowID == nil && o.ValueDestination == "single-row-boolean" { return nil, errors.New("rowID required for single-row-boolean") } - return &internal.Action{ + return &internal.InputDefinitionAction{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, @@ -296,7 +296,7 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -func (i *InputDefinition) ValidateAction(action *internal.Action) error { +func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } diff --git a/input_definition_test.go b/input_definition_test.go index 160064e90..aa6acd5f8 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -29,8 +29,8 @@ func TestInputDefinition_Open(t *testing.T) { // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} - fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} + fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -91,10 +91,10 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatalf("unexpected frame meta data: %v", internalDef) } else if len(internalDef.Fields) != 2 { t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields)) - } else if len(internalDef.Fields[1].Actions) != 1 { - t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].Actions) - } else if internalDef.Fields[1].Actions[0].ValueDestination != "mapping" { - t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].Actions[0]) + } else if len(internalDef.Fields[1].InputDefinitionActions) != 1 { + t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions) + } else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" { + t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0]) } } @@ -105,8 +105,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { // Create Input Definition. input := pilosa.InputDefinition{} frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} - field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} + field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err := input.LoadDefinition(def) if !strings.Contains(err.Error(), "invalid ValueDestination") { @@ -119,32 +119,32 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} + field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []*internal.Action{&action1}} + action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.Action{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { diff --git a/internal/private.pb.go b/internal/private.pb.go index a59d44430..d25558aff 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -25,7 +25,7 @@ Index InputDefinition InputDefinitionField - Action + InputDefinitionAction CreateInputDefinitionMessage DeleteInputDefinitionMessage NodeStatus @@ -273,9 +273,9 @@ func (m *InputDefinition) GetFields() []*InputDefinitionField { } type InputDefinitionField struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` - Actions []*Action `protobuf:"bytes,3,rep,name=Actions" json:"Actions,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + PrimaryKey bool `protobuf:"varint,2,opt,name=PrimaryKey,proto3" json:"PrimaryKey,omitempty"` + InputDefinitionActions []*InputDefinitionAction `protobuf:"bytes,3,rep,name=InputDefinitionActions" json:"InputDefinitionActions,omitempty"` } func (m *InputDefinitionField) Reset() { *m = InputDefinitionField{} } @@ -283,26 +283,26 @@ func (m *InputDefinitionField) String() string { return proto.Compact func (*InputDefinitionField) ProtoMessage() {} func (*InputDefinitionField) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } -func (m *InputDefinitionField) GetActions() []*Action { +func (m *InputDefinitionField) GetInputDefinitionActions() []*InputDefinitionAction { if m != nil { - return m.Actions + return m.InputDefinitionActions } return nil } -type Action struct { +type InputDefinitionAction struct { Frame string `protobuf:"bytes,1,opt,name=Frame,proto3" json:"Frame,omitempty"` ValueDestination string `protobuf:"bytes,2,opt,name=ValueDestination,proto3" json:"ValueDestination,omitempty"` ValueMap map[string]uint64 `protobuf:"bytes,3,rep,name=ValueMap" json:"ValueMap,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` RowID uint64 `protobuf:"varint,4,opt,name=RowID,proto3" json:"RowID,omitempty"` } -func (m *Action) Reset() { *m = Action{} } -func (m *Action) String() string { return proto.CompactTextString(m) } -func (*Action) ProtoMessage() {} -func (*Action) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (m *InputDefinitionAction) Reset() { *m = InputDefinitionAction{} } +func (m *InputDefinitionAction) String() string { return proto.CompactTextString(m) } +func (*InputDefinitionAction) ProtoMessage() {} +func (*InputDefinitionAction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } -func (m *Action) GetValueMap() map[string]uint64 { +func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { if m != nil { return m.ValueMap } @@ -392,7 +392,7 @@ func init() { proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*InputDefinition)(nil), "internal.InputDefinition") proto.RegisterType((*InputDefinitionField)(nil), "internal.InputDefinitionField") - proto.RegisterType((*Action)(nil), "internal.Action") + proto.RegisterType((*InputDefinitionAction)(nil), "internal.InputDefinitionAction") proto.RegisterType((*CreateInputDefinitionMessage)(nil), "internal.CreateInputDefinitionMessage") proto.RegisterType((*DeleteInputDefinitionMessage)(nil), "internal.DeleteInputDefinitionMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") @@ -1030,8 +1030,8 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { } i++ } - if len(m.Actions) > 0 { - for _, msg := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, msg := range m.InputDefinitionActions { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1045,7 +1045,7 @@ func (m *InputDefinitionField) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Action) Marshal() (dAtA []byte, err error) { +func (m *InputDefinitionAction) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1055,7 +1055,7 @@ func (m *Action) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Action) MarshalTo(dAtA []byte) (int, error) { +func (m *InputDefinitionAction) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1538,8 +1538,8 @@ func (m *InputDefinitionField) Size() (n int) { if m.PrimaryKey { n += 2 } - if len(m.Actions) > 0 { - for _, e := range m.Actions { + if len(m.InputDefinitionActions) > 0 { + for _, e := range m.InputDefinitionActions { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -1547,7 +1547,7 @@ func (m *InputDefinitionField) Size() (n int) { return n } -func (m *Action) Size() (n int) { +func (m *InputDefinitionAction) Size() (n int) { var l int _ = l l = len(m.Frame) @@ -3775,7 +3775,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { m.PrimaryKey = bool(v != 0) case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Actions", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field InputDefinitionActions", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3799,8 +3799,8 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Actions = append(m.Actions, &Action{}) - if err := m.Actions[len(m.Actions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.InputDefinitionActions = append(m.InputDefinitionActions, &InputDefinitionAction{}) + if err := m.InputDefinitionActions[len(m.InputDefinitionActions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3825,7 +3825,7 @@ func (m *InputDefinitionField) Unmarshal(dAtA []byte) error { } return nil } -func (m *Action) Unmarshal(dAtA []byte) error { +func (m *InputDefinitionAction) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3848,10 +3848,10 @@ func (m *Action) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Action: wiretype end group for non-group") + return fmt.Errorf("proto: InputDefinitionAction: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Action: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InputDefinitionAction: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4635,58 +4635,59 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 842 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x8e, 0x1b, 0x45, - 0x10, 0x66, 0xec, 0xb1, 0x63, 0xd7, 0x6a, 0x77, 0x9d, 0x66, 0x85, 0x26, 0xab, 0x95, 0xb5, 0xea, - 0x03, 0x59, 0x7c, 0xf0, 0x21, 0x48, 0x08, 0x12, 0x0e, 0x10, 0xdb, 0xd1, 0x5a, 0xe0, 0x00, 0xed, - 0x28, 0x47, 0xa4, 0x5e, 0xbb, 0x80, 0xd1, 0x8e, 0x67, 0xcc, 0x74, 0x8f, 0x37, 0xe6, 0xc0, 0x05, - 0x89, 0x67, 0x40, 0xe2, 0x19, 0x78, 0x0f, 0x8e, 0x5c, 0xb9, 0xa1, 0xe5, 0xc2, 0x63, 0xa0, 0xfe, - 0x9b, 0x19, 0x8f, 0xd7, 0x8e, 0x02, 0xb7, 0xae, 0xaf, 0xbe, 0xae, 0xfa, 0xba, 0xa6, 0xaa, 0x6c, - 0x38, 0x5c, 0xa6, 0xe1, 0x8a, 0x4b, 0xec, 0x2f, 0xd3, 0x44, 0x26, 0xa4, 0x15, 0xc6, 0x12, 0xd3, - 0x98, 0x47, 0xf4, 0x0b, 0x68, 0x8f, 0xe3, 0x39, 0xbe, 0x9a, 0xa0, 0xe4, 0xe4, 0x1c, 0x0e, 0x06, - 0x49, 0x94, 0x2d, 0xe2, 0xcf, 0xf9, 0x15, 0x46, 0x81, 0x77, 0xee, 0x5d, 0xb4, 0x59, 0x19, 0x52, - 0x8c, 0x17, 0xe1, 0x02, 0xbf, 0xca, 0x78, 0x2c, 0xb3, 0x45, 0x50, 0x33, 0x8c, 0x12, 0x44, 0x7f, - 0xf3, 0xa0, 0xfd, 0x2c, 0xe5, 0x0b, 0xd4, 0x11, 0x4f, 0xa1, 0xc5, 0x92, 0x9b, 0x72, 0xb8, 0xdc, - 0x26, 0xef, 0xc2, 0xd1, 0x38, 0x5e, 0x61, 0x2a, 0x70, 0x14, 0xf3, 0xab, 0x08, 0xe7, 0x3a, 0x5c, - 0x8b, 0x55, 0x50, 0x72, 0x06, 0xed, 0x01, 0x9f, 0x7d, 0x87, 0x2f, 0xd6, 0x4b, 0x0c, 0xea, 0x3a, - 0x48, 0x01, 0xe4, 0xde, 0x69, 0xf8, 0x03, 0x06, 0xfe, 0xb9, 0x77, 0x71, 0xc8, 0x0a, 0xa0, 0xaa, - 0xb7, 0xb1, 0xad, 0x97, 0xc2, 0xd1, 0x78, 0xb1, 0x4c, 0x52, 0xc9, 0x50, 0x2c, 0x93, 0x58, 0x20, - 0xe9, 0x40, 0x7d, 0x94, 0xa6, 0x56, 0xae, 0x3a, 0xd2, 0x1f, 0xa1, 0xf3, 0x34, 0x4a, 0x66, 0xd7, - 0x43, 0x2e, 0x39, 0xc3, 0xef, 0x33, 0x14, 0x92, 0x9c, 0x40, 0x43, 0x17, 0xce, 0xf2, 0x8c, 0xa1, - 0x50, 0xfd, 0x78, 0x5b, 0x19, 0x63, 0x28, 0x54, 0xdf, 0xd7, 0xea, 0x7d, 0x66, 0x0c, 0x85, 0x4e, - 0xa3, 0x70, 0x66, 0x54, 0xfb, 0xcc, 0x18, 0x84, 0x80, 0xff, 0x32, 0xc4, 0x1b, 0x2b, 0x55, 0x9f, - 0xe9, 0x18, 0xee, 0x97, 0xf2, 0x5b, 0x99, 0xef, 0x40, 0x93, 0x25, 0x37, 0xe3, 0xa1, 0x08, 0xbc, - 0xf3, 0xfa, 0x85, 0xcf, 0xac, 0xa5, 0x0b, 0xa2, 0xbf, 0x98, 0x72, 0xd5, 0xb4, 0xab, 0x00, 0xe8, - 0x03, 0x68, 0xe8, 0xea, 0xa8, 0x57, 0x16, 0x77, 0xd5, 0x91, 0xfe, 0xea, 0xc1, 0xfd, 0x09, 0x7f, - 0xa5, 0x65, 0x88, 0x3c, 0xcd, 0x25, 0xb4, 0x73, 0x50, 0xb3, 0x0f, 0x1e, 0xf5, 0xfa, 0xae, 0x7d, - 0xfa, 0x5b, 0xfc, 0x02, 0x19, 0xc5, 0x32, 0x5d, 0xb3, 0xe2, 0xf2, 0xe9, 0xc7, 0x70, 0xb4, 0xe9, - 0x54, 0x1a, 0xae, 0x71, 0xed, 0x2a, 0x7d, 0x8d, 0x6b, 0x55, 0x93, 0x15, 0x8f, 0x32, 0x53, 0x3f, - 0x9f, 0x19, 0xe3, 0x71, 0xed, 0x43, 0x8f, 0x7e, 0x0d, 0x64, 0x90, 0x22, 0x97, 0xa8, 0x03, 0x4c, - 0x50, 0x08, 0xfe, 0x2d, 0xee, 0xfe, 0x0a, 0xa6, 0xb2, 0xb5, 0x72, 0x65, 0xcf, 0xa0, 0x3d, 0x16, - 0xb6, 0xb7, 0xf4, 0x97, 0x68, 0xb1, 0x02, 0xa0, 0x3d, 0x20, 0x43, 0x8c, 0x50, 0xa2, 0x1d, 0x87, - 0x3d, 0xf1, 0xe9, 0xd4, 0x69, 0x79, 0x3d, 0x97, 0x3c, 0x04, 0x5f, 0x4d, 0x82, 0x96, 0x72, 0xf0, - 0xe8, 0xed, 0xa2, 0x74, 0xf9, 0xd8, 0x31, 0x4d, 0xa0, 0xa1, 0x0b, 0x6a, 0xa7, 0xe7, 0x35, 0x0f, - 0xbc, 0xa3, 0xcd, 0x5c, 0xaa, 0x7a, 0x35, 0x55, 0x3e, 0x8f, 0x36, 0xd5, 0x27, 0xee, 0xad, 0xff, - 0x35, 0x15, 0x1d, 0x5a, 0x54, 0xb5, 0xeb, 0x73, 0xe5, 0x35, 0x77, 0xf4, 0x79, 0xf7, 0x93, 0xab, - 0x3a, 0xfe, 0xf1, 0x6c, 0xca, 0x37, 0x0b, 0x53, 0xa9, 0x9c, 0x5a, 0x32, 0xae, 0xb1, 0xec, 0x84, - 0xe5, 0x36, 0x79, 0x08, 0x4d, 0x9d, 0x55, 0x04, 0xbe, 0xee, 0xdd, 0xe3, 0x8a, 0x1a, 0x66, 0xdd, - 0x6a, 0x9c, 0x6c, 0x93, 0x37, 0xcc, 0x38, 0x19, 0x8b, 0x8c, 0xa0, 0x33, 0x8e, 0x97, 0x99, 0x1c, - 0xe2, 0x37, 0x61, 0x1c, 0xca, 0x30, 0x89, 0x45, 0xd0, 0xd4, 0xa1, 0x1e, 0x94, 0x15, 0x6d, 0x30, - 0xd8, 0xd6, 0x15, 0xfa, 0xb3, 0x07, 0xc7, 0x15, 0x70, 0xc7, 0xa3, 0x9d, 0xde, 0xda, 0x7e, 0xbd, - 0x1f, 0x40, 0xf3, 0x59, 0x88, 0xd1, 0x5c, 0x04, 0x75, 0x4d, 0xec, 0xee, 0x54, 0xa3, 0x69, 0xcc, - 0xb2, 0xe9, 0x0a, 0x4e, 0xee, 0xf2, 0xdf, 0x29, 0xa6, 0x0b, 0xf0, 0x65, 0x1a, 0x2e, 0x78, 0xba, - 0xfe, 0x0c, 0xd7, 0x76, 0x3b, 0x97, 0x10, 0xd2, 0x83, 0x7b, 0x9f, 0xce, 0x4c, 0x49, 0x8c, 0x88, - 0x4e, 0x21, 0xc2, 0x38, 0x98, 0x23, 0xd0, 0x3f, 0x3d, 0x68, 0x9a, 0x73, 0xd1, 0x52, 0x5e, 0xb9, - 0x7b, 0x7b, 0xd0, 0x79, 0xa9, 0xa6, 0x7d, 0x88, 0x42, 0x86, 0x31, 0x57, 0x4c, 0xdb, 0x73, 0x5b, - 0x38, 0x79, 0x0c, 0x2d, 0x8d, 0x4d, 0xf8, 0x72, 0xfb, 0xf9, 0x26, 0x4b, 0xdf, 0x11, 0xcc, 0x1e, - 0xca, 0xf9, 0x2a, 0xbb, 0xde, 0x94, 0x6e, 0xed, 0x6a, 0xe3, 0xf4, 0x09, 0x1c, 0x6e, 0x5c, 0x78, - 0xa3, 0xdd, 0xf4, 0x93, 0x07, 0x67, 0x6e, 0x21, 0x6c, 0x94, 0x76, 0xff, 0x68, 0xb9, 0x92, 0xd7, - 0x4a, 0x25, 0xff, 0x08, 0xa0, 0xb8, 0x6e, 0x27, 0x79, 0x4f, 0xa3, 0x95, 0xc8, 0xf4, 0x12, 0xce, - 0xdc, 0x06, 0xfb, 0x7f, 0x22, 0x28, 0x07, 0x78, 0x9e, 0xcc, 0x71, 0x2a, 0xb9, 0xcc, 0x84, 0x62, - 0x5c, 0x26, 0x42, 0xba, 0xce, 0x50, 0x67, 0xbd, 0x61, 0x25, 0x97, 0xf9, 0x56, 0xd0, 0x06, 0x79, - 0x0f, 0xee, 0xe9, 0xa0, 0xe8, 0xfa, 0xe1, 0xb8, 0x32, 0xb4, 0xcc, 0xf9, 0xe9, 0x13, 0x38, 0x1c, - 0x44, 0x99, 0x90, 0x98, 0xda, 0x2c, 0x3d, 0x68, 0xa8, 0x9c, 0xee, 0x37, 0xe6, 0xa4, 0xb8, 0x59, - 0x48, 0x61, 0x86, 0xf2, 0xb4, 0xf3, 0xfb, 0x6d, 0xd7, 0xfb, 0xe3, 0xb6, 0xeb, 0xfd, 0x75, 0xdb, - 0xf5, 0x7e, 0xf9, 0xbb, 0xfb, 0xd6, 0x55, 0x53, 0xff, 0xaf, 0x79, 0xff, 0xdf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x41, 0x4c, 0x6a, 0xbc, 0xe8, 0x08, 0x00, 0x00, + // 851 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, + 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x4a, 0xe2, 0x6d, 0xc2, 0x6a, 0x36, 0x8a, 0x4c, 0xd4, 0x07, + 0x36, 0x44, 0x22, 0x87, 0x45, 0x42, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0xca, 0x08, 0xb2, 0x40, 0x67, + 0xb5, 0xdc, 0x90, 0x3a, 0x49, 0x01, 0xa3, 0x8c, 0x67, 0xcc, 0x74, 0x4f, 0xb2, 0xe6, 0xc0, 0x05, + 0x89, 0x67, 0x40, 0xe2, 0xc8, 0x99, 0xf7, 0xe0, 0xc8, 0x23, 0xa0, 0x70, 0xe1, 0x0d, 0xb8, 0xa2, + 0xfe, 0x9b, 0x19, 0x8f, 0x7f, 0xa2, 0xc0, 0x6d, 0xea, 0xeb, 0xaf, 0xaa, 0xbe, 0xae, 0xae, 0x2a, + 0x1b, 0xd6, 0x27, 0x79, 0x7c, 0x25, 0x14, 0x1e, 0x4c, 0xf2, 0x4c, 0x65, 0xb4, 0x17, 0xa7, 0x0a, + 0xf3, 0x54, 0x24, 0xec, 0x33, 0xe8, 0x47, 0xe9, 0x05, 0xbe, 0x3c, 0x41, 0x25, 0xe8, 0x2e, 0xac, + 0x1d, 0x66, 0x49, 0x31, 0x4e, 0x3f, 0x15, 0x67, 0x98, 0x84, 0x64, 0x97, 0xec, 0xf5, 0x79, 0x1d, + 0xd2, 0x8c, 0xe7, 0xf1, 0x18, 0xbf, 0x28, 0x44, 0xaa, 0x8a, 0x71, 0xd8, 0xb2, 0x8c, 0x1a, 0xc4, + 0x7e, 0x23, 0xd0, 0x7f, 0x9a, 0x8b, 0x31, 0x9a, 0x88, 0xdb, 0xd0, 0xe3, 0xd9, 0x75, 0x3d, 0x5c, + 0x69, 0xd3, 0x37, 0x61, 0x23, 0x4a, 0xaf, 0x30, 0x97, 0x78, 0x94, 0x8a, 0xb3, 0x04, 0x2f, 0x4c, + 0xb8, 0x1e, 0x6f, 0xa0, 0x74, 0x07, 0xfa, 0x87, 0xe2, 0xfc, 0x5b, 0x7c, 0x3e, 0x9d, 0x60, 0xd8, + 0x36, 0x41, 0x2a, 0xa0, 0x3c, 0x3d, 0x8d, 0xbf, 0xc7, 0x30, 0xd8, 0x25, 0x7b, 0xeb, 0xbc, 0x02, + 0x9a, 0x7a, 0x3b, 0xf3, 0x7a, 0x19, 0x6c, 0x44, 0xe3, 0x49, 0x96, 0x2b, 0x8e, 0x72, 0x92, 0xa5, + 0x12, 0xe9, 0x00, 0xda, 0x47, 0x79, 0xee, 0xe4, 0xea, 0x4f, 0xf6, 0x03, 0x0c, 0x9e, 0x24, 0xd9, + 0xf9, 0xe5, 0x48, 0x28, 0xc1, 0xf1, 0xbb, 0x02, 0xa5, 0xa2, 0x5b, 0xd0, 0x31, 0x85, 0x73, 0x3c, + 0x6b, 0x68, 0xd4, 0x5c, 0xde, 0x55, 0xc6, 0x1a, 0x1a, 0x35, 0xfe, 0x46, 0x7d, 0xc0, 0xad, 0xa1, + 0xd1, 0xd3, 0x24, 0x3e, 0xb7, 0xaa, 0x03, 0x6e, 0x0d, 0x4a, 0x21, 0x78, 0x11, 0xe3, 0xb5, 0x93, + 0x6a, 0xbe, 0x59, 0x04, 0xf7, 0x6a, 0xf9, 0x9d, 0xcc, 0xfb, 0xd0, 0xe5, 0xd9, 0x75, 0x34, 0x92, + 0x21, 0xd9, 0x6d, 0xef, 0x05, 0xdc, 0x59, 0xa6, 0x20, 0xe6, 0xc5, 0xf4, 0x51, 0xcb, 0x1c, 0x55, + 0x00, 0x7b, 0x00, 0x1d, 0x53, 0x1d, 0x7d, 0xcb, 0xca, 0x57, 0x7f, 0xb2, 0x5f, 0x08, 0xdc, 0x3b, + 0x11, 0x2f, 0x8d, 0x0c, 0x59, 0xa6, 0x39, 0x86, 0x7e, 0x09, 0x1a, 0xf6, 0xda, 0xa3, 0xfd, 0x03, + 0xdf, 0x3e, 0x07, 0x73, 0xfc, 0x0a, 0x39, 0x4a, 0x55, 0x3e, 0xe5, 0x95, 0xf3, 0xf6, 0x87, 0xb0, + 0x31, 0x7b, 0xa8, 0x35, 0x5c, 0xe2, 0xd4, 0x57, 0xfa, 0x12, 0xa7, 0xba, 0x26, 0x57, 0x22, 0x29, + 0x6c, 0xfd, 0x02, 0x6e, 0x8d, 0x0f, 0x5a, 0xef, 0x11, 0xf6, 0x15, 0xd0, 0xc3, 0x1c, 0x85, 0x42, + 0x13, 0xe0, 0x04, 0xa5, 0x14, 0xdf, 0xe0, 0xf2, 0x57, 0xb0, 0x95, 0x6d, 0xd5, 0x2b, 0xbb, 0x03, + 0xfd, 0x48, 0xba, 0xde, 0x32, 0x2f, 0xd1, 0xe3, 0x15, 0xc0, 0xf6, 0x81, 0x8e, 0x30, 0x41, 0x85, + 0x6e, 0x1c, 0x56, 0xc4, 0x67, 0xa7, 0x5e, 0xcb, 0xed, 0x5c, 0xfa, 0x10, 0x02, 0x3d, 0x09, 0x46, + 0xca, 0xda, 0xa3, 0xd7, 0xaa, 0xd2, 0x95, 0x63, 0xc7, 0x0d, 0x81, 0xc5, 0x3e, 0xa8, 0x9b, 0x9e, + 0x5b, 0x2e, 0xb8, 0xa0, 0xcd, 0x7c, 0xaa, 0x76, 0x33, 0x55, 0x39, 0x8f, 0x2e, 0xd5, 0x47, 0xfe, + 0xae, 0xff, 0x35, 0x15, 0x1b, 0x39, 0x54, 0xb7, 0xeb, 0x33, 0x7d, 0x6a, 0x7d, 0xcc, 0xf7, 0xf2, + 0x2b, 0x37, 0x75, 0xfc, 0x4d, 0x5c, 0xca, 0xbb, 0x85, 0x69, 0x54, 0x4e, 0x2f, 0x19, 0xdf, 0x58, + 0x6e, 0xc2, 0x4a, 0x9b, 0x3e, 0x84, 0xae, 0xc9, 0x2a, 0xc3, 0xc0, 0xf4, 0xee, 0x66, 0x43, 0x0d, + 0x77, 0xc7, 0x7a, 0x9c, 0x5c, 0x93, 0x77, 0xec, 0x38, 0x59, 0x8b, 0x1e, 0xc1, 0x20, 0x4a, 0x27, + 0x85, 0x1a, 0xe1, 0xd7, 0x71, 0x1a, 0xab, 0x38, 0x4b, 0x65, 0xd8, 0x35, 0xa1, 0x1e, 0xd4, 0x15, + 0xcd, 0x30, 0xf8, 0x9c, 0x0b, 0xfb, 0x89, 0xc0, 0x66, 0x03, 0x5c, 0x72, 0x69, 0xaf, 0xb7, 0xb5, + 0x5a, 0xef, 0xbb, 0xd0, 0x7d, 0x1a, 0x63, 0x72, 0x21, 0xc3, 0xb6, 0x21, 0x0e, 0x97, 0xaa, 0x31, + 0x34, 0xee, 0xd8, 0xec, 0x57, 0x02, 0x5b, 0x8b, 0x08, 0x0b, 0xd5, 0x0c, 0x01, 0x3e, 0xcf, 0xe3, + 0xb1, 0xc8, 0xa7, 0x9f, 0xe0, 0xd4, 0xad, 0xe7, 0x1a, 0x42, 0xbf, 0x84, 0xfb, 0x8d, 0x58, 0x1f, + 0x9f, 0xdb, 0x12, 0x59, 0x51, 0x6f, 0x2c, 0x15, 0x65, 0x79, 0x7c, 0x89, 0x3b, 0xfb, 0x87, 0xc0, + 0xeb, 0x0b, 0x8f, 0xaa, 0x7e, 0x24, 0xf5, 0xd6, 0xdf, 0x87, 0xc1, 0x0b, 0xbd, 0x2a, 0x46, 0x28, + 0x55, 0x9c, 0x0a, 0xcd, 0x74, 0x0d, 0x3b, 0x87, 0xd3, 0x08, 0x7a, 0x06, 0x3b, 0x11, 0x13, 0x27, + 0xf3, 0xed, 0x5b, 0x64, 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xa5, 0xbb, 0x16, 0x63, 0xb6, 0xae, 0x5f, + 0xe1, 0xc6, 0xd8, 0x7e, 0x0c, 0xeb, 0x33, 0x0e, 0x77, 0xda, 0x73, 0x3f, 0x12, 0xd8, 0xf1, 0xcb, + 0x65, 0x46, 0xca, 0xea, 0x31, 0xf5, 0xaf, 0xd7, 0xaa, 0xbd, 0xde, 0xfb, 0x00, 0x95, 0xbb, 0xdb, + 0x0a, 0x2b, 0x9a, 0xb6, 0x46, 0x66, 0xc7, 0xb0, 0xe3, 0xb7, 0xe1, 0xff, 0x13, 0xc1, 0x04, 0xc0, + 0xb3, 0xec, 0x02, 0x4f, 0x95, 0x50, 0x85, 0xd4, 0x8c, 0xe3, 0x4c, 0x2a, 0xdf, 0x64, 0xfa, 0xdb, + 0x6c, 0x6b, 0x25, 0x54, 0xb9, 0x61, 0x8c, 0x41, 0xdf, 0x82, 0x57, 0x4d, 0x50, 0xf4, 0xbd, 0xb4, + 0xd9, 0x58, 0x00, 0xdc, 0x9f, 0xb3, 0xc7, 0xb0, 0x7e, 0x98, 0x14, 0x52, 0x61, 0xee, 0xb2, 0xec, + 0x43, 0x47, 0xe7, 0xf4, 0xbf, 0x57, 0x5b, 0x95, 0x67, 0x25, 0x85, 0x5b, 0xca, 0x93, 0xc1, 0xef, + 0x37, 0x43, 0xf2, 0xc7, 0xcd, 0x90, 0xfc, 0x79, 0x33, 0x24, 0x3f, 0xff, 0x35, 0x7c, 0xe5, 0xac, + 0x6b, 0xfe, 0x23, 0xbd, 0xf3, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x15, 0xa7, 0xba, 0xef, 0x34, + 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 3ad12df7c..0d76f3b98 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -89,10 +89,10 @@ message InputDefinition { message InputDefinitionField { string Name = 1; bool PrimaryKey = 2; - repeated Action Actions = 3; + repeated InputDefinitionAction InputDefinitionActions = 3; } -message Action { +message InputDefinitionAction { string Frame = 1; string ValueDestination = 2; map ValueMap = 3; From 9229d772e904928a8fb54d5e101558a6dfd06a56 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 15:18:37 -0500 Subject: [PATCH 31/99] change Field to InputDefinitionField --- input_definition.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index 53f3caaa2..1e8fd4680 100644 --- a/input_definition.go +++ b/input_definition.go @@ -40,7 +40,7 @@ type InputDefinition struct { index string broadcaster Broadcaster frames []InputFrame - fields []Field + fields []InputDefinitionField } // NewInputDefinition returns a new instance of InputDefinition. @@ -61,7 +61,7 @@ func NewInputDefinition(path, index, name string) (*InputDefinition, error) { func (i *InputDefinition) Frames() []InputFrame { return i.frames } // Fields returns fields of the input definition was initialized with. -func (i *InputDefinition) Fields() []Field { return i.fields } +func (i *InputDefinition) Fields() []InputDefinitionField { return i.fields } // Open opens and initializes the InputDefinition from file. func (i *InputDefinition) Open() error { @@ -130,7 +130,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { return errors.New("duplicate primaryKey with other field") } - inputField := Field{ + inputField := InputDefinitionField{ Name: field.Name, PrimaryKey: field.PrimaryKey, Actions: actions, @@ -210,15 +210,15 @@ func (i *InputDefinition) saveMeta() error { return nil } -// Field descripes a single field mapping in the InputDefinition. -type Field struct { +// InputDefinitionField descripes a single field mapping in the InputDefinition. +type InputDefinitionField struct { Name string `json:"name,omitempty"` PrimaryKey bool `json:"primaryKey,omitempty"` Actions []Action `json:"actions,omitempty"` } -// Encode converts Field into its internal representation. -func (o *Field) Encode() (*internal.InputDefinitionField, error) { +// Encode converts InputDefinitionField into its internal representation. +func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) { field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} for _, action := range o.Actions { @@ -268,7 +268,7 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` - Fields []Field `json:"fields"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation. From 735a7287b79ee3bd7e5841b05aed13b2fd8b7beb Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 21:57:00 -0500 Subject: [PATCH 32/99] merge action's processing --- handler.go | 63 +++++++++++++++++++++++++++++++++++++++++++ server/server_test.go | 32 ++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/handler.go b/handler.go index 5dd2347e8..def9f8436 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "sort" "strconv" "strings" "time" @@ -1639,3 +1640,65 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } + +// MapAction Process the input data and set a bit +func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var ok bool + bit.ColumnID = colID + bit.RowID, ok = a.ValueMap[value] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", value) + } + + // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, nil +} + +// ValueToRow Sets a bitmap with rowID from the input value +func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// SingleRowBoolean Sets a bitmap with rowID from the Action defintion +func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// InputBits Process and Sort the Input Bits and route to appropriate nodes. +func (h *Handler) InputBits(index, frame string, bits []Bit) error { + client, err := NewClient(h.Host) + if err != nil { + return err + } + + bitsBySlice := Bits(bits).GroupBySlice() + + // Parse path into bits. + for slice, bits := range bitsBySlice { + sort.Sort(BitsByPos(bits)) + + h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) + if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { + return err + } + } + + return nil +} diff --git a/server/server_test.go b/server/server_test.go index cd82ba007..a92da70a8 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -494,6 +494,29 @@ func TestMain_SendReceiveMessage(t *testing.T) { if maxSlices1["i"] != 2 { t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"]) } + + // Write input definition to the first node. + if _, err := m0.CreateDefinition("i", "test", `{ + "frames": [{"name": "event-time", + "options": { + "cacheType": "ranked", + "timeQuantum": "YMD" + }}], + "fields": [{"name": "id", + "primaryKey": true + }]} + `); err != nil { + t.Fatal(err) + } + + frame0 := m0.Server.Holder.Frame("i", "event-time") + if frame0 == nil { + t.Fatal("frame not found") + } + frame1 := m1.Server.Holder.Frame("i", "event-time") + if frame1 == nil { + t.Fatal("frame not found") + } } // availablePorts returns a slice of ports that can be used for testing. @@ -615,6 +638,15 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +// CreateDefinition. +func (m *Main) CreateDefinition(index, def, query string) (string, error) { + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + return resp.Body, nil +} + // SetCommand represents a command to set a bit. type SetCommand struct { ID uint64 From cfe146aa971b4605c277a021c74b1aff2b7dddd2 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 23 Jun 2017 11:17:36 -0500 Subject: [PATCH 33/99] update JSONparser --- handler.go | 89 +++++++++++++++++++++--------------------------------- index.go | 70 ------------------------------------------ 2 files changed, 34 insertions(+), 125 deletions(-) diff --git a/handler.go b/handler.go index def9f8436..008ad89a8 100644 --- a/handler.go +++ b/handler.go @@ -1630,7 +1630,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = index.JSONParser(req.(map[string]interface{}), inputDefName) + err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1641,64 +1641,43 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// MapAction Process the input data and set a bit -func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var ok bool - bit.ColumnID = colID - bit.RowID, ok = a.ValueMap[value] - if !ok { - return nil, fmt.Errorf("Value %s does not exist in definition map", value) +// JSONParser validate input json file and execute SetBit +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { + inputDef := index.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound } - - // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, nil -} - -// ValueToRow Sets a bitmap with rowID from the input value -func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// SingleRowBoolean Sets a bitmap with rowID from the Action defintion -func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err - } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// InputBits Process and Sort the Input Bits and route to appropriate nodes. -func (h *Handler) InputBits(index, frame string, bits []Bit) error { - client, err := NewClient(h.Host) - if err != nil { - return err - } - - bitsBySlice := Bits(bits).GroupBySlice() - - // Parse path into bits. - for slice, bits := range bitsBySlice { - sort.Sort(BitsByPos(bits)) - - h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) - if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { - return err + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } } + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + var colValue uint64 + if _, ok := req[field.Name]; !ok { + continue + } else if field.PrimaryKey { + colValue, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) + } else { + val, ok := req[DefaultColumnLabel] + if !ok { + return errors.New("column ID not provided") + } + colValue = val.(float64) + } + } + + } + return nil } diff --git a/index.go b/index.go index 1114be5d0..c70c9eec4 100644 --- a/index.go +++ b/index.go @@ -722,73 +722,3 @@ func (i *Index) openInputDefinition() error { } return nil } - -func (i *Index) JSONParser(req map[string]interface{}, name string) error { - inputDef := i.inputDefinition(name) - if inputDef == nil { - return ErrInputDefinitionNotFound - } - // if field in input data is not in defined definition, return error - validFields := make(map[string]bool) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - } - for key, _ := range req { - _, ok := validFields[key] - if !ok { - fmt.Errorf("field not found", key) - } - } - - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue - } - - for _, action := range field.Actions { - switch action.ValueDestination { - case "mapping": - value, ok := req[field.Name].(string) - if !ok { - return fmt.Errorf("String type required, got %s:%s", field.Name, value) - } - err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) - if err != nil { - return fmt.Errorf("Error action mapping : %s", err) - } - case "single-row-boolean": - value, ok := req[field.Name].(bool) - if !ok { - return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.StringRowBoolean(action.Frame, action.RowID, value) - if err != nil { - return fmt.Errorf("Error action single-row-boolean : %s", err) - } - case "value-to-row": - value, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.ValueToRow(action.Frame, field.Name, uint64(value)) - if err != nil { - return fmt.Errorf("Error action value-to-row: %s", err) - } - } - } - } - - return nil -} -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { - return nil -} - -func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { - return nil -} - -func (i *Index) ValueToRow(frame, name string, value uint64) error { - return nil -} From 53c7e5946da5d5e37c8d820c5e5683e992a58aac Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 23 Jun 2017 14:39:52 -0500 Subject: [PATCH 34/99] validate columnID match primaryField --- handler.go | 24 ++++++++++++- handler_test.go | 76 +++++++++++++++++++++++++++++++++++++++- input_definition.go | 15 +++----- input_definition_test.go | 12 ++----- server/server_test.go | 2 +- 5 files changed, 106 insertions(+), 23 deletions(-) diff --git a/handler.go b/handler.go index 9a9617994..e2200bf21 100644 --- a/handler.go +++ b/handler.go @@ -1499,6 +1499,7 @@ func errorString(err error) string { return err.Error() } +// handlePOSTInputDefinition handles POST /input-definition request. func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1525,6 +1526,26 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } def.Name = inputDefName + // Validate columnLabel & duplicate primaryKey + numPrimaryKey := 0 + for _, field := range def.Fields { + if field.PrimaryKey { + numPrimaryKey += 1 + if field.Name == index.columnLabel { + continue + } else { + err = fmt.Errorf("primary field's name not match columnLabel") + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + } + if numPrimaryKey > 1 { + err = errors.New("duplicate primaryKey with other field") + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Create InputDefinition. _, err = index.CreateInputDefinition(def) if err == ErrInputDefinitionExists { @@ -1550,6 +1571,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } } +// handleGetInputDefinition handles GET /input-definition request. func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1563,7 +1585,6 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques return } inputDef, _ := index.inputDefinitions[inputDefName] - //inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ Frames: inputDef.frames, Fields: inputDef.fields, @@ -1573,6 +1594,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques } +// handleDeleteInputDefinition handles DELETE /input-definition request. func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] diff --git a/handler_test.go b/handler_test.go index 35d57e912..34e3e4585 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1084,7 +1084,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { }], "fields": [ { - "name": "id", + "name": "columnID", "primaryKey": true }, { @@ -1112,6 +1112,80 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + +} + +//Ensure throwing error if there's duplicated primaryKey field +func TestHandler_DuplicatePrimaryKey(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + inputBody1 := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + }, + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(inputBody1))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `duplicate primaryKey with other field`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +// Eusure throwing error if primary field's name doesn't match columnLabel +func TestHandler_UnmatchColumnID(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{ColumnLabel: "id"}) + inputBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `primary field's name not match columnLabel`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + } // Ensure handler can delete a input definition. diff --git a/input_definition.go b/input_definition.go index 1e8fd4680..6eb72e43e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -99,7 +99,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } - numPrimaryKey := 0 countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action @@ -122,13 +121,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: &action.RowID, }) } - if field.PrimaryKey { - numPrimaryKey += 1 - } - - if numPrimaryKey > 1 { - return errors.New("duplicate primaryKey with other field") - } inputField := InputDefinitionField{ Name: field.Name, @@ -252,6 +244,7 @@ func (o *Action) Encode() (*internal.InputDefinitionAction, error) { }, nil } +// convert pointer to uint64 func convert(x *uint64) uint64 { if x != nil { return *x @@ -267,8 +260,8 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []InputDefinitionField `json:"fields"` + Frames []InputFrame `json:"frames"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation. @@ -288,6 +281,7 @@ func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { return &def, nil } +// AddFrame adds frame to input definition func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) if err := i.saveMeta(); err != nil { @@ -296,6 +290,7 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } +// ValidateAction validate actions from input-definition func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired diff --git a/input_definition_test.go b/input_definition_test.go index aa6acd5f8..76b55c7f0 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -128,16 +128,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} - err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { - t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) - } - - action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + field1 := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { diff --git a/server/server_test.go b/server/server_test.go index 6f4570709..b6af8c423 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -540,7 +540,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { "cacheType": "ranked", "timeQuantum": "YMD" }}], - "fields": [{"name": "id", + "fields": [{"name": "columnID", "primaryKey": true }]} `); err != nil { From 6711026d22be2192d7b1158d50961983bd5248d6 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 11:05:34 -0500 Subject: [PATCH 35/99] JSONparser tests --- handler.go | 45 ++++++++----- handler_test.go | 157 ++++++++++++++++++++++++++++++++++++++++++-- input_definition.go | 4 +- 3 files changed, 184 insertions(+), 22 deletions(-) diff --git a/handler.go b/handler.go index 008ad89a8..56f86d39e 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,6 @@ import ( "net/http" _ "net/http/pprof" "os" - "sort" "strconv" "strings" "time" @@ -1639,6 +1638,10 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } // JSONParser validate input json file and execute SetBit @@ -1648,36 +1651,48 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri return ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error + var columnLabel string validFields := make(map[string]bool) for _, field := range inputDef.Fields() { validFields[field.Name] = true + if field.PrimaryKey { + columnLabel = field.Name + } } for key, _ := range req { _, ok := validFields[key] if !ok { - fmt.Errorf("field not found", key) + return fmt.Errorf("field not found: %s", key) } } + var bits []*Bit for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - var colValue uint64 + //var colValue uint64 if _, ok := req[field.Name]; !ok { continue - } else if field.PrimaryKey { - colValue, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) - } else { - val, ok := req[DefaultColumnLabel] - if !ok { - return errors.New("column ID not provided") - } - colValue = val.(float64) - } + } + value, ok := req[columnLabel] + if !ok { + return fmt.Errorf("columnLabel required") + } + colValue, ok := value.(float64) + if !ok { + return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + for _, action := range field.Actions { + bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + if err != nil { + return fmt.Errorf("error handling action: %s", action.ValueDestination) + } + bits = append(bits, bit) + } } - return nil } + +func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + return nil, nil +} diff --git a/handler_test.go b/handler_test.go index 35d57e912..0c3691f5b 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,10 @@ import ( "context" "encoding/json" "errors" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -27,11 +31,6 @@ import ( "reflect" "strings" "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1174,3 +1173,151 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } } + +var defaultBody = ` + { + "frames":[ + { + "name":"event-time", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + } + ], + "fields":[ + { + "name":"id", + "primaryKey":true + }, + { + "name":"cabType", + "actions":[ + { + "frame":"cab-type", + "valueDestination":"mapping", + "valueMap":{ + "Green":1, + "Yellow":2 + } + } + ] + }, + { + "name":"withPet", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"single-row-boolean", + "rowID":100 + } + ] + }, + { + "name":"distanceMiles", + "actions":[ + { + "frame":"distance-miles", + "valueDestination":"value-to-row" + + } + ] + } + ] + }` + +func TestHandler_CreateInput(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + inputBody := []byte(` + [{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "with-pet": true + }]`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestInput_JSON(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + json string + err string + }{ + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "nofield": true + }]`, + err: "field not found: nofield"}, + {json: `[{ + "id": "abc", + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "float64 require, got value:abc, type: string"}, + {json: `[{ + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "columnLabel required"}, + } + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + for _, test := range tests { + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(test.json)))) + if body := w.Body.String(); body != test.err+"\n" { + t.Fatalf("Expect error: %s, actual: %s", test.err, body) + } + } + +} +func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { + var req pilosa.InputDefinitionInfo + err := json.Unmarshal(body, &req) + if err != nil { + return nil, err + } + def, err := req.Encode() + def.Name = name + return def, err +} diff --git a/input_definition.go b/input_definition.go index 1e8fd4680..c2d16c947 100644 --- a/input_definition.go +++ b/input_definition.go @@ -267,8 +267,8 @@ type InputFrame struct { // InputDefinitionInfo the json message format to create an InputDefinition. type InputDefinitionInfo struct { - Frames []InputFrame `json:"frames"` - Fields []InputDefinitionField `json:"fields"` + Frames []InputFrame `json:"frames"` + Fields []InputDefinitionField `json:"fields"` } // Encode converts InputDefinitionInfo into its internal representation. From 03e5b36af76d6bd2e8d765984a8100d5c6c93819 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 12:19:55 -0500 Subject: [PATCH 36/99] merge handleAction --- handler.go | 35 ++++++++------- handler_test.go | 25 +++++++++-- index.go | 23 ++++++++++ index_test.go | 34 +++++++++++++++ input_definition.go | 52 ++++++++++++++++++++++- input_definition_test.go | 92 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 239 insertions(+), 22 deletions(-) diff --git a/handler.go b/handler.go index 56f86d39e..7fae39217 100644 --- a/handler.go +++ b/handler.go @@ -1629,7 +1629,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1637,18 +1637,24 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + for fr, bs := range bits { + err := index.InputBits(fr, bs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } // JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { - return ErrInputDefinitionNotFound + return nil, ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error var columnLabel string @@ -1662,11 +1668,12 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri for key, _ := range req { _, ok := validFields[key] if !ok { - return fmt.Errorf("field not found: %s", key) + return nil, fmt.Errorf("field not found: %s", key) } } var bits []*Bit + setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data //var colValue uint64 @@ -1675,24 +1682,22 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } value, ok := req[columnLabel] if !ok { - return fmt.Errorf("columnLabel required") + return nil, fmt.Errorf("columnLabel required") } colValue, ok := value.(float64) if !ok { - return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } for _, action := range field.Actions { - bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + frame := action.Frame + bit, err := HandleAction(action, req[field.Name], uint64(colValue)) if err != nil { - return fmt.Errorf("error handling action: %s", action.ValueDestination) + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - bits = append(bits, bit) + //bits = append(bits, bit) + setBits[frame] = append(bits, bit) } } - return nil -} - -func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { - return nil, nil + return setBits, nil } diff --git a/handler_test.go b/handler_test.go index 0c3691f5b..ec79f837a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" @@ -1178,12 +1179,27 @@ var defaultBody = ` { "frames":[ { - "name":"event-time", + "name":"cab-type", "options": { "timeQuantum":"YMD", "inverseEnabled":false, "cacheType":"ranked" } + }, + { + "name":"add-ons", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + }, + { + "name":"distance-miles", + "options": { + "timeQuantum":"YMD", + "cacheType":"ranked" + } } ], "fields":[ @@ -1198,8 +1214,8 @@ var defaultBody = ` "frame":"cab-type", "valueDestination":"mapping", "valueMap":{ - "Green":1, - "Yellow":2 + "green":1, + "yellow":2 } } ] @@ -1246,13 +1262,14 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "with-pet": true + "withPet": true }]`) h := NewHandler() h.Holder = hldr.Holder h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { diff --git a/index.go b/index.go index 69afdbbf9..8d5d4065b 100644 --- a/index.go +++ b/index.go @@ -755,3 +755,26 @@ func (i *Index) openInputDefinition() error { } return nil } + +// InputBits Process the []Bit though the Frame import process +func (i *Index) InputBits(frame string, bits []*Bit) error { + var rowIDs, columnIDs []uint64 + timestamps := make([]*time.Time, len(bits)) + f := i.Frame(frame) + if f == nil { + return fmt.Errorf("Frame not found: %s", frame) + } + + for i, bit := range bits { + rowIDs = append(rowIDs, bit.RowID) + columnIDs = append(columnIDs, bit.ColumnID) + + // Convert timestamps to time.Time. + if bit.Timestamp > 0 { + t := time.Unix(0, bit.Timestamp) + timestamps[i] = &t + } + } + + return f.Import(rowIDs, columnIDs, timestamps) +} diff --git a/index_test.go b/index_test.go index 49ab014b3..d7df43da3 100644 --- a/index_test.go +++ b/index_test.go @@ -454,3 +454,37 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } } + +func TestIndex_InputBits(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } + + // Create frame. + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + var bits []*pilosa.Bit + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) + bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + + err := index.InputBits("f", bits) + if err != nil { + t.Fatal(err) + } + + f := index.Frame("f") + v := f.View(pilosa.ViewStandard) + fragment := v.Fragment(0) + + // Verify the Bits were set + if a := fragment.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{0, 1}) { + t.Fatalf("unexpected bits: %+v", a) + } +} diff --git a/input_definition.go b/input_definition.go index c2d16c947..47236b1cf 100644 --- a/input_definition.go +++ b/input_definition.go @@ -15,12 +15,13 @@ package pilosa import ( + "fmt" "io/ioutil" "os" "path/filepath" "errors" - "fmt" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -295,7 +296,6 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } return nil } - func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired @@ -315,3 +315,51 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) } return nil } + +// HandleAction Process the input data with its action and return a bit to be imported later +// Note: if the Bit should not be set then nil is returned with no error +// From the JSON marshalling the possible types are: float64, boolean, string +// TODO handle Timestams +func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + var err error + var bit Bit + bit.ColumnID = colID + + switch a.ValueDestination { + case Mapping: + v, ok := value.(string) + if !ok { + return nil, fmt.Errorf("Mapping value must be a string %v", value) + } + bit.RowID, ok = a.ValueMap[v] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", v) + } + case SingleRowBool: + switch value.(type) { + case bool: + if value.(bool) { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + case float64: + if value.(float64) >= 1 { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + default: + return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) + } + case ValueToRow: + v, ok := value.(float64) + if !ok { + return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) + } + bit.RowID = uint64(v) + default: + return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) + } + return &bit, err +} diff --git a/input_definition_test.go b/input_definition_test.go index aa6acd5f8..0e078536e 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -18,9 +18,10 @@ import ( "encoding/json" "testing" + "strings" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" - "strings" ) func TestInputDefinition_Open(t *testing.T) { @@ -151,3 +152,92 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected frame required error, actual error: %s", err) } } + +func TestHandleAction(t *testing.T) { + var value interface{} + colID := uint64(0) + rowID := uint64(100) + action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + + value = 1 + b, err := pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected integer type is not handled by single-row-boolean") + } else if !strings.Contains(err.Error(), "single-row-boolean value") { + t.Fatalf("Expected single-row-boolean value error, actual error: %s", err) + } + + value = "1" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore strings, only accept boolean") + } + + value = "t" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + + value = float64(1.5) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } + } + value = float64(0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = false + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = true + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.ColumnID != 0 { + t.Fatalf("Unexpected ColumnID %v", b.ColumnID) + } + } + + action.ValueDestination = pilosa.ValueToRow + rowID = 101 + value = float64(25.0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 25 { + t.Fatalf("Unexpected RowID %v", b.RowID) + } + } + value = "25" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type float64") + } + + action.ValueDestination = pilosa.Mapping + value = "test" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + value = 25 + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + action.ValueDestination = "test" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "Unrecognized Value Destination") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + +} From c441e55bb2acc9b760a7936fa3d3ff547614cdb0 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 14:56:27 -0500 Subject: [PATCH 37/99] Prefaced validValueDestination values Code cleanup --- handler.go | 2 +- input_definition.go | 31 +++++++++++++++++-------------- input_definition_test.go | 18 +++++++++--------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/handler.go b/handler.go index 7fae39217..71d6c25ef 100644 --- a/handler.go +++ b/handler.go @@ -1665,7 +1665,7 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri columnLabel = field.Name } } - for key, _ := range req { + for key := range req { _, ok := validFields[key] if !ok { return nil, fmt.Errorf("field not found: %s", key) diff --git a/input_definition.go b/input_definition.go index 47236b1cf..56b5d7705 100644 --- a/input_definition.go +++ b/input_definition.go @@ -26,13 +26,14 @@ import ( "github.com/pilosa/pilosa/internal" ) +// Action Mapping types const ( - Mapping = "mapping" - ValueToRow = "value-to-row" - SingleRowBool = "single-row-boolean" + InputMapping = "mapping" + InputValueToRow = "value-to-row" + InputSingleRowBool = "single-row-boolean" ) -var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -108,13 +109,12 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.ValueDestination == SingleRowBool && action.Frame != "" { + if action.ValueDestination == InputSingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) - } else { - countRowID[action.Frame] = action.RowID } + countRowID[action.Frame] = action.RowID } actions = append(actions, Action{ Frame: action.Frame, @@ -124,7 +124,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { }) } if field.PrimaryKey { - numPrimaryKey += 1 + numPrimaryKey++ } if numPrimaryKey > 1 { @@ -289,6 +289,7 @@ func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { return &def, nil } +// AddFrame manually add frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) if err := i.saveMeta(); err != nil { @@ -296,19 +297,21 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { } return nil } + +// ValidateAction ensures the input definition action conforms to our specification. func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } validValues := make(map[string]bool) - for _, val := range ValidValueDestination { + for _, val := range validValueDestination { validValues[val] = true } if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } switch action.ValueDestination { - case Mapping: + case InputMapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } @@ -319,14 +322,14 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestams +// TODO handle Timestamps func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID switch a.ValueDestination { - case Mapping: + case InputMapping: v, ok := value.(string) if !ok { return nil, fmt.Errorf("Mapping value must be a string %v", value) @@ -335,7 +338,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { if !ok { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } - case SingleRowBool: + case InputSingleRowBool: switch value.(type) { case bool: if value.(bool) { @@ -352,7 +355,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } - case ValueToRow: + case InputValueToRow: v, ok := value.(float64) if !ok { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) diff --git a/input_definition_test.go b/input_definition_test.go index 0e078536e..8adcbc1cc 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -114,13 +114,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -128,8 +128,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 0} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 0} field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -137,7 +137,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) } - action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -145,7 +145,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.InputSingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { @@ -157,7 +157,7 @@ func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) rowID := uint64(100) - action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} value = 1 b, err := pilosa.HandleAction(action, value, colID) @@ -206,7 +206,7 @@ func TestHandleAction(t *testing.T) { } } - action.ValueDestination = pilosa.ValueToRow + action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) b, err = pilosa.HandleAction(action, value, colID) @@ -221,7 +221,7 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type float64") } - action.ValueDestination = pilosa.Mapping + action.ValueDestination = pilosa.InputMapping value = "test" b, err = pilosa.HandleAction(action, value, colID) if b != nil { From 98fa937af2094444336b99d23e5b1358e9d2797b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 15:19:03 -0500 Subject: [PATCH 38/99] remove prints --- handler.go | 1 - handler_test.go | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 71d6c25ef..341c2d395 100644 --- a/handler.go +++ b/handler.go @@ -1511,7 +1511,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } - fmt.Println() // Find index. index := h.Holder.Index(indexName) diff --git a/handler_test.go b/handler_test.go index ec79f837a..0fe1f6c47 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,11 +19,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -32,6 +27,11 @@ import ( "reflect" "strings" "testing" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1269,7 +1269,6 @@ func TestHandler_CreateInput(t *testing.T) { h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) - fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { From d8e7769f5b48d20bcc27649790d9aa5779b67699 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:51:57 -0500 Subject: [PATCH 39/99] handle input nil bit --- index.go | 3 +++ index_test.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index 8d5d4065b..7c97c0498 100644 --- a/index.go +++ b/index.go @@ -766,6 +766,9 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { } for i, bit := range bits { + if bit == nil { + continue + } rowIDs = append(rowIDs, bit.RowID) columnIDs = append(columnIDs, bit.ColumnID) diff --git a/index_test.go b/index_test.go index d7df43da3..8a7367636 100644 --- a/index_test.go +++ b/index_test.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -456,6 +457,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } func TestIndex_InputBits(t *testing.T) { + var bits []*pilosa.Bit index := MustOpenIndex() defer index.Close() @@ -464,17 +466,22 @@ func TestIndex_InputBits(t *testing.T) { t.Fatal(err) } + err := index.InputBits("f", bits) + if !strings.Contains(err.Error(), "Frame not found") { + t.Fatalf("Expected Frame not found error, actual error: %s", err) + } + // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - var bits []*pilosa.Bit bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + bits = append(bits, nil) - err := index.InputBits("f", bits) + err = index.InputBits("f", bits) if err != nil { t.Fatal(err) } From 66900e48369f53851744a0e2836e59d94e8bd8e9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:52:51 -0500 Subject: [PATCH 40/99] accumulate set bits per frame, don't mix into one set across frames --- handler.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 341c2d395..a44f05ae9 100644 --- a/handler.go +++ b/handler.go @@ -1628,7 +1628,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.InputJsonDataParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1649,8 +1649,8 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { +// InputJsonDataParser validate input json file and execute SetBit +func (h *Handler) InputJsonDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { return nil, ErrInputDefinitionNotFound @@ -1671,11 +1671,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } } - var bits []*Bit setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - //var colValue uint64 if _, ok := req[field.Name]; !ok { continue } @@ -1694,8 +1692,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - //bits = append(bits, bit) - setBits[frame] = append(bits, bit) + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } } } return setBits, nil From 904552b289cbf2433dc2cad7a641fbe6d9d9fed2 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 26 Jun 2017 17:39:58 -0500 Subject: [PATCH 41/99] adjusted some of the InputDefinition comments --- handler.go | 31 +++++++++++++++---------------- handler_test.go | 8 ++++---- index.go | 18 +++++++++--------- input_definition.go | 13 +++++++------ 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/handler.go b/handler.go index e2200bf21..9e9542c82 100644 --- a/handler.go +++ b/handler.go @@ -1499,11 +1499,18 @@ func errorString(err error) string { return err.Error() } -// handlePOSTInputDefinition handles POST /input-definition request. +// handlePostInputDefinition handles POST /input-definition request. func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1512,13 +1519,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - + // Encode InputDefinition to its internal representation. def, err := req.Encode() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1526,22 +1527,20 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } def.Name = inputDefName - // Validate columnLabel & duplicate primaryKey + // Validate columnLabel and duplicate primaryKey. numPrimaryKey := 0 for _, field := range def.Fields { if field.PrimaryKey { numPrimaryKey += 1 - if field.Name == index.columnLabel { - continue - } else { - err = fmt.Errorf("primary field's name not match columnLabel") + if field.Name != index.columnLabel { + err = fmt.Errorf("PrimaryKey field name does not match columnLabel") http.Error(w, err.Error(), http.StatusBadRequest) return } } } if numPrimaryKey > 1 { - err = errors.New("duplicate primaryKey with other field") + err = errors.New("InputDefinition can only contain one PrimaryKey") http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -1576,7 +1575,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - //Find index. + // Find index. index := h.Holder.Index(indexName) if index == nil { if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { @@ -1620,7 +1619,7 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req Name: inputDefName, }) if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) } if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { diff --git a/handler_test.go b/handler_test.go index 34e3e4585..33f79474e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1115,7 +1115,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } -//Ensure throwing error if there's duplicated primaryKey field +// Ensure throwing error if there's duplicated primaryKey field. func TestHandler_DuplicatePrimaryKey(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -1148,12 +1148,12 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(inputBody1))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `duplicate primaryKey with other field`+"\n" { + } else if body := w.Body.String(); body != `InputDefinition can only contain one PrimaryKey`+"\n" { t.Fatalf("unexpected body: %s", body) } } -// Eusure throwing error if primary field's name doesn't match columnLabel +// Eusure throwing error if primary field's name doesn't match columnLabel. func TestHandler_UnmatchColumnID(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -1182,7 +1182,7 @@ func TestHandler_UnmatchColumnID(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `primary field's name not match columnLabel`+"\n" { + } else if body := w.Body.String(); body != `PrimaryKey field name does not match columnLabel`+"\n" { t.Fatalf("unexpected body: %s", body) } diff --git a/index.go b/index.go index 69afdbbf9..3d9c1a06d 100644 --- a/index.go +++ b/index.go @@ -51,14 +51,14 @@ type Index struct { // Frames by name. frames map[string]*Frame - // Max Slice on any node in the cluster, according to this node + // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - // Column attribute storage and cache + // Column attribute storage and cache. columnAttrStore *AttrStore - // InputDefinition by name + // InputDefinitions by name. inputDefinitions map[string]*InputDefinition broadcaster Broadcaster @@ -336,7 +336,7 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } -// InputDefinitionPath returns the path to a inputdefinition in the index. +// InputDefinitionPath returns the path to an input definition in the index. func (i *Index) InputDefinitionPath() string { return filepath.Join(i.path, InputDefinitionDir) } @@ -618,10 +618,10 @@ type IndexOptions struct { } // Encode converts i into its internal representation. -func (o *IndexOptions) Encode() *internal.IndexMeta { +func (i *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ - ColumnLabel: o.ColumnLabel, - TimeQuantum: string(o.TimeQuantum), + ColumnLabel: i.ColumnLabel, + TimeQuantum: string(i.TimeQuantum), } } @@ -702,7 +702,7 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { return inputDef, nil } -// DeleteInputDefinition removes a input definition from the index. +// DeleteInputDefinition removes an input definition from the index. func (i *Index) DeleteInputDefinition(name string) error { i.mu.Lock() defer i.mu.Unlock() @@ -742,7 +742,7 @@ func (i *Index) openInputDefinition() error { input.Open() i.inputDefinitions[file.Name()] = input - // Create frame if it doesn't exist + // Create frame if it doesn't exist. for _, fr := range input.frames { _, err := i.CreateFrame(fr.Name, fr.Options) if err == ErrFrameExists { diff --git a/input_definition.go b/input_definition.go index 6eb72e43e..c83a43f5b 100644 --- a/input_definition.go +++ b/input_definition.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -80,7 +81,7 @@ func (i *InputDefinition) Open() error { return nil } -// LoadDefinition loads the protobuf format of a defition +// LoadDefinition loads the protobuf format of a definition. func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name @@ -146,7 +147,7 @@ func (i *InputDefinition) loadMeta() error { return i.LoadDefinition(&pb) } -//saveMeta writes meta data for the input definition file. +// saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { return err @@ -223,7 +224,7 @@ func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) return &field, nil } -// Action descripes the mapping method for the field in the InputDefinition. +// Action describes the mapping method for the field in the InputDefinition. type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` @@ -258,7 +259,7 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -// InputDefinitionInfo the json message format to create an InputDefinition. +// InputDefinitionInfo represents the json message format needed to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` Fields []InputDefinitionField `json:"fields"` @@ -281,7 +282,7 @@ func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { return &def, nil } -// AddFrame adds frame to input definition +// AddFrame adds frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) if err := i.saveMeta(); err != nil { @@ -290,7 +291,7 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -// ValidateAction validate actions from input-definition +// ValidateAction validates actions from InputDefinition. func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired From def02c45e73cc059cf44292a22eb6cb753c418d7 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 08:50:33 -0500 Subject: [PATCH 42/99] The Input process must respect the Action Frame assignments --- handler_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index 0fe1f6c47..37fd9f25c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1274,6 +1274,32 @@ func TestHandler_CreateInput(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + + // Verify the bits set per frame + // f := index.Frame("cab-type") + f0 := index.Frame("distance-miles") + v0 := f0.View(pilosa.ViewStandard) + fragment0 := v0.Fragment(0) + + // Verify the distanceMiles Bit was set + if a := fragment0.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + + f1 := index.Frame("add-ons") + v1 := f1.View(pilosa.ViewStandard) + fragment1 := v1.Fragment(0) + + // Verify the add-ons frame does not have a distanceMiles Bit set + // The Input process must respect the Action Frame assignments + if a := fragment1.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{}) { + t.Fatalf("unexpected bits: %+v", a) + } + // Verify the withPet Bit was set + if a := fragment1.Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + } func TestInput_JSON(t *testing.T) { @@ -1325,8 +1351,8 @@ func TestInput_JSON(t *testing.T) { t.Fatalf("Expect error: %s, actual: %s", test.err, body) } } - } + func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { var req pilosa.InputDefinitionInfo err := json.Unmarshal(body, &req) From 16db983bf1d12f0fd73dc7b4dda0e56e74a6729d Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:02:25 -0500 Subject: [PATCH 43/99] removed float64 check in single row boolean --- input_definition.go | 6 ------ input_definition_test.go | 16 ++++++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/input_definition.go b/input_definition.go index 56b5d7705..40b8451b8 100644 --- a/input_definition.go +++ b/input_definition.go @@ -346,12 +346,6 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { } else { // value is not True. return nil, err } - case float64: - if value.(float64) >= 1 { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } diff --git a/input_definition_test.go b/input_definition_test.go index 8adcbc1cc..595f57f10 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -179,17 +179,10 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - value = float64(1.5) + value = float64(1) b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - if b.RowID != 100 { - t.Fatalf("Unexpected rowID %v", b.RowID) - } - } - value = float64(0) - b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - t.Fatalf("Expected Ignore values that do not equate to True") + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false @@ -204,6 +197,9 @@ func TestHandleAction(t *testing.T) { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) } + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } } action.ValueDestination = pilosa.InputValueToRow From 7b66fcf26c8dc0d9c8fc514509d2f93b3a16486b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:39:11 -0500 Subject: [PATCH 44/99] cleanup type detection for consistency --- input_definition.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index 40b8451b8..a8e2e709e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -339,16 +339,14 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } case InputSingleRowBool: - switch value.(type) { - case bool: - if value.(bool) { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } - default: + v, ok := value.(bool) + if !ok { return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } + if v == false { // False returns a nil error and nil bit. + return nil, err + } + bit.RowID = *a.RowID case InputValueToRow: v, ok := value.(float64) if !ok { From b2bbff6a33bc5e6c00a9fde9f8695a4cd0bb37a6 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 11:07:54 -0500 Subject: [PATCH 45/99] conver invalid input name --- input_definition_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/input_definition_test.go b/input_definition_test.go index 595f57f10..753681e51 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -32,8 +32,14 @@ func TestInputDefinition_Open(t *testing.T) { frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) + if !strings.Contains(err.Error(), "invalid index or frame's name") { + t.Fatalf("Expected Invalid name error, actual error: %s", err) + } + + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } From 1407a250080dc90dac1147ee4a232e1b0819024a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 27 Jun 2017 12:36:02 -0500 Subject: [PATCH 46/99] fix reviews, more input definition tests --- handler.go | 22 +++--- handler_test.go | 116 +++++++++++++++++++++--------- index.go | 4 +- index_test.go | 33 +++++---- internal/private.pb.go | 155 +++++++++++++++-------------------------- internal/private.proto | 1 - pilosa.go | 17 +++-- 7 files changed, 181 insertions(+), 167 deletions(-) diff --git a/handler.go b/handler.go index 9e9542c82..bb3ed54a4 100644 --- a/handler.go +++ b/handler.go @@ -1533,15 +1533,13 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque if field.PrimaryKey { numPrimaryKey += 1 if field.Name != index.columnLabel { - err = fmt.Errorf("PrimaryKey field name does not match columnLabel") - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, ErrInputDefinitionColumnLabel.Error(), http.StatusBadRequest) return } } } if numPrimaryKey > 1 { - err = errors.New("InputDefinition can only contain one PrimaryKey") - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, ErrInputDefinitionPrimaryKey.Error(), http.StatusBadRequest) return } @@ -1558,14 +1556,13 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ Index: indexName, - Name: inputDefName, Definition: def, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } @@ -1578,11 +1575,10 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } + inputDef, _ := index.inputDefinitions[inputDefName] if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ Frames: inputDef.frames, @@ -1601,9 +1597,7 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1622,9 +1616,9 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type postInputDefinitionResponse struct{} +type defaultInputDefinitionResponse struct{} diff --git a/handler_test.go b/handler_test.go index 33f79474e..de2831d37 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1113,6 +1113,14 @@ func TestHandler_CreateInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s", body) } + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusConflict { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionExists.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + } // Ensure throwing error if there's duplicated primaryKey field. @@ -1120,7 +1128,12 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - inputBody1 := []byte(` + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + + //Ensure throwing error if there's duplicated primaryKey field + invalidPrimaryKey := []byte(` { "frames":[{ "name":"event-time", @@ -1141,24 +1154,18 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { } ] }`) - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(inputBody1))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `InputDefinition can only contain one PrimaryKey`+"\n" { + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionPrimaryKey.Error()+"\n" { t.Fatalf("unexpected body: %s", body) } -} -// Eusure throwing error if primary field's name doesn't match columnLabel. -func TestHandler_UnmatchColumnID(t *testing.T) { - hldr := MustOpenHolder() - defer hldr.Close() - hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{ColumnLabel: "id"}) - inputBody := []byte(` + // Eusure throwing error if primary field's name doesn't match columnLabel + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + unmatchColumnBody := []byte(` { "frames":[{ "name":"event-time", @@ -1175,14 +1182,39 @@ func TestHandler_UnmatchColumnID(t *testing.T) { } ] }`) - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `PrimaryKey field name does not match columnLabel`+"\n" { + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Eusure throwing error if request body is invalid + jsonErrorBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + + + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `unexpected EOF`+"\n" { t.Fatalf("unexpected body: %s", body) } @@ -1192,8 +1224,21 @@ func TestHandler_UnmatchColumnID(t *testing.T) { func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + // Test index not found + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Test input definition is deleted + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} @@ -1202,11 +1247,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { if err != nil { t.Fatal(err) } - - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -1217,30 +1258,41 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } } -// Return existing input definition +// Ensure handler can get existing input definition func TestHandler_GetInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + + // Return error if index does not exist + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) + } + + // Return existing input definition + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } - response := &pilosa.InputDefinitionInfo{Frames: inputDef.Frames(), Fields: inputDef.Fields()} expect, err := json.Marshal(response) if err != nil { t.Fatal(err) } - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) diff --git a/index.go b/index.go index 3d9c1a06d..ca5428e06 100644 --- a/index.go +++ b/index.go @@ -656,9 +656,9 @@ func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefin func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { if pb.Name == "" { - return nil, errors.New("input-definition name required") + return nil, ErrInputDefinitionNameRequired } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { - return nil, errors.New("frames and fields are required") + return nil, ErrInputDefinitionAttrsRequired } for _, fr := range pb.Frames { diff --git a/index_test.go b/index_test.go index 49ab014b3..349dc0522 100644 --- a/index_test.go +++ b/index_test.go @@ -378,16 +378,31 @@ func TestIndex_CreateInputDefinition(t *testing.T) { } } +// Ensure create input definition handle correct error func TestIndex_CreateExistingInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() + // Test frames and fields are required + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionAttrsRequired { + t.Fatal(err) + } + + //Test input definition name is required + def = internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err = index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionNameRequired { + t.Fatal(err) + } + // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} - _, err := index.CreateInputDefinition(&def) + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -397,18 +412,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } } -func TestIndex_CreateEmptyInputDefinition(t *testing.T) { - index := MustOpenIndex() - defer index.Close() - - // Create Input Definition. - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} - _, err := index.CreateInputDefinition(&def) - if err.Error() != "frames and fields are required" { - t.Fatal(err) - } -} - +// Ensure to delete existing input definition. func TestIndex_DeleteInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() @@ -433,6 +437,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { } } +// Ensure that frame in input definition will be created when server restart func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() diff --git a/internal/private.pb.go b/internal/private.pb.go index 8194a13de..abde718e9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -314,7 +314,6 @@ func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { type CreateInputDefinitionMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` } @@ -1160,12 +1159,6 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Name) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } if m.Definition != nil { dAtA[i] = 0x1a i++ @@ -1695,10 +1688,6 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Name) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.Definition != nil { l = m.Definition.Size() n += 1 + l + sovPrivate(uint64(l)) @@ -4284,35 +4273,6 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { } m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Definition", wireType) @@ -5030,63 +4990,62 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 915 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, + // 912 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, + 0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, 0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd, - 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0x66, 0xda, 0x49, 0xcc, 0x81, 0x0b, - 0x12, 0xcf, 0x80, 0xc4, 0x91, 0x97, 0xe1, 0x08, 0x6f, 0x80, 0xc2, 0x85, 0x37, 0xe0, 0x8a, 0xba, - 0xba, 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x2d, 0xdc, 0xba, 0xbe, 0xae, 0x9f, 0xaf, 0xaa, 0xab, 0x6a, - 0x06, 0xd6, 0xa7, 0x69, 0x78, 0xae, 0x34, 0x1e, 0x4c, 0xd3, 0x44, 0x27, 0xbc, 0x15, 0xc6, 0x1a, - 0xd3, 0x58, 0x45, 0xe2, 0x0b, 0x68, 0x0f, 0xe2, 0x53, 0xbc, 0x1c, 0xa2, 0x56, 0x7c, 0x17, 0xd6, - 0x0e, 0x93, 0x68, 0x36, 0x89, 0x3f, 0x57, 0x27, 0x18, 0x75, 0xd9, 0x2e, 0xdb, 0x6b, 0xcb, 0x32, - 0x64, 0x34, 0x9e, 0x85, 0x13, 0xfc, 0x6a, 0xa6, 0x62, 0x3d, 0x9b, 0x74, 0x6b, 0x56, 0xa3, 0x04, - 0x89, 0x3f, 0x18, 0xb4, 0x9f, 0xa4, 0x6a, 0x82, 0xe4, 0x71, 0x1b, 0x5a, 0x32, 0xb9, 0x28, 0xbb, - 0xcb, 0x65, 0xfe, 0x0e, 0x6c, 0x0c, 0xe2, 0x73, 0x4c, 0x33, 0x3c, 0x8a, 0xd5, 0x49, 0x84, 0xa7, - 0xe4, 0xae, 0x25, 0x2b, 0x28, 0xdf, 0x81, 0xf6, 0xa1, 0x1a, 0xbf, 0xc4, 0x67, 0xf3, 0x29, 0x76, - 0xeb, 0xe4, 0xa4, 0x00, 0xf2, 0xdb, 0x51, 0xf8, 0x3d, 0x76, 0x83, 0x5d, 0xb6, 0xb7, 0x2e, 0x0b, - 0xa0, 0xca, 0xb7, 0xb1, 0xc4, 0x97, 0x0b, 0x78, 0x43, 0xaa, 0xf8, 0x45, 0xce, 0xa1, 0x49, 0x1c, - 0x16, 0x30, 0x21, 0x60, 0x63, 0x30, 0x99, 0x26, 0xa9, 0x96, 0x98, 0x4d, 0x93, 0x38, 0x43, 0xde, - 0x81, 0xfa, 0x51, 0x9a, 0xba, 0x94, 0xcc, 0x51, 0xfc, 0x00, 0x9d, 0xc7, 0x51, 0x32, 0x3e, 0xeb, - 0x2b, 0xad, 0x24, 0x7e, 0x37, 0xc3, 0x4c, 0xf3, 0x2d, 0x68, 0x50, 0x71, 0x9d, 0x9e, 0x15, 0x0c, - 0x4a, 0x05, 0x72, 0xd5, 0xb3, 0x82, 0x41, 0xc9, 0x9e, 0x32, 0x0c, 0xa4, 0x15, 0x0c, 0x3a, 0x8a, - 0xc2, 0xb1, 0xcd, 0x2c, 0x90, 0x56, 0xe0, 0x1c, 0x82, 0xe7, 0x21, 0x5e, 0xb8, 0x74, 0xe8, 0x2c, - 0x06, 0x70, 0xa7, 0x14, 0xdf, 0xd1, 0xbc, 0x0b, 0x4d, 0x99, 0x5c, 0x0c, 0xfa, 0x59, 0x97, 0xed, - 0xd6, 0xf7, 0x02, 0xe9, 0x24, 0x2a, 0x1a, 0xbd, 0xaa, 0xb9, 0xaa, 0xd1, 0x55, 0x01, 0x88, 0x7b, - 0xd0, 0xa0, 0x0a, 0x9a, 0x2c, 0x0b, 0x5b, 0x73, 0x14, 0xbf, 0x30, 0xb8, 0x33, 0x54, 0x97, 0x44, - 0x23, 0xcb, 0xc3, 0x1c, 0x43, 0x3b, 0x07, 0x49, 0x7b, 0xed, 0xc1, 0xfe, 0x81, 0x6f, 0xb1, 0x83, - 0x25, 0xfd, 0x02, 0x39, 0x8a, 0x75, 0x3a, 0x97, 0x85, 0xf1, 0xf6, 0xc7, 0xb0, 0xb1, 0x78, 0x69, - 0x38, 0x9c, 0xe1, 0xdc, 0x57, 0xfa, 0x0c, 0xe7, 0xa6, 0x26, 0xe7, 0x2a, 0x9a, 0xd9, 0xfa, 0x05, - 0xd2, 0x0a, 0x1f, 0xd5, 0x3e, 0x60, 0xe2, 0x1b, 0xe0, 0x87, 0x29, 0x2a, 0x8d, 0xe4, 0x60, 0x88, - 0x59, 0xa6, 0x5e, 0xe0, 0xea, 0x57, 0xb0, 0x95, 0xad, 0x95, 0x2b, 0xbb, 0x03, 0xed, 0x41, 0xe6, - 0xfa, 0x8f, 0x5e, 0xa2, 0x25, 0x0b, 0x40, 0xec, 0x03, 0xef, 0x63, 0x84, 0x1a, 0xdd, 0xc8, 0xdc, - 0xe0, 0x5f, 0x8c, 0x3c, 0x97, 0xdb, 0x75, 0xf9, 0x7d, 0x08, 0xcc, 0xb4, 0x10, 0x95, 0xb5, 0x07, - 0x6f, 0x16, 0xa5, 0xcb, 0x47, 0x53, 0x92, 0x82, 0x08, 0xbd, 0x53, 0x37, 0x61, 0xb7, 0x24, 0x78, - 0x4d, 0x9b, 0xf9, 0x50, 0xf5, 0x6a, 0xa8, 0x7c, 0x66, 0x5d, 0xa8, 0x4f, 0x7c, 0xae, 0xff, 0x35, - 0x94, 0xe8, 0x3b, 0xd4, 0xb4, 0xeb, 0x53, 0x73, 0x6b, 0x6d, 0xe8, 0xbc, 0x3a, 0xe5, 0x2a, 0x8f, - 0xbf, 0x99, 0x0b, 0xf9, 0x6a, 0x6e, 0x2a, 0x95, 0x33, 0x8b, 0xc8, 0x37, 0x96, 0x9b, 0xb0, 0x5c, - 0xe6, 0xf7, 0xa1, 0x49, 0x51, 0xb3, 0x6e, 0x40, 0xbd, 0xbb, 0x59, 0x61, 0x23, 0xdd, 0xb5, 0x19, - 0x27, 0xd7, 0xe4, 0x0d, 0x3b, 0x4e, 0x56, 0xe2, 0x47, 0xd0, 0x19, 0xc4, 0xd3, 0x99, 0xee, 0xe3, - 0xb7, 0x61, 0x1c, 0xea, 0x30, 0x89, 0xb3, 0x6e, 0x93, 0x5c, 0xdd, 0x2b, 0x33, 0x5a, 0xd0, 0x90, - 0x4b, 0x26, 0xe2, 0x27, 0x06, 0x9b, 0x15, 0x70, 0x45, 0xd2, 0x9e, 0x6f, 0xed, 0x66, 0xbe, 0x0f, - 0xa1, 0xf9, 0x24, 0xc4, 0xe8, 0x34, 0xeb, 0xd6, 0x49, 0xb1, 0xb7, 0x92, 0x0d, 0xa9, 0x49, 0xa7, - 0x2d, 0x7e, 0x65, 0xb0, 0x75, 0x9d, 0xc2, 0xb5, 0x6c, 0x7a, 0x00, 0x5f, 0xa6, 0xe1, 0x44, 0xa5, - 0xf3, 0xcf, 0x70, 0xee, 0x56, 0x78, 0x09, 0xe1, 0x5f, 0xc3, 0xdd, 0x8a, 0xaf, 0x4f, 0xc7, 0xb6, - 0x44, 0x96, 0xd4, 0xdb, 0x2b, 0x49, 0x59, 0x3d, 0xb9, 0xc2, 0x5c, 0xfc, 0xc3, 0xe0, 0xad, 0x6b, - 0xaf, 0x8a, 0x7e, 0x64, 0xe5, 0xd6, 0xdf, 0x87, 0xce, 0x73, 0xb3, 0x2a, 0xfa, 0x98, 0xe9, 0x30, - 0x56, 0x46, 0xd3, 0x35, 0xec, 0x12, 0xce, 0x07, 0xd0, 0x22, 0x6c, 0xa8, 0xa6, 0x8e, 0xe6, 0x7b, - 0xb7, 0xd0, 0x3c, 0xf0, 0xfa, 0x76, 0xa7, 0xe5, 0xe6, 0x86, 0x0c, 0x6d, 0x5d, 0xbf, 0xc2, 0x49, - 0xd8, 0x7e, 0x04, 0xeb, 0x0b, 0x06, 0xaf, 0xb4, 0xe7, 0x7e, 0x64, 0xb0, 0xe3, 0x97, 0xcb, 0x02, - 0x95, 0x9b, 0xc7, 0xd4, 0xbf, 0x5e, 0xad, 0xf4, 0x7a, 0x1f, 0x02, 0x14, 0xe6, 0x6e, 0x2b, 0xdc, - 0xd0, 0xb4, 0x25, 0x65, 0x71, 0x0c, 0x3b, 0x7e, 0x1b, 0xfe, 0x3f, 0x12, 0x42, 0x01, 0x3c, 0x4d, - 0x4e, 0x71, 0xa4, 0x95, 0x9e, 0x65, 0x46, 0xe3, 0x38, 0xc9, 0xb4, 0x6f, 0x32, 0x73, 0xa6, 0x6d, - 0xad, 0x95, 0xce, 0x37, 0x0c, 0x09, 0xfc, 0x5d, 0x78, 0x9d, 0x9c, 0xa2, 0xef, 0xa5, 0xcd, 0xca, - 0x02, 0x90, 0xfe, 0x5e, 0x3c, 0x82, 0xf5, 0xc3, 0x68, 0x96, 0x69, 0x4c, 0x5d, 0x94, 0x7d, 0x68, - 0x98, 0x98, 0xfe, 0x7b, 0xb5, 0x55, 0x58, 0x16, 0x54, 0xa4, 0x55, 0x11, 0x0f, 0x61, 0x8d, 0x5a, - 0x68, 0x34, 0x7e, 0x89, 0x13, 0x45, 0xf3, 0x67, 0xc7, 0x8a, 0x2d, 0xcd, 0xdf, 0xc2, 0x1c, 0x8d, - 0xa0, 0xb1, 0x7a, 0x6e, 0x38, 0x04, 0xf4, 0x47, 0xe3, 0x0a, 0x41, 0x3f, 0x33, 0x1d, 0xa8, 0x0f, - 0x43, 0xfb, 0x0c, 0x75, 0x69, 0x8e, 0x84, 0xa8, 0x4b, 0xea, 0x1d, 0x83, 0xa8, 0xcb, 0xc7, 0x9d, - 0xdf, 0xae, 0x7a, 0xec, 0xf7, 0xab, 0x1e, 0xfb, 0xf3, 0xaa, 0xc7, 0x7e, 0xfe, 0xab, 0xf7, 0xda, - 0x49, 0x93, 0x7e, 0xea, 0xde, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb5, 0x12, 0x7e, 0xdc, 0xe5, - 0x09, 0x00, 0x00, + 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0xa6, 0xdb, 0x49, 0xcc, 0x81, 0x23, + 0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0x8a, 0xba, 0xba, + 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x85, 0x5b, 0xd7, 0xeb, 0xd7, 0x55, 0xaf, 0x6a, 0xaa, 0xca, 0x86, + 0xf5, 0x69, 0x16, 0x9d, 0x4b, 0x8d, 0x07, 0xd3, 0x2c, 0xd5, 0x29, 0x6b, 0x45, 0x89, 0xc6, 0x2c, + 0x91, 0x31, 0xff, 0x0a, 0xda, 0x83, 0xe4, 0x14, 0x2f, 0x87, 0xa8, 0x25, 0xdb, 0x85, 0xb5, 0xc3, + 0x34, 0x9e, 0x4d, 0x92, 0x2f, 0xe5, 0x09, 0xc6, 0xdd, 0x60, 0x37, 0xd8, 0x6b, 0x8b, 0x32, 0x64, + 0x18, 0x2f, 0xa2, 0x09, 0x7e, 0x33, 0x93, 0x89, 0x9e, 0x4d, 0xba, 0x35, 0xcb, 0x28, 0x41, 0xfc, + 0xcf, 0x00, 0xda, 0xcf, 0x32, 0x39, 0x41, 0xf2, 0xb8, 0x0d, 0x2d, 0x91, 0x5e, 0x94, 0xdd, 0xe5, + 0x36, 0x7b, 0x0f, 0x36, 0x06, 0xc9, 0x39, 0x66, 0x0a, 0x8f, 0x12, 0x79, 0x12, 0xe3, 0x29, 0xb9, + 0x6b, 0x89, 0x0a, 0xca, 0x76, 0xa0, 0x7d, 0x28, 0xc7, 0xaf, 0xf1, 0xc5, 0x7c, 0x8a, 0xdd, 0x3a, + 0x39, 0x29, 0x80, 0xfc, 0x76, 0x14, 0xfd, 0x88, 0xdd, 0x70, 0x37, 0xd8, 0x5b, 0x17, 0x05, 0x50, + 0xd5, 0xdb, 0x58, 0xd2, 0xcb, 0x38, 0xbc, 0x25, 0x64, 0xf2, 0x2a, 0xd7, 0xd0, 0x24, 0x0d, 0x0b, + 0x18, 0xe7, 0xb0, 0x31, 0x98, 0x4c, 0xd3, 0x4c, 0x0b, 0x54, 0xd3, 0x34, 0x51, 0xc8, 0x3a, 0x50, + 0x3f, 0xca, 0x32, 0x97, 0x92, 0x39, 0xf2, 0x9f, 0xa0, 0xf3, 0x34, 0x4e, 0xc7, 0x67, 0x7d, 0xa9, + 0xa5, 0xc0, 0x1f, 0x66, 0xa8, 0x34, 0xdb, 0x82, 0x06, 0x15, 0xd7, 0xf1, 0xac, 0x61, 0x50, 0x2a, + 0x90, 0xab, 0x9e, 0x35, 0x0c, 0x4a, 0xef, 0x29, 0xc3, 0x50, 0x58, 0xc3, 0xa0, 0xa3, 0x38, 0x1a, + 0xdb, 0xcc, 0x42, 0x61, 0x0d, 0xc6, 0x20, 0x7c, 0x19, 0xe1, 0x85, 0x4b, 0x87, 0xce, 0x7c, 0x00, + 0xf7, 0x4a, 0xf1, 0x9d, 0xcc, 0xfb, 0xd0, 0x14, 0xe9, 0xc5, 0xa0, 0xaf, 0xba, 0xc1, 0x6e, 0x7d, + 0x2f, 0x14, 0xce, 0xa2, 0xa2, 0xd1, 0x57, 0x35, 0x57, 0x35, 0xba, 0x2a, 0x00, 0xfe, 0x00, 0x1a, + 0x54, 0x41, 0x93, 0x65, 0xf1, 0xd6, 0x1c, 0xf9, 0xaf, 0x01, 0xdc, 0x1b, 0xca, 0x4b, 0x92, 0xa1, + 0xf2, 0x30, 0xc7, 0xd0, 0xce, 0x41, 0x62, 0xaf, 0x3d, 0xda, 0x3f, 0xf0, 0x2d, 0x76, 0xb0, 0xc4, + 0x2f, 0x90, 0xa3, 0x44, 0x67, 0x73, 0x51, 0x3c, 0xde, 0xfe, 0x14, 0x36, 0x16, 0x2f, 0x8d, 0x86, + 0x33, 0x9c, 0xfb, 0x4a, 0x9f, 0xe1, 0xdc, 0xd4, 0xe4, 0x5c, 0xc6, 0x33, 0x5b, 0xbf, 0x50, 0x58, + 0xe3, 0x93, 0xda, 0x47, 0x01, 0xff, 0x0e, 0xd8, 0x61, 0x86, 0x52, 0x23, 0x39, 0x18, 0xa2, 0x52, + 0xf2, 0x15, 0xae, 0xfe, 0x0a, 0xb6, 0xb2, 0xb5, 0x72, 0x65, 0x77, 0xa0, 0x3d, 0x50, 0xae, 0xff, + 0xe8, 0x4b, 0xb4, 0x44, 0x01, 0xf0, 0x7d, 0x60, 0x7d, 0x8c, 0x51, 0xa3, 0x1b, 0x99, 0x1b, 0xfc, + 0xf3, 0x91, 0xd7, 0x72, 0x3b, 0x97, 0x3d, 0x84, 0xd0, 0x4c, 0x0b, 0x49, 0x59, 0x7b, 0xf4, 0x76, + 0x51, 0xba, 0x7c, 0x34, 0x05, 0x11, 0x78, 0xe4, 0x9d, 0xba, 0x09, 0xbb, 0x25, 0xc1, 0x6b, 0xda, + 0xcc, 0x87, 0xaa, 0x57, 0x43, 0xe5, 0x33, 0xeb, 0x42, 0x7d, 0xe6, 0x73, 0xfd, 0xbf, 0xa1, 0x78, + 0xdf, 0xa1, 0xa6, 0x5d, 0x9f, 0x9b, 0x5b, 0xfb, 0x86, 0xce, 0xab, 0x53, 0xae, 0xea, 0xf8, 0x27, + 0x70, 0x21, 0xef, 0xe6, 0xa6, 0x52, 0x39, 0xb3, 0x88, 0x7c, 0x63, 0xb9, 0x09, 0xcb, 0x6d, 0xf6, + 0x10, 0x9a, 0x14, 0x55, 0x75, 0x43, 0xea, 0xdd, 0xcd, 0x8a, 0x1a, 0xe1, 0xae, 0xcd, 0x38, 0xb9, + 0x26, 0x6f, 0xd8, 0x71, 0xb2, 0x16, 0x3b, 0x82, 0xce, 0x20, 0x99, 0xce, 0x74, 0x1f, 0xbf, 0x8f, + 0x92, 0x48, 0x47, 0x69, 0xa2, 0xba, 0x4d, 0x72, 0xf5, 0xa0, 0xac, 0x68, 0x81, 0x21, 0x96, 0x9e, + 0xf0, 0x9f, 0x03, 0xd8, 0xac, 0x80, 0x2b, 0x92, 0xf6, 0x7a, 0x6b, 0x37, 0xeb, 0x7d, 0x0c, 0xcd, + 0x67, 0x11, 0xc6, 0xa7, 0xaa, 0x5b, 0x27, 0x62, 0x6f, 0xa5, 0x1a, 0xa2, 0x09, 0xc7, 0xe6, 0xbf, + 0x05, 0xb0, 0x75, 0x1d, 0xe1, 0x5a, 0x35, 0x3d, 0x80, 0xaf, 0xb3, 0x68, 0x22, 0xb3, 0xf9, 0x17, + 0x38, 0x77, 0x2b, 0xbc, 0x84, 0xb0, 0x6f, 0xe1, 0x7e, 0xc5, 0xd7, 0xe7, 0x63, 0x5b, 0x22, 0x2b, + 0xea, 0xdd, 0x95, 0xa2, 0x2c, 0x4f, 0xac, 0x78, 0xce, 0xff, 0x0d, 0xe0, 0x9d, 0x6b, 0xaf, 0x8a, + 0x7e, 0x0c, 0xca, 0xad, 0xbf, 0x0f, 0x9d, 0x97, 0x66, 0x55, 0xf4, 0x51, 0xe9, 0x28, 0x91, 0x86, + 0xe9, 0x1a, 0x76, 0x09, 0x67, 0x03, 0x68, 0x11, 0x36, 0x94, 0x53, 0x27, 0xf3, 0x83, 0x5b, 0x64, + 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xf9, 0x73, 0x23, 0x86, 0xb6, 0xae, 0x5f, 0xe1, 0x64, 0x6c, 0x3f, + 0x81, 0xf5, 0x85, 0x07, 0x77, 0xda, 0x73, 0x29, 0xec, 0xf8, 0xdd, 0xb2, 0xa0, 0xe4, 0xe6, 0x29, + 0xfd, 0x18, 0xa0, 0xa0, 0xba, 0x05, 0x70, 0x43, 0x7f, 0x96, 0xc8, 0xfc, 0x18, 0x76, 0xfc, 0xe2, + 0xbb, 0x43, 0x40, 0xdf, 0x2d, 0xb5, 0xa2, 0x5b, 0xb8, 0x04, 0x78, 0x9e, 0x9e, 0xe2, 0x48, 0x4b, + 0x3d, 0x53, 0x86, 0x71, 0x9c, 0x2a, 0xed, 0xfb, 0xc9, 0x9c, 0x69, 0x31, 0x6b, 0xa9, 0xf3, 0x65, + 0x42, 0x06, 0x7b, 0x1f, 0xde, 0x24, 0xa7, 0xe8, 0xdb, 0x66, 0xb3, 0x32, 0xeb, 0xc2, 0xdf, 0xf3, + 0x27, 0xb0, 0x7e, 0x18, 0xcf, 0x94, 0xc6, 0xcc, 0x45, 0xd9, 0x87, 0x86, 0x89, 0xe9, 0x7f, 0x9a, + 0xb6, 0x8a, 0x97, 0x85, 0x14, 0x61, 0x29, 0xfc, 0x31, 0xac, 0x51, 0xb7, 0x8c, 0xc6, 0xaf, 0x71, + 0x22, 0x69, 0xd4, 0xec, 0x04, 0x05, 0x4b, 0xa3, 0xb6, 0x30, 0x32, 0x23, 0x68, 0xac, 0x1e, 0x11, + 0x06, 0x21, 0xfd, 0x79, 0x71, 0x85, 0xa0, 0xff, 0x2d, 0x1d, 0xa8, 0x0f, 0x23, 0xfb, 0x19, 0xea, + 0xc2, 0x1c, 0x09, 0x91, 0x97, 0xd4, 0x26, 0x06, 0x91, 0x97, 0x4f, 0x3b, 0xbf, 0x5f, 0xf5, 0x82, + 0x3f, 0xae, 0x7a, 0xc1, 0x5f, 0x57, 0xbd, 0xe0, 0x97, 0xbf, 0x7b, 0x6f, 0x9c, 0x34, 0xe9, 0xff, + 0xdb, 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x29, 0x07, 0x36, 0x04, 0xd0, 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index f69b0de7b..3173b12a2 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -103,7 +103,6 @@ message InputDefinitionAction { message CreateInputDefinitionMessage { string Index = 1; - string Name = 2; InputDefinition Definition = 3; } diff --git a/pilosa.go b/pilosa.go index 69e961df9..6d90c0d75 100644 --- a/pilosa.go +++ b/pilosa.go @@ -30,12 +30,17 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") - ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") + ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") + + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionPrimaryKey = errors.New("input-definition can only contain one PrimaryKey") + ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") + ErrInputDefinitionNameRequired = errors.New("input-definition name required") + ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") From d87f1bbca1fa088670eefecce7e82e87dbc5f5ae Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 14:06:12 -0500 Subject: [PATCH 47/99] Input Definition Struct encoding error tests --- input_definition.go | 2 +- input_definition_test.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/input_definition.go b/input_definition.go index a8e2e709e..aba684fb6 100644 --- a/input_definition.go +++ b/input_definition.go @@ -232,7 +232,7 @@ func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) return &field, nil } -// Action descripes the mapping method for the field in the InputDefinition. +// Action describes the mapping method for the field in the InputDefinition. type Action struct { Frame string `json:"frame,omitempty"` ValueDestination string `json:"valueDestination,omitempty"` diff --git a/input_definition_test.go b/input_definition_test.go index 753681e51..925adb724 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -120,12 +120,6 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} - _, err = act.Encode() - if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { - t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) - } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -159,6 +153,21 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } } +func TestActionEncoding(t *testing.T) { + action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err := action.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } + + field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} + info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} + _, err = info.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } +} + func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) From 1f90461a092fa74a26552825e48f9666c4e141a4 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 27 Jun 2017 16:06:21 -0500 Subject: [PATCH 48/99] rebase input-definition --- handler.go | 57 ++++++++------ handler_test.go | 154 ++++++++++++++++++++++++++++++++++---- index.go | 22 +++--- index_test.go | 33 +++++---- input_definition.go | 15 +--- input_definition_test.go | 12 +-- internal/private.pb.go | 155 ++++++++++++++------------------------- internal/private.proto | 1 - pilosa.go | 8 +- server/server_test.go | 2 +- 10 files changed, 276 insertions(+), 183 deletions(-) diff --git a/handler.go b/handler.go index a44f05ae9..99695ee7e 100644 --- a/handler.go +++ b/handler.go @@ -1500,10 +1500,18 @@ func errorString(err error) string { return err.Error() } +// handlePostInputDefinition handles POST /input-definition request. func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1512,13 +1520,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - + // Encode InputDefinition to its internal representation. def, err := req.Encode() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1526,6 +1528,22 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } def.Name = inputDefName + // Validate columnLabel and duplicate primaryKey. + numPrimaryKey := 0 + for _, field := range def.Fields { + if field.PrimaryKey { + numPrimaryKey += 1 + if field.Name != index.columnLabel { + http.Error(w, ErrInputDefinitionColumnLabel.Error(), http.StatusBadRequest) + return + } + } + } + if numPrimaryKey > 1 { + http.Error(w, ErrInputDefinitionPrimaryKey.Error(), http.StatusBadRequest) + return + } + // Create InputDefinition. _, err = index.CreateInputDefinition(def) if err == ErrInputDefinitionExists { @@ -1539,32 +1557,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque err = h.Broadcaster.SendSync( &internal.CreateInputDefinitionMessage{ Index: indexName, - Name: inputDefName, Definition: def, }) if err != nil { h.logger().Printf("problem sending CreateInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } +// handleGetInputDefinition handles GET /input-definition request. func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - //Find index. + // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } + inputDef, _ := index.inputDefinitions[inputDefName] - //inputInfo := InputDefinitionInfo{Frames: inputDef.frames, Fields: inputDef.fields} if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ Frames: inputDef.frames, Fields: inputDef.fields, @@ -1574,6 +1590,7 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques } +// handleDeleteInputDefinition handles DELETE /input-definition request. func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] @@ -1581,9 +1598,7 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req // Find index. index := h.Holder.Index(indexName) if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) - } + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } @@ -1599,15 +1614,15 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req Name: inputDefName, }) if err != nil { - h.logger().Printf("problem sending CreateInputDefinition message: %s", err) + h.logger().Printf("problem sending DeleteInputDefinition message: %s", err) } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type postInputDefinitionResponse struct{} +type defaultInputDefinitionResponse struct{} func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] @@ -1644,7 +1659,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 37fd9f25c..05c0db4d4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1084,7 +1084,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { }], "fields": [ { - "name": "id", + "name": "columnID", "primaryKey": true }, { @@ -1112,14 +1112,133 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusConflict { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionExists.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + +} + +// Ensure throwing error if there's duplicated primaryKey field. +func TestHandler_DuplicatePrimaryKey(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + + //Ensure throwing error if there's duplicated primaryKey field + invalidPrimaryKey := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + }, + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionPrimaryKey.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Eusure throwing error if primary field's name doesn't match columnLabel + hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{ColumnLabel: "id"}) + unmatchColumnBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + } + ] + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Eusure throwing error if request body is invalid + jsonErrorBody := []byte(` + { + "frames":[{ + "name":"event-time", + "options":{ + "timeQuantum": "YMD", + "inverseEnabled": false, + "cacheType": "ranked" + } + }], + "fields": [ + { + "name": "columnID", + "primaryKey": true + + + }`) + + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `unexpected EOF`+"\n" { + t.Fatalf("unexpected body: %s", body) + } + } // Ensure handler can delete a input definition. func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + // Test index not found + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + + // Test input definition is deleted + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} @@ -1128,11 +1247,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { if err != nil { t.Fatal(err) } - - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -1143,30 +1258,41 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } } -// Return existing input definition +// Ensure handler can get existing input definition func TestHandler_GetInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + + // Return error if index does not exist + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) + } + + // Return existing input definition + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) inputDef, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } - response := &pilosa.InputDefinitionInfo{Frames: inputDef.Frames(), Fields: inputDef.Fields()} expect, err := json.Marshal(response) if err != nil { t.Fatal(err) } - h := NewHandler() - h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - w := httptest.NewRecorder() + + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) diff --git a/index.go b/index.go index 7c97c0498..54a8f1433 100644 --- a/index.go +++ b/index.go @@ -51,14 +51,14 @@ type Index struct { // Frames by name. frames map[string]*Frame - // Max Slice on any node in the cluster, according to this node + // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - // Column attribute storage and cache + // Column attribute storage and cache. columnAttrStore *AttrStore - // InputDefinition by name + // InputDefinitions by name. inputDefinitions map[string]*InputDefinition broadcaster Broadcaster @@ -336,7 +336,7 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } -// InputDefinitionPath returns the path to a inputdefinition in the index. +// InputDefinitionPath returns the path to an input definition in the index. func (i *Index) InputDefinitionPath() string { return filepath.Join(i.path, InputDefinitionDir) } @@ -618,10 +618,10 @@ type IndexOptions struct { } // Encode converts i into its internal representation. -func (o *IndexOptions) Encode() *internal.IndexMeta { +func (i *IndexOptions) Encode() *internal.IndexMeta { return &internal.IndexMeta{ - ColumnLabel: o.ColumnLabel, - TimeQuantum: string(o.TimeQuantum), + ColumnLabel: i.ColumnLabel, + TimeQuantum: string(i.TimeQuantum), } } @@ -656,9 +656,9 @@ func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefin func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { if pb.Name == "" { - return nil, errors.New("input-definition name required") + return nil, ErrInputDefinitionNameRequired } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { - return nil, errors.New("frames and fields are required") + return nil, ErrInputDefinitionAttrsRequired } for _, fr := range pb.Frames { @@ -702,7 +702,7 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { return inputDef, nil } -// DeleteInputDefinition removes a input definition from the index. +// DeleteInputDefinition removes an input definition from the index. func (i *Index) DeleteInputDefinition(name string) error { i.mu.Lock() defer i.mu.Unlock() @@ -742,7 +742,7 @@ func (i *Index) openInputDefinition() error { input.Open() i.inputDefinitions[file.Name()] = input - // Create frame if it doesn't exist + // Create frame if it doesn't exist. for _, fr := range input.frames { _, err := i.CreateFrame(fr.Name, fr.Options) if err == ErrFrameExists { diff --git a/index_test.go b/index_test.go index 8a7367636..aaebe288d 100644 --- a/index_test.go +++ b/index_test.go @@ -379,16 +379,31 @@ func TestIndex_CreateInputDefinition(t *testing.T) { } } +// Ensure create input definition handle correct error func TestIndex_CreateExistingInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() + // Test frames and fields are required + def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionAttrsRequired { + t.Fatal(err) + } + + //Test input definition name is required + def = internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err = index.CreateInputDefinition(&def) + if err != pilosa.ErrInputDefinitionNameRequired { + t.Fatal(err) + } + // Create Input Definition. frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} - _, err := index.CreateInputDefinition(&def) + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + _, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } @@ -398,18 +413,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { } } -func TestIndex_CreateEmptyInputDefinition(t *testing.T) { - index := MustOpenIndex() - defer index.Close() - - // Create Input Definition. - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} - _, err := index.CreateInputDefinition(&def) - if err.Error() != "frames and fields are required" { - t.Fatal(err) - } -} - +// Ensure to delete existing input definition. func TestIndex_DeleteInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() @@ -434,6 +438,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { } } +// Ensure that frame in input definition will be created when server restart func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() diff --git a/input_definition.go b/input_definition.go index aba684fb6..bef566416 100644 --- a/input_definition.go +++ b/input_definition.go @@ -82,7 +82,7 @@ func (i *InputDefinition) Open() error { return nil } -// LoadDefinition loads the protobuf format of a defition +// LoadDefinition loads the protobuf format of a definition. func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { // Copy metadata fields. i.name = pb.Name @@ -101,7 +101,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } - numPrimaryKey := 0 countRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action @@ -123,13 +122,6 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { RowID: &action.RowID, }) } - if field.PrimaryKey { - numPrimaryKey++ - } - - if numPrimaryKey > 1 { - return errors.New("duplicate primaryKey with other field") - } inputField := InputDefinitionField{ Name: field.Name, @@ -155,7 +147,7 @@ func (i *InputDefinition) loadMeta() error { return i.LoadDefinition(&pb) } -//saveMeta writes meta data for the input definition file. +// saveMeta writes meta data for the input definition file. func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { return err @@ -253,6 +245,7 @@ func (o *Action) Encode() (*internal.InputDefinitionAction, error) { }, nil } +// convert pointer to uint64 func convert(x *uint64) uint64 { if x != nil { return *x @@ -266,7 +259,7 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -// InputDefinitionInfo the json message format to create an InputDefinition. +// InputDefinitionInfo represents the json message format needed to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` Fields []InputDefinitionField `json:"fields"` diff --git a/input_definition_test.go b/input_definition_test.go index 925adb724..112d8eac9 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -129,16 +129,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 0} - field1 := internal.InputDefinitionField{Name: "newID", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} - err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "duplicate primaryKey with other field") { - t.Fatalf("Expected duplicate primaryKey error, actual error: %s", err) - } - - action1 = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - field1 = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} + field1 := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "duplicate rowID with other field") { diff --git a/internal/private.pb.go b/internal/private.pb.go index 8194a13de..abde718e9 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -314,7 +314,6 @@ func (m *InputDefinitionAction) GetValueMap() map[string]uint64 { type CreateInputDefinitionMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` Definition *InputDefinition `protobuf:"bytes,3,opt,name=Definition" json:"Definition,omitempty"` } @@ -1160,12 +1159,6 @@ func (m *CreateInputDefinitionMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Name) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } if m.Definition != nil { dAtA[i] = 0x1a i++ @@ -1695,10 +1688,6 @@ func (m *CreateInputDefinitionMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Name) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } if m.Definition != nil { l = m.Definition.Size() n += 1 + l + sovPrivate(uint64(l)) @@ -4284,35 +4273,6 @@ func (m *CreateInputDefinitionMessage) Unmarshal(dAtA []byte) error { } m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Definition", wireType) @@ -5030,63 +4990,62 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 915 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xa6, 0xed, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, + // 912 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, + 0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, 0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd, - 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0x66, 0xda, 0x49, 0xcc, 0x81, 0x0b, - 0x12, 0xcf, 0x80, 0xc4, 0x91, 0x97, 0xe1, 0x08, 0x6f, 0x80, 0xc2, 0x85, 0x37, 0xe0, 0x8a, 0xba, - 0xba, 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x2d, 0xdc, 0xba, 0xbe, 0xae, 0x9f, 0xaf, 0xaa, 0xab, 0x6a, - 0x06, 0xd6, 0xa7, 0x69, 0x78, 0xae, 0x34, 0x1e, 0x4c, 0xd3, 0x44, 0x27, 0xbc, 0x15, 0xc6, 0x1a, - 0xd3, 0x58, 0x45, 0xe2, 0x0b, 0x68, 0x0f, 0xe2, 0x53, 0xbc, 0x1c, 0xa2, 0x56, 0x7c, 0x17, 0xd6, - 0x0e, 0x93, 0x68, 0x36, 0x89, 0x3f, 0x57, 0x27, 0x18, 0x75, 0xd9, 0x2e, 0xdb, 0x6b, 0xcb, 0x32, - 0x64, 0x34, 0x9e, 0x85, 0x13, 0xfc, 0x6a, 0xa6, 0x62, 0x3d, 0x9b, 0x74, 0x6b, 0x56, 0xa3, 0x04, - 0x89, 0x3f, 0x18, 0xb4, 0x9f, 0xa4, 0x6a, 0x82, 0xe4, 0x71, 0x1b, 0x5a, 0x32, 0xb9, 0x28, 0xbb, - 0xcb, 0x65, 0xfe, 0x0e, 0x6c, 0x0c, 0xe2, 0x73, 0x4c, 0x33, 0x3c, 0x8a, 0xd5, 0x49, 0x84, 0xa7, - 0xe4, 0xae, 0x25, 0x2b, 0x28, 0xdf, 0x81, 0xf6, 0xa1, 0x1a, 0xbf, 0xc4, 0x67, 0xf3, 0x29, 0x76, - 0xeb, 0xe4, 0xa4, 0x00, 0xf2, 0xdb, 0x51, 0xf8, 0x3d, 0x76, 0x83, 0x5d, 0xb6, 0xb7, 0x2e, 0x0b, - 0xa0, 0xca, 0xb7, 0xb1, 0xc4, 0x97, 0x0b, 0x78, 0x43, 0xaa, 0xf8, 0x45, 0xce, 0xa1, 0x49, 0x1c, - 0x16, 0x30, 0x21, 0x60, 0x63, 0x30, 0x99, 0x26, 0xa9, 0x96, 0x98, 0x4d, 0x93, 0x38, 0x43, 0xde, - 0x81, 0xfa, 0x51, 0x9a, 0xba, 0x94, 0xcc, 0x51, 0xfc, 0x00, 0x9d, 0xc7, 0x51, 0x32, 0x3e, 0xeb, - 0x2b, 0xad, 0x24, 0x7e, 0x37, 0xc3, 0x4c, 0xf3, 0x2d, 0x68, 0x50, 0x71, 0x9d, 0x9e, 0x15, 0x0c, - 0x4a, 0x05, 0x72, 0xd5, 0xb3, 0x82, 0x41, 0xc9, 0x9e, 0x32, 0x0c, 0xa4, 0x15, 0x0c, 0x3a, 0x8a, - 0xc2, 0xb1, 0xcd, 0x2c, 0x90, 0x56, 0xe0, 0x1c, 0x82, 0xe7, 0x21, 0x5e, 0xb8, 0x74, 0xe8, 0x2c, - 0x06, 0x70, 0xa7, 0x14, 0xdf, 0xd1, 0xbc, 0x0b, 0x4d, 0x99, 0x5c, 0x0c, 0xfa, 0x59, 0x97, 0xed, - 0xd6, 0xf7, 0x02, 0xe9, 0x24, 0x2a, 0x1a, 0xbd, 0xaa, 0xb9, 0xaa, 0xd1, 0x55, 0x01, 0x88, 0x7b, - 0xd0, 0xa0, 0x0a, 0x9a, 0x2c, 0x0b, 0x5b, 0x73, 0x14, 0xbf, 0x30, 0xb8, 0x33, 0x54, 0x97, 0x44, - 0x23, 0xcb, 0xc3, 0x1c, 0x43, 0x3b, 0x07, 0x49, 0x7b, 0xed, 0xc1, 0xfe, 0x81, 0x6f, 0xb1, 0x83, - 0x25, 0xfd, 0x02, 0x39, 0x8a, 0x75, 0x3a, 0x97, 0x85, 0xf1, 0xf6, 0xc7, 0xb0, 0xb1, 0x78, 0x69, - 0x38, 0x9c, 0xe1, 0xdc, 0x57, 0xfa, 0x0c, 0xe7, 0xa6, 0x26, 0xe7, 0x2a, 0x9a, 0xd9, 0xfa, 0x05, - 0xd2, 0x0a, 0x1f, 0xd5, 0x3e, 0x60, 0xe2, 0x1b, 0xe0, 0x87, 0x29, 0x2a, 0x8d, 0xe4, 0x60, 0x88, - 0x59, 0xa6, 0x5e, 0xe0, 0xea, 0x57, 0xb0, 0x95, 0xad, 0x95, 0x2b, 0xbb, 0x03, 0xed, 0x41, 0xe6, - 0xfa, 0x8f, 0x5e, 0xa2, 0x25, 0x0b, 0x40, 0xec, 0x03, 0xef, 0x63, 0x84, 0x1a, 0xdd, 0xc8, 0xdc, - 0xe0, 0x5f, 0x8c, 0x3c, 0x97, 0xdb, 0x75, 0xf9, 0x7d, 0x08, 0xcc, 0xb4, 0x10, 0x95, 0xb5, 0x07, - 0x6f, 0x16, 0xa5, 0xcb, 0x47, 0x53, 0x92, 0x82, 0x08, 0xbd, 0x53, 0x37, 0x61, 0xb7, 0x24, 0x78, - 0x4d, 0x9b, 0xf9, 0x50, 0xf5, 0x6a, 0xa8, 0x7c, 0x66, 0x5d, 0xa8, 0x4f, 0x7c, 0xae, 0xff, 0x35, - 0x94, 0xe8, 0x3b, 0xd4, 0xb4, 0xeb, 0x53, 0x73, 0x6b, 0x6d, 0xe8, 0xbc, 0x3a, 0xe5, 0x2a, 0x8f, - 0xbf, 0x99, 0x0b, 0xf9, 0x6a, 0x6e, 0x2a, 0x95, 0x33, 0x8b, 0xc8, 0x37, 0x96, 0x9b, 0xb0, 0x5c, - 0xe6, 0xf7, 0xa1, 0x49, 0x51, 0xb3, 0x6e, 0x40, 0xbd, 0xbb, 0x59, 0x61, 0x23, 0xdd, 0xb5, 0x19, - 0x27, 0xd7, 0xe4, 0x0d, 0x3b, 0x4e, 0x56, 0xe2, 0x47, 0xd0, 0x19, 0xc4, 0xd3, 0x99, 0xee, 0xe3, - 0xb7, 0x61, 0x1c, 0xea, 0x30, 0x89, 0xb3, 0x6e, 0x93, 0x5c, 0xdd, 0x2b, 0x33, 0x5a, 0xd0, 0x90, - 0x4b, 0x26, 0xe2, 0x27, 0x06, 0x9b, 0x15, 0x70, 0x45, 0xd2, 0x9e, 0x6f, 0xed, 0x66, 0xbe, 0x0f, - 0xa1, 0xf9, 0x24, 0xc4, 0xe8, 0x34, 0xeb, 0xd6, 0x49, 0xb1, 0xb7, 0x92, 0x0d, 0xa9, 0x49, 0xa7, - 0x2d, 0x7e, 0x65, 0xb0, 0x75, 0x9d, 0xc2, 0xb5, 0x6c, 0x7a, 0x00, 0x5f, 0xa6, 0xe1, 0x44, 0xa5, - 0xf3, 0xcf, 0x70, 0xee, 0x56, 0x78, 0x09, 0xe1, 0x5f, 0xc3, 0xdd, 0x8a, 0xaf, 0x4f, 0xc7, 0xb6, - 0x44, 0x96, 0xd4, 0xdb, 0x2b, 0x49, 0x59, 0x3d, 0xb9, 0xc2, 0x5c, 0xfc, 0xc3, 0xe0, 0xad, 0x6b, - 0xaf, 0x8a, 0x7e, 0x64, 0xe5, 0xd6, 0xdf, 0x87, 0xce, 0x73, 0xb3, 0x2a, 0xfa, 0x98, 0xe9, 0x30, - 0x56, 0x46, 0xd3, 0x35, 0xec, 0x12, 0xce, 0x07, 0xd0, 0x22, 0x6c, 0xa8, 0xa6, 0x8e, 0xe6, 0x7b, - 0xb7, 0xd0, 0x3c, 0xf0, 0xfa, 0x76, 0xa7, 0xe5, 0xe6, 0x86, 0x0c, 0x6d, 0x5d, 0xbf, 0xc2, 0x49, - 0xd8, 0x7e, 0x04, 0xeb, 0x0b, 0x06, 0xaf, 0xb4, 0xe7, 0x7e, 0x64, 0xb0, 0xe3, 0x97, 0xcb, 0x02, - 0x95, 0x9b, 0xc7, 0xd4, 0xbf, 0x5e, 0xad, 0xf4, 0x7a, 0x1f, 0x02, 0x14, 0xe6, 0x6e, 0x2b, 0xdc, - 0xd0, 0xb4, 0x25, 0x65, 0x71, 0x0c, 0x3b, 0x7e, 0x1b, 0xfe, 0x3f, 0x12, 0x42, 0x01, 0x3c, 0x4d, - 0x4e, 0x71, 0xa4, 0x95, 0x9e, 0x65, 0x46, 0xe3, 0x38, 0xc9, 0xb4, 0x6f, 0x32, 0x73, 0xa6, 0x6d, - 0xad, 0x95, 0xce, 0x37, 0x0c, 0x09, 0xfc, 0x5d, 0x78, 0x9d, 0x9c, 0xa2, 0xef, 0xa5, 0xcd, 0xca, - 0x02, 0x90, 0xfe, 0x5e, 0x3c, 0x82, 0xf5, 0xc3, 0x68, 0x96, 0x69, 0x4c, 0x5d, 0x94, 0x7d, 0x68, - 0x98, 0x98, 0xfe, 0x7b, 0xb5, 0x55, 0x58, 0x16, 0x54, 0xa4, 0x55, 0x11, 0x0f, 0x61, 0x8d, 0x5a, - 0x68, 0x34, 0x7e, 0x89, 0x13, 0x45, 0xf3, 0x67, 0xc7, 0x8a, 0x2d, 0xcd, 0xdf, 0xc2, 0x1c, 0x8d, - 0xa0, 0xb1, 0x7a, 0x6e, 0x38, 0x04, 0xf4, 0x47, 0xe3, 0x0a, 0x41, 0x3f, 0x33, 0x1d, 0xa8, 0x0f, - 0x43, 0xfb, 0x0c, 0x75, 0x69, 0x8e, 0x84, 0xa8, 0x4b, 0xea, 0x1d, 0x83, 0xa8, 0xcb, 0xc7, 0x9d, - 0xdf, 0xae, 0x7a, 0xec, 0xf7, 0xab, 0x1e, 0xfb, 0xf3, 0xaa, 0xc7, 0x7e, 0xfe, 0xab, 0xf7, 0xda, - 0x49, 0x93, 0x7e, 0xea, 0xde, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb5, 0x12, 0x7e, 0xdc, 0xe5, - 0x09, 0x00, 0x00, + 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0xa6, 0xdb, 0x49, 0xcc, 0x81, 0x23, + 0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0x8a, 0xba, 0xba, + 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x85, 0x5b, 0xd7, 0xeb, 0xd7, 0x55, 0xaf, 0x6a, 0xaa, 0xca, 0x86, + 0xf5, 0x69, 0x16, 0x9d, 0x4b, 0x8d, 0x07, 0xd3, 0x2c, 0xd5, 0x29, 0x6b, 0x45, 0x89, 0xc6, 0x2c, + 0x91, 0x31, 0xff, 0x0a, 0xda, 0x83, 0xe4, 0x14, 0x2f, 0x87, 0xa8, 0x25, 0xdb, 0x85, 0xb5, 0xc3, + 0x34, 0x9e, 0x4d, 0x92, 0x2f, 0xe5, 0x09, 0xc6, 0xdd, 0x60, 0x37, 0xd8, 0x6b, 0x8b, 0x32, 0x64, + 0x18, 0x2f, 0xa2, 0x09, 0x7e, 0x33, 0x93, 0x89, 0x9e, 0x4d, 0xba, 0x35, 0xcb, 0x28, 0x41, 0xfc, + 0xcf, 0x00, 0xda, 0xcf, 0x32, 0x39, 0x41, 0xf2, 0xb8, 0x0d, 0x2d, 0x91, 0x5e, 0x94, 0xdd, 0xe5, + 0x36, 0x7b, 0x0f, 0x36, 0x06, 0xc9, 0x39, 0x66, 0x0a, 0x8f, 0x12, 0x79, 0x12, 0xe3, 0x29, 0xb9, + 0x6b, 0x89, 0x0a, 0xca, 0x76, 0xa0, 0x7d, 0x28, 0xc7, 0xaf, 0xf1, 0xc5, 0x7c, 0x8a, 0xdd, 0x3a, + 0x39, 0x29, 0x80, 0xfc, 0x76, 0x14, 0xfd, 0x88, 0xdd, 0x70, 0x37, 0xd8, 0x5b, 0x17, 0x05, 0x50, + 0xd5, 0xdb, 0x58, 0xd2, 0xcb, 0x38, 0xbc, 0x25, 0x64, 0xf2, 0x2a, 0xd7, 0xd0, 0x24, 0x0d, 0x0b, + 0x18, 0xe7, 0xb0, 0x31, 0x98, 0x4c, 0xd3, 0x4c, 0x0b, 0x54, 0xd3, 0x34, 0x51, 0xc8, 0x3a, 0x50, + 0x3f, 0xca, 0x32, 0x97, 0x92, 0x39, 0xf2, 0x9f, 0xa0, 0xf3, 0x34, 0x4e, 0xc7, 0x67, 0x7d, 0xa9, + 0xa5, 0xc0, 0x1f, 0x66, 0xa8, 0x34, 0xdb, 0x82, 0x06, 0x15, 0xd7, 0xf1, 0xac, 0x61, 0x50, 0x2a, + 0x90, 0xab, 0x9e, 0x35, 0x0c, 0x4a, 0xef, 0x29, 0xc3, 0x50, 0x58, 0xc3, 0xa0, 0xa3, 0x38, 0x1a, + 0xdb, 0xcc, 0x42, 0x61, 0x0d, 0xc6, 0x20, 0x7c, 0x19, 0xe1, 0x85, 0x4b, 0x87, 0xce, 0x7c, 0x00, + 0xf7, 0x4a, 0xf1, 0x9d, 0xcc, 0xfb, 0xd0, 0x14, 0xe9, 0xc5, 0xa0, 0xaf, 0xba, 0xc1, 0x6e, 0x7d, + 0x2f, 0x14, 0xce, 0xa2, 0xa2, 0xd1, 0x57, 0x35, 0x57, 0x35, 0xba, 0x2a, 0x00, 0xfe, 0x00, 0x1a, + 0x54, 0x41, 0x93, 0x65, 0xf1, 0xd6, 0x1c, 0xf9, 0xaf, 0x01, 0xdc, 0x1b, 0xca, 0x4b, 0x92, 0xa1, + 0xf2, 0x30, 0xc7, 0xd0, 0xce, 0x41, 0x62, 0xaf, 0x3d, 0xda, 0x3f, 0xf0, 0x2d, 0x76, 0xb0, 0xc4, + 0x2f, 0x90, 0xa3, 0x44, 0x67, 0x73, 0x51, 0x3c, 0xde, 0xfe, 0x14, 0x36, 0x16, 0x2f, 0x8d, 0x86, + 0x33, 0x9c, 0xfb, 0x4a, 0x9f, 0xe1, 0xdc, 0xd4, 0xe4, 0x5c, 0xc6, 0x33, 0x5b, 0xbf, 0x50, 0x58, + 0xe3, 0x93, 0xda, 0x47, 0x01, 0xff, 0x0e, 0xd8, 0x61, 0x86, 0x52, 0x23, 0x39, 0x18, 0xa2, 0x52, + 0xf2, 0x15, 0xae, 0xfe, 0x0a, 0xb6, 0xb2, 0xb5, 0x72, 0x65, 0x77, 0xa0, 0x3d, 0x50, 0xae, 0xff, + 0xe8, 0x4b, 0xb4, 0x44, 0x01, 0xf0, 0x7d, 0x60, 0x7d, 0x8c, 0x51, 0xa3, 0x1b, 0x99, 0x1b, 0xfc, + 0xf3, 0x91, 0xd7, 0x72, 0x3b, 0x97, 0x3d, 0x84, 0xd0, 0x4c, 0x0b, 0x49, 0x59, 0x7b, 0xf4, 0x76, + 0x51, 0xba, 0x7c, 0x34, 0x05, 0x11, 0x78, 0xe4, 0x9d, 0xba, 0x09, 0xbb, 0x25, 0xc1, 0x6b, 0xda, + 0xcc, 0x87, 0xaa, 0x57, 0x43, 0xe5, 0x33, 0xeb, 0x42, 0x7d, 0xe6, 0x73, 0xfd, 0xbf, 0xa1, 0x78, + 0xdf, 0xa1, 0xa6, 0x5d, 0x9f, 0x9b, 0x5b, 0xfb, 0x86, 0xce, 0xab, 0x53, 0xae, 0xea, 0xf8, 0x27, + 0x70, 0x21, 0xef, 0xe6, 0xa6, 0x52, 0x39, 0xb3, 0x88, 0x7c, 0x63, 0xb9, 0x09, 0xcb, 0x6d, 0xf6, + 0x10, 0x9a, 0x14, 0x55, 0x75, 0x43, 0xea, 0xdd, 0xcd, 0x8a, 0x1a, 0xe1, 0xae, 0xcd, 0x38, 0xb9, + 0x26, 0x6f, 0xd8, 0x71, 0xb2, 0x16, 0x3b, 0x82, 0xce, 0x20, 0x99, 0xce, 0x74, 0x1f, 0xbf, 0x8f, + 0x92, 0x48, 0x47, 0x69, 0xa2, 0xba, 0x4d, 0x72, 0xf5, 0xa0, 0xac, 0x68, 0x81, 0x21, 0x96, 0x9e, + 0xf0, 0x9f, 0x03, 0xd8, 0xac, 0x80, 0x2b, 0x92, 0xf6, 0x7a, 0x6b, 0x37, 0xeb, 0x7d, 0x0c, 0xcd, + 0x67, 0x11, 0xc6, 0xa7, 0xaa, 0x5b, 0x27, 0x62, 0x6f, 0xa5, 0x1a, 0xa2, 0x09, 0xc7, 0xe6, 0xbf, + 0x05, 0xb0, 0x75, 0x1d, 0xe1, 0x5a, 0x35, 0x3d, 0x80, 0xaf, 0xb3, 0x68, 0x22, 0xb3, 0xf9, 0x17, + 0x38, 0x77, 0x2b, 0xbc, 0x84, 0xb0, 0x6f, 0xe1, 0x7e, 0xc5, 0xd7, 0xe7, 0x63, 0x5b, 0x22, 0x2b, + 0xea, 0xdd, 0x95, 0xa2, 0x2c, 0x4f, 0xac, 0x78, 0xce, 0xff, 0x0d, 0xe0, 0x9d, 0x6b, 0xaf, 0x8a, + 0x7e, 0x0c, 0xca, 0xad, 0xbf, 0x0f, 0x9d, 0x97, 0x66, 0x55, 0xf4, 0x51, 0xe9, 0x28, 0x91, 0x86, + 0xe9, 0x1a, 0x76, 0x09, 0x67, 0x03, 0x68, 0x11, 0x36, 0x94, 0x53, 0x27, 0xf3, 0x83, 0x5b, 0x64, + 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xf9, 0x73, 0x23, 0x86, 0xb6, 0xae, 0x5f, 0xe1, 0x64, 0x6c, 0x3f, + 0x81, 0xf5, 0x85, 0x07, 0x77, 0xda, 0x73, 0x29, 0xec, 0xf8, 0xdd, 0xb2, 0xa0, 0xe4, 0xe6, 0x29, + 0xfd, 0x18, 0xa0, 0xa0, 0xba, 0x05, 0x70, 0x43, 0x7f, 0x96, 0xc8, 0xfc, 0x18, 0x76, 0xfc, 0xe2, + 0xbb, 0x43, 0x40, 0xdf, 0x2d, 0xb5, 0xa2, 0x5b, 0xb8, 0x04, 0x78, 0x9e, 0x9e, 0xe2, 0x48, 0x4b, + 0x3d, 0x53, 0x86, 0x71, 0x9c, 0x2a, 0xed, 0xfb, 0xc9, 0x9c, 0x69, 0x31, 0x6b, 0xa9, 0xf3, 0x65, + 0x42, 0x06, 0x7b, 0x1f, 0xde, 0x24, 0xa7, 0xe8, 0xdb, 0x66, 0xb3, 0x32, 0xeb, 0xc2, 0xdf, 0xf3, + 0x27, 0xb0, 0x7e, 0x18, 0xcf, 0x94, 0xc6, 0xcc, 0x45, 0xd9, 0x87, 0x86, 0x89, 0xe9, 0x7f, 0x9a, + 0xb6, 0x8a, 0x97, 0x85, 0x14, 0x61, 0x29, 0xfc, 0x31, 0xac, 0x51, 0xb7, 0x8c, 0xc6, 0xaf, 0x71, + 0x22, 0x69, 0xd4, 0xec, 0x04, 0x05, 0x4b, 0xa3, 0xb6, 0x30, 0x32, 0x23, 0x68, 0xac, 0x1e, 0x11, + 0x06, 0x21, 0xfd, 0x79, 0x71, 0x85, 0xa0, 0xff, 0x2d, 0x1d, 0xa8, 0x0f, 0x23, 0xfb, 0x19, 0xea, + 0xc2, 0x1c, 0x09, 0x91, 0x97, 0xd4, 0x26, 0x06, 0x91, 0x97, 0x4f, 0x3b, 0xbf, 0x5f, 0xf5, 0x82, + 0x3f, 0xae, 0x7a, 0xc1, 0x5f, 0x57, 0xbd, 0xe0, 0x97, 0xbf, 0x7b, 0x6f, 0x9c, 0x34, 0xe9, 0xff, + 0xdb, 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x29, 0x07, 0x36, 0x04, 0xd0, 0x09, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index f69b0de7b..3173b12a2 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -103,7 +103,6 @@ message InputDefinitionAction { message CreateInputDefinitionMessage { string Index = 1; - string Name = 2; InputDefinition Definition = 3; } diff --git a/pilosa.go b/pilosa.go index 3edcbfad9..bde8cc2c7 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,8 +36,11 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrInputDefinitionNotFound = errors.New("input-definition not found") + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionPrimaryKey = errors.New("input-definition can only contain one PrimaryKey") + ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") + ErrInputDefinitionNameRequired = errors.New("input-definition name required") + ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") @@ -45,6 +48,7 @@ var ( ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") ErrRangeCacheNotAllowed = errors.New("range cache not allowed") ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/server/server_test.go b/server/server_test.go index 6f4570709..b6af8c423 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -540,7 +540,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { "cacheType": "ranked", "timeQuantum": "YMD" }}], - "fields": [{"name": "id", + "fields": [{"name": "columnID", "primaryKey": true }]} `); err != nil { From 13da69692760c626ffceb94c36c2e3f56f88264e Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 12:41:52 -0500 Subject: [PATCH 49/99] JSON parser --- handler.go | 33 +++++++++++++++++++++++++++++++++ index.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/handler.go b/handler.go index bb3ed54a4..a214a872e 100644 --- a/handler.go +++ b/handler.go @@ -115,6 +115,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") @@ -1518,6 +1519,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } + fmt.Println() // Encode InputDefinition to its internal representation. def, err := req.Encode() @@ -1622,3 +1624,34 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req } type defaultInputDefinitionResponse struct{} + +func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + inputDefName := mux.Vars(r)["input-definition"] + + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + return + } + + + // Decode request. + var reqs []interface{} + err := json.NewDecoder(r.Body).Decode(&reqs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + for _, req := range reqs { + definition := index.inputDefinition(inputDefName) + err = index.JSONParser(req.(map[string]interface{}), definition) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } +} + + diff --git a/index.go b/index.go index ca5428e06..7676daf77 100644 --- a/index.go +++ b/index.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "reflect" ) // Default index settings. @@ -755,3 +756,49 @@ func (i *Index) openInputDefinition() error { } return nil } + + +func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { + fmt.Println(i.Frames) + for _, field := range inputDef.fields{ + if _, ok := req[field.Name]; !ok { + return fmt.Errorf("field not found") + } + for _, action := range field.Actions { + switch action.ValueDestination { + case "map": + err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "stringToBool": + fmt.Println("HERE") + err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + case "valueToRow": + err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + if err != nil { + return fmt.Errorf("Error map value: %s", err) + } + } + } + } + val := reflect.ValueOf(Action{}) + for i := 0; i < val.Type().NumField(); i++{ + fmt.Println(val.Type().Field(i).Type) + } + return nil +} +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ + return nil +} + +func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ + return nil +} + +func (i *Index) ValueToRow(frame, name string, value uint64) error{ + return nil +} From f6e82fc5250506747ec51446abd1d5d291a81279 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 21 Jun 2017 15:10:05 -0500 Subject: [PATCH 50/99] update definition validation --- input_definition.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/input_definition.go b/input_definition.go index c83a43f5b..6a580a48b 100644 --- a/input_definition.go +++ b/input_definition.go @@ -253,6 +253,15 @@ func convert(x *uint64) uint64 { return 0 } +func convert(x *uint64) uint64 { + if x != nil { + return *x + } + var v int64 = -1 + var v2 uint64 = uint64(v) + return v2 +} + // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` From f701f9d273541121882ceba074d61a9dd7aa0a4c Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 11:05:33 -0500 Subject: [PATCH 51/99] handle actions in JSONparser --- handler.go | 12 ++++---- index.go | 71 ++++++++++++++++++++++++++++++--------------- input_definition.go | 4 +-- pilosa.go | 1 + 4 files changed, 54 insertions(+), 34 deletions(-) diff --git a/handler.go b/handler.go index a214a872e..7ff5cf225 100644 --- a/handler.go +++ b/handler.go @@ -1519,7 +1519,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque http.Error(w, err.Error(), http.StatusBadRequest) return } - fmt.Println() // Encode InputDefinition to its internal representation. def, err := req.Encode() @@ -1636,7 +1635,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1645,13 +1643,13 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - definition := index.inputDefinition(inputDefName) - err = index.JSONParser(req.(map[string]interface{}), definition) - if err != nil { + err = index.JSONParser(req.(map[string]interface{}), inputDefName) + if err == ErrInputDefinitionNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } } - - diff --git a/index.go b/index.go index 7676daf77..01705bb44 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "reflect" ) // Default index settings. @@ -757,48 +756,72 @@ func (i *Index) openInputDefinition() error { return nil } - -func (i *Index) JSONParser(req map[string]interface{}, inputDef *InputDefinition) error { - fmt.Println(i.Frames) - for _, field := range inputDef.fields{ - if _, ok := req[field.Name]; !ok { - return fmt.Errorf("field not found") +func (i *Index) JSONParser(req map[string]interface{}, name string) error { + inputDef := i.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound + } + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + } + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } + } + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + for _, action := range field.Actions { switch action.ValueDestination { - case "map": + case "mapping": + value, ok := req[field.Name].(string) + if !ok { + return fmt.Errorf("String type required, got %s:%s", field.Name, value) + } err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action mapping : %s", err) } - case "stringToBool": - fmt.Println("HERE") - err := i.StringToBool(action.Frame, action.RowID, req[field.Name].(bool)) - if err != nil { - return fmt.Errorf("Error map value: %s", err) + case "single-row-boolean": + value, ok := req[field.Name].(bool) + if !ok { + return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) } - case "valueToRow": - err := i.ValueToRow(action.Frame, field.Name, req[field.Name].(uint64)) + err := i.StringRowBoolean(action.Frame, action.RowID, value) if err != nil { - return fmt.Errorf("Error map value: %s", err) + return fmt.Errorf("Error action single-row-boolean : %s", err) + } + case "value-to-row": + value, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) + } + err := i.ValueToRow(action.Frame, field.Name, uint64(value)) + if err != nil { + return fmt.Errorf("Error action value-to-row: %s", err) } } } } - val := reflect.ValueOf(Action{}) - for i := 0; i < val.Type().NumField(); i++{ - fmt.Println(val.Type().Field(i).Type) - } + return nil } -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error{ +func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { return nil } -func (i *Index) StringToBool(frame string, rowID uint64, value bool) error{ +func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { return nil } -func (i *Index) ValueToRow(frame, name string, value uint64) error{ +func (i *Index) ValueToRow(frame, name string, value uint64) error { return nil } diff --git a/input_definition.go b/input_definition.go index 6a580a48b..5c3154305 100644 --- a/input_definition.go +++ b/input_definition.go @@ -257,9 +257,7 @@ func convert(x *uint64) uint64 { if x != nil { return *x } - var v int64 = -1 - var v2 uint64 = uint64(v) - return v2 + return 0 } // InputFrame defines the frame used in the input definition. diff --git a/pilosa.go b/pilosa.go index 6d90c0d75..bde8cc2c7 100644 --- a/pilosa.go +++ b/pilosa.go @@ -48,6 +48,7 @@ var ( ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") ErrRangeCacheNotAllowed = errors.New("range cache not allowed") ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From 8f4278465b66dd465071c6705392c5e8e3b99ab2 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 22 Jun 2017 21:57:00 -0500 Subject: [PATCH 52/99] merge action's processing --- handler.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/handler.go b/handler.go index 7ff5cf225..f3af90305 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "sort" "strconv" "strings" "time" @@ -1653,3 +1654,65 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } + +// MapAction Process the input data and set a bit +func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var ok bool + bit.ColumnID = colID + bit.RowID, ok = a.ValueMap[value] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", value) + } + + // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, nil +} + +// ValueToRow Sets a bitmap with rowID from the input value +func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// SingleRowBoolean Sets a bitmap with rowID from the Action defintion +func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { + var bit Bit + var err error + bit.ColumnID = colID + bit.RowID, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, err + } + // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) + return &bit, err +} + +// InputBits Process and Sort the Input Bits and route to appropriate nodes. +func (h *Handler) InputBits(index, frame string, bits []Bit) error { + client, err := NewClient(h.Host) + if err != nil { + return err + } + + bitsBySlice := Bits(bits).GroupBySlice() + + // Parse path into bits. + for slice, bits := range bitsBySlice { + sort.Sort(BitsByPos(bits)) + + h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) + if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { + return err + } + } + + return nil +} From 57d035260dd4fe351ee27de712c14347919dec44 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 23 Jun 2017 11:17:36 -0500 Subject: [PATCH 53/99] update JSONparser --- handler.go | 89 +++++++++++++++++++++--------------------------------- index.go | 70 ------------------------------------------ 2 files changed, 34 insertions(+), 125 deletions(-) diff --git a/handler.go b/handler.go index f3af90305..f7849efe9 100644 --- a/handler.go +++ b/handler.go @@ -1644,7 +1644,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = index.JSONParser(req.(map[string]interface{}), inputDefName) + err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1655,64 +1655,43 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// MapAction Process the input data and set a bit -func (h *Handler) MapAction(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var ok bool - bit.ColumnID = colID - bit.RowID, ok = a.ValueMap[value] - if !ok { - return nil, fmt.Errorf("Value %s does not exist in definition map", value) +// JSONParser validate input json file and execute SetBit +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { + inputDef := index.inputDefinition(name) + if inputDef == nil { + return ErrInputDefinitionNotFound } - - // _, err := i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, nil -} - -// ValueToRow Sets a bitmap with rowID from the input value -func (h *Handler) ValueToRow(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err + // if field in input data is not in defined definition, return error + validFields := make(map[string]bool) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// SingleRowBoolean Sets a bitmap with rowID from the Action defintion -func (h *Handler) SingleRowBoolean(a *Action, value string, colID uint64) (*Bit, error) { - var bit Bit - var err error - bit.ColumnID = colID - bit.RowID, err = strconv.ParseUint(value, 10, 64) - if err != nil { - return nil, err - } - // _, err = i.Frame(f).SetBit(ViewStandard, rowID, colID, nil) - return &bit, err -} - -// InputBits Process and Sort the Input Bits and route to appropriate nodes. -func (h *Handler) InputBits(index, frame string, bits []Bit) error { - client, err := NewClient(h.Host) - if err != nil { - return err - } - - bitsBySlice := Bits(bits).GroupBySlice() - - // Parse path into bits. - for slice, bits := range bitsBySlice { - sort.Sort(BitsByPos(bits)) - - h.logger().Printf("inputing slice: %d, n=%d", slice, len(bits)) - if err := client.Import(context.Background(), index, frame, slice, bits); err != nil { - return err + for key, _ := range req { + _, ok := validFields[key] + if !ok { + fmt.Errorf("field not found", key) } } + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + var colValue uint64 + if _, ok := req[field.Name]; !ok { + continue + } else if field.PrimaryKey { + colValue, ok := req[field.Name].(float64) + if !ok { + return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) + } else { + val, ok := req[DefaultColumnLabel] + if !ok { + return errors.New("column ID not provided") + } + colValue = val.(float64) + } + } + + } + return nil } diff --git a/index.go b/index.go index 01705bb44..ca5428e06 100644 --- a/index.go +++ b/index.go @@ -755,73 +755,3 @@ func (i *Index) openInputDefinition() error { } return nil } - -func (i *Index) JSONParser(req map[string]interface{}, name string) error { - inputDef := i.inputDefinition(name) - if inputDef == nil { - return ErrInputDefinitionNotFound - } - // if field in input data is not in defined definition, return error - validFields := make(map[string]bool) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - } - for key, _ := range req { - _, ok := validFields[key] - if !ok { - fmt.Errorf("field not found", key) - } - } - - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue - } - - for _, action := range field.Actions { - switch action.ValueDestination { - case "mapping": - value, ok := req[field.Name].(string) - if !ok { - return fmt.Errorf("String type required, got %s:%s", field.Name, value) - } - err := i.MapAction(action.Frame, action.ValueMap, req[field.Name].(string)) - if err != nil { - return fmt.Errorf("Error action mapping : %s", err) - } - case "single-row-boolean": - value, ok := req[field.Name].(bool) - if !ok { - return fmt.Errorf("Bool type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.StringRowBoolean(action.Frame, action.RowID, value) - if err != nil { - return fmt.Errorf("Error action single-row-boolean : %s", err) - } - case "value-to-row": - value, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("Float type required, got %s:%s", field.Name, req[field.Name]) - } - err := i.ValueToRow(action.Frame, field.Name, uint64(value)) - if err != nil { - return fmt.Errorf("Error action value-to-row: %s", err) - } - } - } - } - - return nil -} -func (i *Index) MapAction(frame string, valueMap map[string]uint64, value string) error { - return nil -} - -func (i *Index) StringRowBoolean(frame string, rowID *uint64, value bool) error { - return nil -} - -func (i *Index) ValueToRow(frame, name string, value uint64) error { - return nil -} From 3a659c216d4130d85f527226e5f0fe450e4b08fa Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 11:05:34 -0500 Subject: [PATCH 54/99] JSONparser tests --- handler.go | 45 +++++++++----- handler_test.go | 157 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 182 insertions(+), 20 deletions(-) diff --git a/handler.go b/handler.go index f7849efe9..2474c537a 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,6 @@ import ( "net/http" _ "net/http/pprof" "os" - "sort" "strconv" "strings" "time" @@ -1653,6 +1652,10 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } } + + if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } } // JSONParser validate input json file and execute SetBit @@ -1662,36 +1665,48 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri return ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error + var columnLabel string validFields := make(map[string]bool) for _, field := range inputDef.Fields() { validFields[field.Name] = true + if field.PrimaryKey { + columnLabel = field.Name + } } for key, _ := range req { _, ok := validFields[key] if !ok { - fmt.Errorf("field not found", key) + return fmt.Errorf("field not found: %s", key) } } + var bits []*Bit for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - var colValue uint64 + //var colValue uint64 if _, ok := req[field.Name]; !ok { continue - } else if field.PrimaryKey { - colValue, ok := req[field.Name].(float64) - if !ok { - return fmt.Errorf("float type required, got %s:%s", field.Name, colValue) - } else { - val, ok := req[DefaultColumnLabel] - if !ok { - return errors.New("column ID not provided") - } - colValue = val.(float64) - } + } + value, ok := req[columnLabel] + if !ok { + return fmt.Errorf("columnLabel required") + } + colValue, ok := value.(float64) + if !ok { + return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + for _, action := range field.Actions { + bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + if err != nil { + return fmt.Errorf("error handling action: %s", action.ValueDestination) + } + bits = append(bits, bit) + } } - return nil } + +func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + return nil, nil +} diff --git a/handler_test.go b/handler_test.go index de2831d37..bb9b5359d 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,10 @@ import ( "context" "encoding/json" "errors" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -27,11 +31,6 @@ import ( "reflect" "strings" "testing" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1300,3 +1299,151 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } } + +var defaultBody = ` + { + "frames":[ + { + "name":"event-time", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + } + ], + "fields":[ + { + "name":"id", + "primaryKey":true + }, + { + "name":"cabType", + "actions":[ + { + "frame":"cab-type", + "valueDestination":"mapping", + "valueMap":{ + "Green":1, + "Yellow":2 + } + } + ] + }, + { + "name":"withPet", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"single-row-boolean", + "rowID":100 + } + ] + }, + { + "name":"distanceMiles", + "actions":[ + { + "frame":"distance-miles", + "valueDestination":"value-to-row" + + } + ] + } + ] + }` + +func TestHandler_CreateInput(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + inputBody := []byte(` + [{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "with-pet": true + }]`) + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestInput_JSON(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + defBody := []byte(defaultBody) + def, err := EncodeInputDef("input1", defBody) + if err != nil { + t.Fatal(err) + } + _, err = index.CreateInputDefinition(def) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + json string + err string + }{ + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "nofield": true + }]`, + err: "field not found: nofield"}, + {json: `[{ + "id": "abc", + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "float64 require, got value:abc, type: string"}, + {json: `[{ + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }]`, + err: "columnLabel required"}, + } + h := NewHandler() + h.Holder = hldr.Holder + h.Cluster = NewCluster(1) + for _, test := range tests { + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(test.json)))) + if body := w.Body.String(); body != test.err+"\n" { + t.Fatalf("Expect error: %s, actual: %s", test.err, body) + } + } + +} +func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { + var req pilosa.InputDefinitionInfo + err := json.Unmarshal(body, &req) + if err != nil { + return nil, err + } + def, err := req.Encode() + def.Name = name + return def, err +} From 8653154dee22e321a2fd0b7761b12f11304cb725 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 26 Jun 2017 12:19:55 -0500 Subject: [PATCH 55/99] merge handleAction --- handler.go | 35 ++++++++------- handler_test.go | 25 +++++++++-- index.go | 23 ++++++++++ index_test.go | 34 +++++++++++++++ input_definition.go | 50 +++++++++++++++++++++- input_definition_test.go | 92 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 238 insertions(+), 21 deletions(-) diff --git a/handler.go b/handler.go index 2474c537a..a40bd8766 100644 --- a/handler.go +++ b/handler.go @@ -1643,7 +1643,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - err = h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1651,18 +1651,24 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + for fr, bs := range bits { + err := index.InputBits(fr, bs) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } // JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) error { +func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { - return ErrInputDefinitionNotFound + return nil, ErrInputDefinitionNotFound } // if field in input data is not in defined definition, return error var columnLabel string @@ -1676,11 +1682,12 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri for key, _ := range req { _, ok := validFields[key] if !ok { - return fmt.Errorf("field not found: %s", key) + return nil, fmt.Errorf("field not found: %s", key) } } var bits []*Bit + setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data //var colValue uint64 @@ -1689,24 +1696,22 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } value, ok := req[columnLabel] if !ok { - return fmt.Errorf("columnLabel required") + return nil, fmt.Errorf("columnLabel required") } colValue, ok := value.(float64) if !ok { - return fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } for _, action := range field.Actions { - bit, err := h.HandleAction(action, req[field.Name], uint64(colValue)) + frame := action.Frame + bit, err := HandleAction(action, req[field.Name], uint64(colValue)) if err != nil { - return fmt.Errorf("error handling action: %s", action.ValueDestination) + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - bits = append(bits, bit) + //bits = append(bits, bit) + setBits[frame] = append(bits, bit) } } - return nil -} - -func (h *Handler) HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { - return nil, nil + return setBits, nil } diff --git a/handler_test.go b/handler_test.go index bb9b5359d..cc71c76ff 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" @@ -1304,12 +1305,27 @@ var defaultBody = ` { "frames":[ { - "name":"event-time", + "name":"cab-type", "options": { "timeQuantum":"YMD", "inverseEnabled":false, "cacheType":"ranked" } + }, + { + "name":"add-ons", + "options": { + "timeQuantum":"YMD", + "inverseEnabled":false, + "cacheType":"ranked" + } + }, + { + "name":"distance-miles", + "options": { + "timeQuantum":"YMD", + "cacheType":"ranked" + } } ], "fields":[ @@ -1324,8 +1340,8 @@ var defaultBody = ` "frame":"cab-type", "valueDestination":"mapping", "valueMap":{ - "Green":1, - "Yellow":2 + "green":1, + "yellow":2 } } ] @@ -1372,13 +1388,14 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "with-pet": true + "withPet": true }]`) h := NewHandler() h.Holder = hldr.Holder h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { diff --git a/index.go b/index.go index ca5428e06..3dad6afd8 100644 --- a/index.go +++ b/index.go @@ -755,3 +755,26 @@ func (i *Index) openInputDefinition() error { } return nil } + +// InputBits Process the []Bit though the Frame import process +func (i *Index) InputBits(frame string, bits []*Bit) error { + var rowIDs, columnIDs []uint64 + timestamps := make([]*time.Time, len(bits)) + f := i.Frame(frame) + if f == nil { + return fmt.Errorf("Frame not found: %s", frame) + } + + for i, bit := range bits { + rowIDs = append(rowIDs, bit.RowID) + columnIDs = append(columnIDs, bit.ColumnID) + + // Convert timestamps to time.Time. + if bit.Timestamp > 0 { + t := time.Unix(0, bit.Timestamp) + timestamps[i] = &t + } + } + + return f.Import(rowIDs, columnIDs, timestamps) +} diff --git a/index_test.go b/index_test.go index 349dc0522..14e1ee617 100644 --- a/index_test.go +++ b/index_test.go @@ -459,3 +459,37 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } } + +func TestIndex_InputBits(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } + + // Create frame. + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + var bits []*pilosa.Bit + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) + bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) + bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + + err := index.InputBits("f", bits) + if err != nil { + t.Fatal(err) + } + + f := index.Frame("f") + v := f.View(pilosa.ViewStandard) + fragment := v.Fragment(0) + + // Verify the Bits were set + if a := fragment.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{0, 1}) { + t.Fatalf("unexpected bits: %+v", a) + } +} diff --git a/input_definition.go b/input_definition.go index 5c3154305..cc42bb4e4 100644 --- a/input_definition.go +++ b/input_definition.go @@ -15,12 +15,12 @@ package pilosa import ( + "fmt" "io/ioutil" "os" "path/filepath" "errors" - "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -318,3 +318,51 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) } return nil } + +// HandleAction Process the input data with its action and return a bit to be imported later +// Note: if the Bit should not be set then nil is returned with no error +// From the JSON marshalling the possible types are: float64, boolean, string +// TODO handle Timestams +func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { + var err error + var bit Bit + bit.ColumnID = colID + + switch a.ValueDestination { + case Mapping: + v, ok := value.(string) + if !ok { + return nil, fmt.Errorf("Mapping value must be a string %v", value) + } + bit.RowID, ok = a.ValueMap[v] + if !ok { + return nil, fmt.Errorf("Value %s does not exist in definition map", v) + } + case SingleRowBool: + switch value.(type) { + case bool: + if value.(bool) { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + case float64: + if value.(float64) >= 1 { + bit.RowID = *a.RowID + } else { // value is not True. + return nil, err + } + default: + return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) + } + case ValueToRow: + v, ok := value.(float64) + if !ok { + return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) + } + bit.RowID = uint64(v) + default: + return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) + } + return &bit, err +} diff --git a/input_definition_test.go b/input_definition_test.go index 76b55c7f0..cc4614faf 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -18,9 +18,10 @@ import ( "encoding/json" "testing" + "strings" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" - "strings" ) func TestInputDefinition_Open(t *testing.T) { @@ -143,3 +144,92 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected frame required error, actual error: %s", err) } } + +func TestHandleAction(t *testing.T) { + var value interface{} + colID := uint64(0) + rowID := uint64(100) + action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + + value = 1 + b, err := pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected integer type is not handled by single-row-boolean") + } else if !strings.Contains(err.Error(), "single-row-boolean value") { + t.Fatalf("Expected single-row-boolean value error, actual error: %s", err) + } + + value = "1" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore strings, only accept boolean") + } + + value = "t" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + + value = float64(1.5) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } + } + value = float64(0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = false + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that do not equate to True") + } + + value = true + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.ColumnID != 0 { + t.Fatalf("Unexpected ColumnID %v", b.ColumnID) + } + } + + action.ValueDestination = pilosa.ValueToRow + rowID = 101 + value = float64(25.0) + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + if b.RowID != 25 { + t.Fatalf("Unexpected RowID %v", b.RowID) + } + } + value = "25" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type float64") + } + + action.ValueDestination = pilosa.Mapping + value = "test" + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + value = 25 + b, err = pilosa.HandleAction(action, value, colID) + if b != nil { + t.Fatalf("Expected Ignore values that are not type string") + } + + action.ValueDestination = "test" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "Unrecognized Value Destination") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) + } + +} From 3594eb9abde6353b7b5aadf5d39aaa1a697669b9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 14:56:27 -0500 Subject: [PATCH 56/99] Prefaced validValueDestination values Code cleanup --- handler.go | 2 +- input_definition.go | 30 +++++++++++++++--------------- input_definition_test.go | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/handler.go b/handler.go index a40bd8766..37b59a64c 100644 --- a/handler.go +++ b/handler.go @@ -1679,7 +1679,7 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri columnLabel = field.Name } } - for key, _ := range req { + for key := range req { _, ok := validFields[key] if !ok { return nil, fmt.Errorf("field not found: %s", key) diff --git a/input_definition.go b/input_definition.go index cc42bb4e4..28b49b180 100644 --- a/input_definition.go +++ b/input_definition.go @@ -26,13 +26,14 @@ import ( "github.com/pilosa/pilosa/internal" ) +// Action Mapping types const ( - Mapping = "mapping" - ValueToRow = "value-to-row" - SingleRowBool = "single-row-boolean" + InputMapping = "mapping" + InputValueToRow = "value-to-row" + InputSingleRowBool = "single-row-boolean" ) -var ValidValueDestination = []string{Mapping, ValueToRow, SingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -107,13 +108,12 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { if err := i.ValidateAction(action); err != nil { return err } - if action.ValueDestination == SingleRowBool && action.Frame != "" { + if action.ValueDestination == InputSingleRowBool && action.Frame != "" { val, ok := countRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) - } else { - countRowID[action.Frame] = action.RowID } + countRowID[action.Frame] = action.RowID } actions = append(actions, Action{ Frame: action.Frame, @@ -289,7 +289,7 @@ func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { return &def, nil } -// AddFrame adds frame to input definition. +// AddFrame manually add frame to input definition. func (i *InputDefinition) AddFrame(frame InputFrame) error { i.frames = append(i.frames, frame) if err := i.saveMeta(); err != nil { @@ -298,20 +298,20 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -// ValidateAction validates actions from InputDefinition. +// ValidateAction ensures the input definition action conforms to our specification. func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { if action.Frame == "" { return ErrFrameRequired } validValues := make(map[string]bool) - for _, val := range ValidValueDestination { + for _, val := range validValueDestination { validValues[val] = true } if _, ok := validValues[action.ValueDestination]; !ok { return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) } switch action.ValueDestination { - case Mapping: + case InputMapping: if len(action.ValueMap) == 0 { return errors.New("valueMap required for map") } @@ -322,14 +322,14 @@ func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestams +// TODO handle Timestamps func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID switch a.ValueDestination { - case Mapping: + case InputMapping: v, ok := value.(string) if !ok { return nil, fmt.Errorf("Mapping value must be a string %v", value) @@ -338,7 +338,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { if !ok { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } - case SingleRowBool: + case InputSingleRowBool: switch value.(type) { case bool: if value.(bool) { @@ -355,7 +355,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } - case ValueToRow: + case InputValueToRow: v, ok := value.(float64) if !ok { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) diff --git a/input_definition_test.go b/input_definition_test.go index cc4614faf..86639c653 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -114,13 +114,13 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.SingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} _, err = act.Encode() if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.Mapping, RowID: 100} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) @@ -128,8 +128,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} + action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} field1 := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} err = input.LoadDefinition(def) @@ -137,7 +137,7 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - action = internal.InputDefinitionAction{ValueDestination: pilosa.SingleRowBool, RowID: 100} + action = internal.InputDefinitionAction{ValueDestination: pilosa.InputSingleRowBool, RowID: 100} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} err = input.LoadDefinition(def) if !strings.Contains(err.Error(), "frame required") { @@ -149,7 +149,7 @@ func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) rowID := uint64(100) - action := pilosa.Action{ValueDestination: pilosa.SingleRowBool, RowID: &rowID} + action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} value = 1 b, err := pilosa.HandleAction(action, value, colID) @@ -198,7 +198,7 @@ func TestHandleAction(t *testing.T) { } } - action.ValueDestination = pilosa.ValueToRow + action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) b, err = pilosa.HandleAction(action, value, colID) @@ -213,7 +213,7 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type float64") } - action.ValueDestination = pilosa.Mapping + action.ValueDestination = pilosa.InputMapping value = "test" b, err = pilosa.HandleAction(action, value, colID) if b != nil { From 9a7887fc2148b934e80710da80f6d6c5fb965872 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 15:19:03 -0500 Subject: [PATCH 57/99] remove prints --- handler_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/handler_test.go b/handler_test.go index cc71c76ff..d0b054dd3 100644 --- a/handler_test.go +++ b/handler_test.go @@ -19,11 +19,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "io" "io/ioutil" "net/http" @@ -32,6 +27,11 @@ import ( "reflect" "strings" "testing" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // Ensure the handler returns "not found" for invalid paths. @@ -1395,7 +1395,6 @@ func TestHandler_CreateInput(t *testing.T) { h.Cluster = NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) - fmt.Print(w.Body.String()) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { From 7cbe67964044b91f8d614c98d8c2c9d574899acd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:51:57 -0500 Subject: [PATCH 58/99] handle input nil bit --- index.go | 3 +++ index_test.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index 3dad6afd8..54a8f1433 100644 --- a/index.go +++ b/index.go @@ -766,6 +766,9 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { } for i, bit := range bits { + if bit == nil { + continue + } rowIDs = append(rowIDs, bit.RowID) columnIDs = append(columnIDs, bit.ColumnID) diff --git a/index_test.go b/index_test.go index 14e1ee617..aaebe288d 100644 --- a/index_test.go +++ b/index_test.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -461,6 +462,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { } func TestIndex_InputBits(t *testing.T) { + var bits []*pilosa.Bit index := MustOpenIndex() defer index.Close() @@ -469,17 +471,22 @@ func TestIndex_InputBits(t *testing.T) { t.Fatal(err) } + err := index.InputBits("f", bits) + if !strings.Contains(err.Error(), "Frame not found") { + t.Fatalf("Expected Frame not found error, actual error: %s", err) + } + // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - var bits []*pilosa.Bit bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0}) bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1}) bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1}) + bits = append(bits, nil) - err := index.InputBits("f", bits) + err = index.InputBits("f", bits) if err != nil { t.Fatal(err) } From 7592a43b9d9b2dcbe0cfc40d1de37b1c1ee55f8b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 26 Jun 2017 16:52:51 -0500 Subject: [PATCH 59/99] accumulate set bits per frame, don't mix into one set across frames --- handler.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/handler.go b/handler.go index 37b59a64c..40facf079 100644 --- a/handler.go +++ b/handler.go @@ -1643,7 +1643,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - bits, err := h.JSONParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.InputJsonDataParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1664,8 +1664,8 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// JSONParser validate input json file and execute SetBit -func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { +// InputJsonDataParser validate input json file and execute SetBit +func (h *Handler) InputJsonDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef := index.inputDefinition(name) if inputDef == nil { return nil, ErrInputDefinitionNotFound @@ -1686,11 +1686,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri } } - var bits []*Bit setBits := make(map[string][]*Bit) for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data - //var colValue uint64 if _, ok := req[field.Name]; !ok { continue } @@ -1709,8 +1707,9 @@ func (h *Handler) JSONParser(req map[string]interface{}, index *Index, name stri if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } - //bits = append(bits, bit) - setBits[frame] = append(bits, bit) + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } } } return setBits, nil From 26469db490ee172e681b409f43ae7aac86c384f6 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 08:50:33 -0500 Subject: [PATCH 60/99] The Input process must respect the Action Frame assignments --- handler_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index d0b054dd3..05c0db4d4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1400,6 +1400,32 @@ func TestHandler_CreateInput(t *testing.T) { } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) } + + // Verify the bits set per frame + // f := index.Frame("cab-type") + f0 := index.Frame("distance-miles") + v0 := f0.View(pilosa.ViewStandard) + fragment0 := v0.Fragment(0) + + // Verify the distanceMiles Bit was set + if a := fragment0.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + + f1 := index.Frame("add-ons") + v1 := f1.View(pilosa.ViewStandard) + fragment1 := v1.Fragment(0) + + // Verify the add-ons frame does not have a distanceMiles Bit set + // The Input process must respect the Action Frame assignments + if a := fragment1.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{}) { + t.Fatalf("unexpected bits: %+v", a) + } + // Verify the withPet Bit was set + if a := fragment1.Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1}) { + t.Fatalf("unexpected bits: %+v", a) + } + } func TestInput_JSON(t *testing.T) { @@ -1451,8 +1477,8 @@ func TestInput_JSON(t *testing.T) { t.Fatalf("Expect error: %s, actual: %s", test.err, body) } } - } + func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) { var req pilosa.InputDefinitionInfo err := json.Unmarshal(body, &req) From 453e5417f8ba8b928fe2efaaa3f972846b4bbb85 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:02:25 -0500 Subject: [PATCH 61/99] removed float64 check in single row boolean --- input_definition.go | 6 ------ input_definition_test.go | 16 ++++++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/input_definition.go b/input_definition.go index 28b49b180..b7f92a3a2 100644 --- a/input_definition.go +++ b/input_definition.go @@ -346,12 +346,6 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { } else { // value is not True. return nil, err } - case float64: - if value.(float64) >= 1 { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } default: return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } diff --git a/input_definition_test.go b/input_definition_test.go index 86639c653..0c03083f7 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -171,17 +171,10 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - value = float64(1.5) + value = float64(1) b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - if b.RowID != 100 { - t.Fatalf("Unexpected rowID %v", b.RowID) - } - } - value = float64(0) - b, err = pilosa.HandleAction(action, value, colID) - if b != nil { - t.Fatalf("Expected Ignore values that do not equate to True") + if !strings.Contains(err.Error(), "must equate to a Bool") { + t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false @@ -196,6 +189,9 @@ func TestHandleAction(t *testing.T) { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) } + if b.RowID != 100 { + t.Fatalf("Unexpected rowID %v", b.RowID) + } } action.ValueDestination = pilosa.InputValueToRow From 3ac40a4a5fd404c391bb9e07fdb0192cc0684eac Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 10:39:11 -0500 Subject: [PATCH 62/99] cleanup type detection for consistency --- input_definition.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index b7f92a3a2..f9de32c87 100644 --- a/input_definition.go +++ b/input_definition.go @@ -339,16 +339,14 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("Value %s does not exist in definition map", v) } case InputSingleRowBool: - switch value.(type) { - case bool: - if value.(bool) { - bit.RowID = *a.RowID - } else { // value is not True. - return nil, err - } - default: + v, ok := value.(bool) + if !ok { return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value) } + if v == false { // False returns a nil error and nil bit. + return nil, err + } + bit.RowID = *a.RowID case InputValueToRow: v, ok := value.(float64) if !ok { From 2ceeba690c1fe61b6f286d4bf88c4b279b0e236f Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 11:07:54 -0500 Subject: [PATCH 63/99] conver invalid input name --- input_definition_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/input_definition_test.go b/input_definition_test.go index 0c03083f7..5acb0c2bc 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -32,8 +32,14 @@ func TestInputDefinition_Open(t *testing.T) { frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} inputDef, err := index.CreateInputDefinition(&def) + if !strings.Contains(err.Error(), "invalid index or frame's name") { + t.Fatalf("Expected Invalid name error, actual error: %s", err) + } + + def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} + inputDef, err = index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) } From ad5518cb4b8ff2b977511e98bdcab82adfe480c6 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 27 Jun 2017 14:06:12 -0500 Subject: [PATCH 64/99] Input Definition Struct encoding error tests --- input_definition_test.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/input_definition_test.go b/input_definition_test.go index 5acb0c2bc..112d8eac9 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -120,12 +120,6 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - act := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} - _, err = act.Encode() - if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { - t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) - } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} @@ -151,6 +145,21 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } } +func TestActionEncoding(t *testing.T) { + action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + _, err := action.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } + + field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} + info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} + _, err = info.Encode() + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } +} + func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) From de42e3edc78adc34d3e2ee52515b1c591ce1ae79 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 27 Jun 2017 16:06:21 -0500 Subject: [PATCH 65/99] rebase input-definition --- handler.go | 2 +- input_definition.go | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/handler.go b/handler.go index 40facf079..99695ee7e 100644 --- a/handler.go +++ b/handler.go @@ -1659,7 +1659,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } } - if err := json.NewEncoder(w).Encode(postInputDefinitionResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } diff --git a/input_definition.go b/input_definition.go index f9de32c87..bef566416 100644 --- a/input_definition.go +++ b/input_definition.go @@ -253,13 +253,6 @@ func convert(x *uint64) uint64 { return 0 } -func convert(x *uint64) uint64 { - if x != nil { - return *x - } - return 0 -} - // InputFrame defines the frame used in the input definition. type InputFrame struct { Name string `json:"name,omitempty"` From f68110be8b1362db1939d4f945b5c6742ef4e2f6 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 28 Jun 2017 13:45:07 -0500 Subject: [PATCH 66/99] refactor frame options to reduce code duplication --- frame.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frame.go b/frame.go index 17abb5f44..ce7a6eb6b 100644 --- a/frame.go +++ b/frame.go @@ -213,6 +213,11 @@ func (f *Frame) CacheSize() uint32 { // Options returns all options for this frame. func (f *Frame) Options() FrameOptions { f.mu.Lock() + defer f.mu.Unlock() + return f.options() +} + +func (f *Frame) options() FrameOptions { opt := FrameOptions{ RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, @@ -221,7 +226,6 @@ func (f *Frame) Options() FrameOptions { CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, } - f.mu.Unlock() return opt } @@ -329,14 +333,8 @@ func (f *Frame) loadMeta() error { // saveMeta writes meta data for the frame. func (f *Frame) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.FrameMeta{ - RowLabel: f.rowLabel, - InverseEnabled: f.inverseEnabled, - RangeEnabled: f.rangeEnabled, - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: string(f.timeQuantum), - }) + fo := f.options() + buf, err := proto.Marshal(fo.Encode()) if err != nil { return err } From e4c71ff1ed2f33a4feb1cec93d7a1e80ddd216bb Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 28 Jun 2017 13:46:41 -0500 Subject: [PATCH 67/99] refactor some of the input defintion logic to better utilize Encode() --- handler.go | 14 +++--- handler_test.go | 4 +- index.go | 8 ++-- input_definition.go | 92 +++++++++++++++++----------------------- input_definition_test.go | 8 ++-- 5 files changed, 57 insertions(+), 69 deletions(-) diff --git a/handler.go b/handler.go index 99695ee7e..7e1ce9066 100644 --- a/handler.go +++ b/handler.go @@ -1520,12 +1520,16 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } + // TODO: validation before/after encode? + // Encode InputDefinition to its internal representation. - def, err := req.Encode() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + def := req.Encode() + /* + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + */ def.Name = inputDefName // Validate columnLabel and duplicate primaryKey. diff --git a/handler_test.go b/handler_test.go index 05c0db4d4..b5cb4b661 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1485,7 +1485,7 @@ func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) if err != nil { return nil, err } - def, err := req.Encode() + def := req.Encode() def.Name = name - return def, err + return def, nil } diff --git a/index.go b/index.go index 54a8f1433..62fd87410 100644 --- a/index.go +++ b/index.go @@ -155,7 +155,7 @@ func (i *Index) Open() error { return err } - if err := i.openInputDefinition(); err != nil { + if err := i.openInputDefinitions(); err != nil { return err } @@ -336,7 +336,7 @@ func (i *Index) SetTimeQuantum(q TimeQuantum) error { // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } -// InputDefinitionPath returns the path to an input definition in the index. +// InputDefinitionPath returns the path to the input definition directory for the index. func (i *Index) InputDefinitionPath() string { return filepath.Join(i.path, InputDefinitionDir) } @@ -723,8 +723,8 @@ func (i *Index) DeleteInputDefinition(name string) error { return nil } -// openInputDefinition opens and initializes the input definitions inside the index. -func (i *Index) openInputDefinition() error { +// openInputDefinitions opens and initializes the input definitions inside the index. +func (i *Index) openInputDefinitions() error { inputDef, err := os.Open(i.InputDefinitionPath()) if os.IsNotExist(err) { return nil diff --git a/input_definition.go b/input_definition.go index bef566416..c27b2911d 100644 --- a/input_definition.go +++ b/input_definition.go @@ -26,7 +26,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -// Action Mapping types +// Action types. const ( InputMapping = "mapping" InputValueToRow = "value-to-row" @@ -101,7 +101,7 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } - countRowID := make(map[string]uint64) + accountRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action for _, action := range field.InputDefinitionActions { @@ -109,11 +109,11 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { return err } if action.ValueDestination == InputSingleRowBool && action.Frame != "" { - val, ok := countRowID[action.Frame] + val, ok := accountRowID[action.Frame] if ok && val == action.RowID { return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) } - countRowID[action.Frame] = action.RowID + accountRowID[action.Frame] = action.RowID } actions = append(actions, Action{ Frame: action.Frame, @@ -152,40 +152,18 @@ func (i *InputDefinition) saveMeta() error { if err := os.MkdirAll(i.path, 0777); err != nil { return err } - // Marshal metadata. + var frames []*internal.Frame for _, fr := range i.frames { - frameMeta := &internal.FrameMeta{ - RowLabel: fr.Options.RowLabel, - InverseEnabled: fr.Options.InverseEnabled, - CacheType: fr.Options.CacheType, - CacheSize: fr.Options.CacheSize, - TimeQuantum: string(fr.Options.TimeQuantum), - } - frame := &internal.Frame{Name: fr.Name, Meta: frameMeta} - frames = append(frames, frame) + frames = append(frames, fr.Encode()) } var fields []*internal.InputDefinitionField for _, field := range i.fields { - var actions []*internal.InputDefinitionAction - for _, action := range field.Actions { - actionMeta := &internal.InputDefinitionAction{ - Frame: action.Frame, - ValueDestination: action.ValueDestination, - ValueMap: action.ValueMap, - RowID: convert(action.RowID), - } - actions = append(actions, actionMeta) - } - - fieldMeta := &internal.InputDefinitionField{ - Name: field.Name, - PrimaryKey: field.PrimaryKey, - InputDefinitionActions: actions, - } - fields = append(fields, fieldMeta) + fields = append(fields, field.Encode()) } + + // Marshal input definition. buf, err := proto.Marshal(&internal.InputDefinition{ Name: i.name, Frames: frames, @@ -211,17 +189,16 @@ type InputDefinitionField struct { } // Encode converts InputDefinitionField into its internal representation. -func (o *InputDefinitionField) Encode() (*internal.InputDefinitionField, error) { - field := internal.InputDefinitionField{Name: o.Name, PrimaryKey: o.PrimaryKey} - +func (o *InputDefinitionField) Encode() *internal.InputDefinitionField { + var actions []*internal.InputDefinitionAction for _, action := range o.Actions { - actionEncode, err := action.Encode() - if err != nil { - return nil, err - } - field.InputDefinitionActions = append(field.InputDefinitionActions, actionEncode) + actions = append(actions, action.Encode()) + } + return &internal.InputDefinitionField{ + Name: o.Name, + PrimaryKey: o.PrimaryKey, + InputDefinitionActions: actions, } - return &field, nil } // Action describes the mapping method for the field in the InputDefinition. @@ -233,16 +210,19 @@ type Action struct { } // Encode converts Action into its internal representation. -func (o *Action) Encode() (*internal.InputDefinitionAction, error) { - if o.RowID == nil && o.ValueDestination == "single-row-boolean" { - return nil, errors.New("rowID required for single-row-boolean") - } +func (o *Action) Encode() *internal.InputDefinitionAction { + // TODO: this check needs to happen somewhere other than Encode() + /* + if o.RowID == nil && o.ValueDestination == InputSingleRowBool { + return nil, errors.New("rowID required for single-row-boolean") + } + */ return &internal.InputDefinitionAction{ Frame: o.Frame, ValueDestination: o.ValueDestination, ValueMap: o.ValueMap, RowID: convert(o.RowID), - }, nil + } } // convert pointer to uint64 @@ -259,6 +239,15 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } +// Encode converts InputFrame into its internal representation. +func (f *InputFrame) Encode() *internal.Frame { + return &internal.Frame{ + Name: f.Name, + Meta: f.Options.Encode(), + } + +} + // InputDefinitionInfo represents the json message format needed to create an InputDefinition. type InputDefinitionInfo struct { Frames []InputFrame `json:"frames"` @@ -266,20 +255,15 @@ type InputDefinitionInfo struct { } // Encode converts InputDefinitionInfo into its internal representation. -func (i *InputDefinitionInfo) Encode() (*internal.InputDefinition, error) { +func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { var def internal.InputDefinition for _, f := range i.Frames { - def.Frames = append(def.Frames, &internal.Frame{Name: f.Name, Meta: f.Options.Encode()}) + def.Frames = append(def.Frames, f.Encode()) } for _, f := range i.Fields { - fEncode, err := f.Encode() - if err != nil { - return nil, err - } - def.Fields = append(def.Fields, fEncode) + def.Fields = append(def.Fields, f.Encode()) } - - return &def, nil + return &def } // AddFrame manually add frame to input definition. diff --git a/input_definition_test.go b/input_definition_test.go index 112d8eac9..612dbcbce 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -87,10 +87,7 @@ func TestInputDefinition_Encoding(t *testing.T) { t.Fatal(err) } - internalDef, err := def.Encode() - if err != nil { - t.Fatal(err) - } + internalDef := def.Encode() if internalDef.Frames[0].Name != "event-time" { t.Fatalf("unexpected frame: %v", internalDef) @@ -145,6 +142,8 @@ func TestInputDefinition_LoadDefinition(t *testing.T) { } } +/* +// TODO: handle validation outside of the Encode() func TestActionEncoding(t *testing.T) { action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} _, err := action.Encode() @@ -159,6 +158,7 @@ func TestActionEncoding(t *testing.T) { t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) } } +*/ func TestHandleAction(t *testing.T) { var value interface{} From 6028b50b067874a2e117cf708fcc2f5ad1e2eedd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 28 Jun 2017 16:33:42 -0500 Subject: [PATCH 68/99] Validate incoming input definitions through the InputDefinitionInfo Validate rather than through Encode and piecemeal throughout the code. --- handler.go | 28 ++------ handler_test.go | 2 +- input_definition.go | 140 +++++++++++++++++++++++++-------------- input_definition_test.go | 110 ++++++++++++++++++------------ pilosa.go | 23 ++++--- 5 files changed, 174 insertions(+), 129 deletions(-) diff --git a/handler.go b/handler.go index 7e1ce9066..ed34f19d0 100644 --- a/handler.go +++ b/handler.go @@ -1520,34 +1520,16 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // TODO: validation before/after encode? + // validation definition before/after encode? + if err := req.Validate(index.ColumnLabel()); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Encode InputDefinition to its internal representation. def := req.Encode() - /* - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - */ def.Name = inputDefName - // Validate columnLabel and duplicate primaryKey. - numPrimaryKey := 0 - for _, field := range def.Fields { - if field.PrimaryKey { - numPrimaryKey += 1 - if field.Name != index.columnLabel { - http.Error(w, ErrInputDefinitionColumnLabel.Error(), http.StatusBadRequest) - return - } - } - } - if numPrimaryKey > 1 { - http.Error(w, ErrInputDefinitionPrimaryKey.Error(), http.StatusBadRequest) - return - } - // Create InputDefinition. _, err = index.CreateInputDefinition(def) if err == ErrInputDefinitionExists { diff --git a/handler_test.go b/handler_test.go index b5cb4b661..e47e04cc0 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1159,7 +1159,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionPrimaryKey.Error()+"\n" { + } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionDupePrimaryKey.Error()+"\n" { t.Fatalf("unexpected body: %s", body) } diff --git a/input_definition.go b/input_definition.go index c27b2911d..32e61effa 100644 --- a/input_definition.go +++ b/input_definition.go @@ -15,13 +15,12 @@ package pilosa import ( + "errors" "fmt" "io/ioutil" "os" "path/filepath" - "errors" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -101,20 +100,9 @@ func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error { i.frames = append(i.frames, inputFrame) } - accountRowID := make(map[string]uint64) for _, field := range pb.Fields { var actions []Action for _, action := range field.InputDefinitionActions { - if err := i.ValidateAction(action); err != nil { - return err - } - if action.ValueDestination == InputSingleRowBool && action.Frame != "" { - val, ok := accountRowID[action.Frame] - if ok && val == action.RowID { - return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) - } - accountRowID[action.Frame] = action.RowID - } actions = append(actions, Action{ Frame: action.Frame, ValueDestination: action.ValueDestination, @@ -209,19 +197,34 @@ type Action struct { RowID *uint64 `json:"rowID,omitempty"` } -// Encode converts Action into its internal representation. -func (o *Action) Encode() *internal.InputDefinitionAction { - // TODO: this check needs to happen somewhere other than Encode() - /* - if o.RowID == nil && o.ValueDestination == InputSingleRowBool { - return nil, errors.New("rowID required for single-row-boolean") +// Validate ensures the input definition action conforms to our specification. +func (a *Action) Validate() error { + if a.Frame == "" { + return ErrFrameRequired + } + validValues := make(map[string]bool) + for _, val := range validValueDestination { + validValues[val] = true + } + if _, ok := validValues[a.ValueDestination]; !ok { + return fmt.Errorf("invalid ValueDestination: %s", a.ValueDestination) + } + switch a.ValueDestination { + case InputMapping: + if len(a.ValueMap) == 0 { + return errors.New("valueMap required for map") } - */ + } + return nil +} + +// Encode converts Action into its internal representation. +func (a *Action) Encode() *internal.InputDefinitionAction { return &internal.InputDefinitionAction{ - Frame: o.Frame, - ValueDestination: o.ValueDestination, - ValueMap: o.ValueMap, - RowID: convert(o.RowID), + Frame: a.Frame, + ValueDestination: a.ValueDestination, + ValueMap: a.ValueMap, + RowID: convert(a.RowID), } } @@ -239,13 +242,21 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -// Encode converts InputFrame into its internal representation. -func (f *InputFrame) Encode() *internal.Frame { - return &internal.Frame{ - Name: f.Name, - Meta: f.Options.Encode(), +// Validate the InputFrame data +func (i *InputFrame) Validate() error { + if err := ValidateName(i.Name); err != nil { + return err } + return nil +} + +// Encode converts InputFrame into its internal representation. +func (i *InputFrame) Encode() *internal.Frame { + return &internal.Frame{ + Name: i.Name, + Meta: i.Options.Encode(), + } } // InputDefinitionInfo represents the json message format needed to create an InputDefinition. @@ -254,6 +265,56 @@ type InputDefinitionInfo struct { Fields []InputDefinitionField `json:"fields"` } +// Validate the InputDefinitionInfo data +func (i *InputDefinitionInfo) Validate(columnLabel string) error { + numPrimaryKey := 0 + accountRowID := make(map[string]uint64) + + if len(i.Frames) == 0 { + return fmt.Errorf("At least one frame is required per Input Definition") + } + + for _, frame := range i.Frames { + if err := frame.Validate(); err != nil { + return err + } + // TODO frame option validation + } + + // Validate columnLabel and duplicate primaryKey. + for _, field := range i.Fields { + if field.PrimaryKey { + numPrimaryKey++ + if field.Name != columnLabel { + return ErrInputDefinitionColumnLabel + } + } + for _, action := range field.Actions { + if err := action.Validate(); err != nil { + return err + } + if action.ValueDestination == InputSingleRowBool && action.Frame != "" { + if action.RowID == nil { + return fmt.Errorf("rowID required for single-row-boolean Field %s", field.Name) + } + val, ok := accountRowID[action.Frame] + if ok && val == convert(action.RowID) { + return fmt.Errorf("duplicate rowID with other field: %v", action.RowID) + } + accountRowID[action.Frame] = convert(action.RowID) + } + } + } + if len(i.Fields) > 0 && numPrimaryKey == 0 { + return ErrInputDefinitionHasPrimaryKey + } + if numPrimaryKey > 1 { + return ErrInputDefinitionDupePrimaryKey + } + + return nil +} + // Encode converts InputDefinitionInfo into its internal representation. func (i *InputDefinitionInfo) Encode() *internal.InputDefinition { var def internal.InputDefinition @@ -275,27 +336,6 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { return nil } -// ValidateAction ensures the input definition action conforms to our specification. -func (i *InputDefinition) ValidateAction(action *internal.InputDefinitionAction) error { - if action.Frame == "" { - return ErrFrameRequired - } - validValues := make(map[string]bool) - for _, val := range validValueDestination { - validValues[val] = true - } - if _, ok := validValues[action.ValueDestination]; !ok { - return fmt.Errorf("invalid ValueDestination: %s", action.ValueDestination) - } - switch action.ValueDestination { - case InputMapping: - if len(action.ValueMap) == 0 { - return errors.New("valueMap required for map") - } - } - return nil -} - // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string diff --git a/input_definition_test.go b/input_definition_test.go index 612dbcbce..6e007fc32 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -102,64 +102,86 @@ func TestInputDefinition_Encoding(t *testing.T) { } } -func TestInputDefinition_LoadDefinition(t *testing.T) { - index := MustOpenIndex() - defer index.Close() +// Test The Action validation cases +func TestActionValidation(t *testing.T) { + rowID := uint64(100) - // Create Input Definition. - input := pilosa.InputDefinition{} - frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} - action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} - field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def := &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err := input.LoadDefinition(def) + action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} + field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} + info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} + err := info.Validate("id") + if !strings.Contains(err.Error(), "one frame is required per Input Definition") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } + + frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") + if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { + t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) + } + + frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{RowLabel: "row"}} + action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") + if !strings.Contains(err.Error(), "invalid index or frame's name") { + t.Fatalf("Expected iinvalid index or frame's name error, actual error: %s", err) + } + + frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} + action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") + if !strings.Contains(err.Error(), "frame required") { + t.Fatalf("Expected frame required error, actual error: %s", err) + } + + action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("test") + if !strings.Contains(err.Error(), "PrimaryKey field name does not match columnLabel") { + t.Fatalf("Expected PrimaryKey field name does not match columnLabel error, actual error: %s", err) + } + + action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") + if !strings.Contains(err.Error(), "input-definition must contain one PrimaryKey") { + t.Fatalf("Expected input-definition must contain one PrimaryKey error, actual error: %s", err) + } + + action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") if !strings.Contains(err.Error(), "invalid ValueDestination") { t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: 100} - field = internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) + action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} + err = info.Validate("id") if !strings.Contains(err.Error(), "valueMap required for map") { t.Fatalf("Expected valueMap required for map error, actual error: %s", err) } - action = internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - action1 := internal.InputDefinitionAction{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - field1 := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action1}} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field, &field1}} - err = input.LoadDefinition(def) + action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false, Actions: []pilosa.Action{action}} + action1 := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + field1 := pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action1}} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} + err = info.Validate("id") if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } - - action = internal.InputDefinitionAction{ValueDestination: pilosa.InputSingleRowBool, RowID: 100} - def = &internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}} - err = input.LoadDefinition(def) - if !strings.Contains(err.Error(), "frame required") { - t.Fatalf("Expected frame required error, actual error: %s", err) - } } -/* -// TODO: handle validation outside of the Encode() -func TestActionEncoding(t *testing.T) { - action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}} - _, err := action.Encode() - if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { - t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) - } - - field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} - info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} - _, err = info.Encode() - if !strings.Contains(err.Error(), "rowID required for single-row-boolean") { - t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err) - } -} -*/ - func TestHandleAction(t *testing.T) { var value interface{} colID := uint64(0) diff --git a/pilosa.go b/pilosa.go index bde8cc2c7..cb650e02d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,18 +36,19 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") - ErrInputDefinitionExists = errors.New("input-definition already exists") - ErrInputDefinitionPrimaryKey = errors.New("input-definition can only contain one PrimaryKey") - ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") - ErrInputDefinitionNameRequired = errors.New("input-definition name required") - ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") + ErrInputDefinitionExists = errors.New("input-definition already exists") + ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") + ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey") + ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") + ErrInputDefinitionNameRequired = errors.New("input-definition name required") + ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") + ErrRangeCacheNotAllowed = errors.New("range cache not allowed") + ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrInvalidView = errors.New("invalid view") From 3e2bbc3717db8af9e7ff92e0c68def18aee6e965 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 08:50:53 -0500 Subject: [PATCH 69/99] require fields and frames in definition --- input_definition.go | 11 +++++------ input_definition_test.go | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/input_definition.go b/input_definition.go index 32e61effa..acfe3b40b 100644 --- a/input_definition.go +++ b/input_definition.go @@ -247,7 +247,7 @@ func (i *InputFrame) Validate() error { if err := ValidateName(i.Name); err != nil { return err } - + // TODO frame option validation return nil } @@ -270,15 +270,14 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { numPrimaryKey := 0 accountRowID := make(map[string]uint64) - if len(i.Frames) == 0 { - return fmt.Errorf("At least one frame is required per Input Definition") + if len(i.Frames) == 0 || len(i.Fields) == 0 { + return ErrInputDefinitionAttrsRequired } for _, frame := range i.Frames { if err := frame.Validate(); err != nil { return err } - // TODO frame option validation } // Validate columnLabel and duplicate primaryKey. @@ -293,7 +292,7 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { if err := action.Validate(); err != nil { return err } - if action.ValueDestination == InputSingleRowBool && action.Frame != "" { + if action.ValueDestination == InputSingleRowBool { if action.RowID == nil { return fmt.Errorf("rowID required for single-row-boolean Field %s", field.Name) } @@ -305,13 +304,13 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { } } } + if len(i.Fields) > 0 && numPrimaryKey == 0 { return ErrInputDefinitionHasPrimaryKey } if numPrimaryKey > 1 { return ErrInputDefinitionDupePrimaryKey } - return nil } diff --git a/input_definition_test.go b/input_definition_test.go index 6e007fc32..32219b68c 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -110,7 +110,7 @@ func TestActionValidation(t *testing.T) { field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}} info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} err := info.Validate("id") - if !strings.Contains(err.Error(), "one frame is required per Input Definition") { + if err != pilosa.ErrInputDefinitionAttrsRequired { t.Fatalf("Expected frame required error, actual error: %s", err) } From 56f8705407d0f31d46e72017611829dc3cf78e8a Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 08:51:29 -0500 Subject: [PATCH 70/99] return an error if the definition does not exist --- index.go | 20 +++++++++----------- index_test.go | 25 ++++++++++++------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/index.go b/index.go index 62fd87410..f155bc991 100644 --- a/index.go +++ b/index.go @@ -349,13 +349,13 @@ func (i *Index) Frame(name string) *Frame { } // InputDefinition returns an input definition in the index by name. -func (i *Index) InputDefinition(name string) *InputDefinition { +func (i *Index) InputDefinition(name string) (*InputDefinition, error) { i.mu.Lock() defer i.mu.Unlock() if inputDef, ok := i.inputDefinitions[name]; ok { - return inputDef + return inputDef, nil } - return nil + return nil, ErrInputDefinitionNotFound } func (i *Index) frame(name string) *Frame { return i.frames[name] } @@ -657,8 +657,6 @@ func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefin func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) { if pb.Name == "" { return nil, ErrInputDefinitionNameRequired - } else if len(pb.Frames) == 0 || len(pb.Fields) == 0 { - return nil, ErrInputDefinitionAttrsRequired } for _, fr := range pb.Frames { @@ -704,15 +702,15 @@ func (i *Index) newInputDefinition(name string) (*InputDefinition, error) { // DeleteInputDefinition removes an input definition from the index. func (i *Index) DeleteInputDefinition(name string) error { + // Fail if input definition doesn't exist. + _, err := i.InputDefinition(name) + if err != nil { + return err + } + i.mu.Lock() defer i.mu.Unlock() - // Ignore if input definition doesn't exist. - inputDef := i.inputDefinition(name) - if inputDef == nil { - return nil - } - // Delete input definition file. if err := os.Remove(filepath.Join(i.InputDefinitionPath(), name)); err != nil { return err diff --git a/index_test.go b/index_test.go index aaebe288d..9a3f4133c 100644 --- a/index_test.go +++ b/index_test.go @@ -384,16 +384,9 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { index := MustOpenIndex() defer index.Close() - // Test frames and fields are required - def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} - _, err := index.CreateInputDefinition(&def) - if err != pilosa.ErrInputDefinitionAttrsRequired { - t.Fatal(err) - } - //Test input definition name is required - def = internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} - _, err = index.CreateInputDefinition(&def) + def := internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}} + _, err := index.CreateInputDefinition(&def) if err != pilosa.ErrInputDefinitionNameRequired { t.Fatal(err) } @@ -426,15 +419,21 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { _, err := index.CreateInputDefinition(&def) if err != nil { t.Fatal(err) - } else if index.InputDefinition("test") == nil { - t.Fatal("No input definition created") + } + + _, err = index.InputDefinition("test") + if err != nil { + t.Fatal(err) } err = index.DeleteInputDefinition("test") if err != nil { t.Fatal(err) - } else if index.InputDefinition("test") != nil { - t.Fatal("input definition isn't deleted") + } + + _, err = index.InputDefinition("test") + if err != pilosa.ErrInputDefinitionNotFound { + t.Fatal(err) } } From 8e8796993334dd8f3ee3ff4eee4d2e8a5fdca9a7 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 08:52:18 -0500 Subject: [PATCH 71/99] Input definition GET and DELETE handle the case when the definition does not exist --- handler.go | 21 +++++++++++++-------- handler_test.go | 13 +++++++++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/handler.go b/handler.go index ed34f19d0..63ecffac0 100644 --- a/handler.go +++ b/handler.go @@ -1566,8 +1566,13 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques return } - inputDef, _ := index.inputDefinitions[inputDefName] - if err := json.NewEncoder(w).Encode(InputDefinitionInfo{ + inputDef, err := index.InputDefinition(inputDefName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + if err = json.NewEncoder(w).Encode(InputDefinitionInfo{ Frames: inputDef.frames, Fields: inputDef.fields, }); err != nil { @@ -1629,7 +1634,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { return } for _, req := range reqs { - bits, err := h.InputJsonDataParser(req.(map[string]interface{}), index, inputDefName) + bits, err := h.InputJSONDataParser(req.(map[string]interface{}), index, inputDefName) if err == ErrInputDefinitionNotFound { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1650,11 +1655,11 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// InputJsonDataParser validate input json file and execute SetBit -func (h *Handler) InputJsonDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { - inputDef := index.inputDefinition(name) - if inputDef == nil { - return nil, ErrInputDefinitionNotFound +// InputJSONDataParser validate input json file and execute SetBit +func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { + inputDef, err := index.InputDefinition(name) + if err != nil { + return nil, err } // if field in input data is not in defined definition, return error var columnLabel string diff --git a/handler_test.go b/handler_test.go index e47e04cc0..a88ab1f64 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1253,8 +1253,10 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if index.InputDefinition("test") != nil { - t.Fatalf("unexpected result: %v", index.InputDefinition("test")) + } + _, err = index.InputDefinition("test") + if err != pilosa.ErrInputDefinitionNotFound { + t.Fatal(err) } } @@ -1299,6 +1301,13 @@ func TestHandler_GetInputDefinition(t *testing.T) { } else if body := w.Body.String(); body != string(expect)+"\n" { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } + + // Check non existant definition + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } } var defaultBody = ` From 681c48fd12cd3bf2f61227ac3543890c7279936e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 09:31:21 -0500 Subject: [PATCH 72/99] add more tests for index and input-definition not found --- handler.go | 2 +- handler_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index 63ecffac0..88f1e3fa6 100644 --- a/handler.go +++ b/handler.go @@ -1595,7 +1595,7 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req // Delete input definition from the index. if err := index.DeleteInputDefinition(inputDefName); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusNotFound) return } diff --git a/handler_test.go b/handler_test.go index a88ab1f64..a0654adb4 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,6 +1121,15 @@ func TestHandler_CreateInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s", body) } + // Test index not found + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input-definition/input2", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s", body) + } + } // Ensure throwing error if there's duplicated primaryKey field. @@ -1247,6 +1256,14 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { if err != nil { t.Fatal(err) } + + // Test definition not found + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/foo", strings.NewReader(""))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -1402,7 +1419,24 @@ func TestHandler_CreateInput(t *testing.T) { h := NewHandler() h.Holder = hldr.Holder h.Cluster = NewCluster(1) + + // Return error if index does not exist w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input/input1", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { + t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) + } + + // Check non existant definition + w = httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody))) + if w.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d", w.Code) + } + + w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) From a80be54a60e0db6ffddbe8362c97fbfd324bd512 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 10:13:47 -0500 Subject: [PATCH 73/99] validate missing frame from input data --- handler_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/handler_test.go b/handler_test.go index a0654adb4..33c537fa8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1389,6 +1389,16 @@ var defaultBody = ` "frame":"distance-miles", "valueDestination":"value-to-row" + } + ] + }, + { + "name":"noFrame", + "actions":[ + { + "frame":"foo", + "valueDestination":"value-to-row" + } ] } @@ -1509,6 +1519,20 @@ func TestInput_JSON(t *testing.T) { "withPet": true }]`, err: "columnLabel required"}, + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "withPet": true + }`, + err: "unexpected EOF"}, + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "noFrame": 1 + }]`, + err: "Frame not found: foo"}, } h := NewHandler() h.Holder = hldr.Holder From c47fffebd36d6e7d30279787323904c8214d8b12 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 11:11:53 -0500 Subject: [PATCH 74/99] find string in slice --- input_definition.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/input_definition.go b/input_definition.go index acfe3b40b..5f748f571 100644 --- a/input_definition.go +++ b/input_definition.go @@ -202,13 +202,10 @@ func (a *Action) Validate() error { if a.Frame == "" { return ErrFrameRequired } - validValues := make(map[string]bool) - for _, val := range validValueDestination { - validValues[val] = true - } - if _, ok := validValues[a.ValueDestination]; !ok { + if !foundItem(validValueDestination, a.ValueDestination) { return fmt.Errorf("invalid ValueDestination: %s", a.ValueDestination) } + switch a.ValueDestination { case InputMapping: if len(a.ValueMap) == 0 { @@ -228,7 +225,7 @@ func (a *Action) Encode() *internal.InputDefinitionAction { } } -// convert pointer to uint64 +// convert pointer to uint64. func convert(x *uint64) uint64 { if x != nil { return *x @@ -242,7 +239,7 @@ type InputFrame struct { Options FrameOptions `json:"options,omitempty"` } -// Validate the InputFrame data +// Validate the InputFrame data. func (i *InputFrame) Validate() error { if err := ValidateName(i.Name); err != nil { return err @@ -265,7 +262,7 @@ type InputDefinitionInfo struct { Fields []InputDefinitionField `json:"fields"` } -// Validate the InputDefinitionInfo data +// Validate the InputDefinitionInfo data. func (i *InputDefinitionInfo) Validate(columnLabel string) error { numPrimaryKey := 0 accountRowID := make(map[string]uint64) From 57a67f16ed6bbdc853278fe244bfbd0c90dd0b15 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 11:12:06 -0500 Subject: [PATCH 75/99] error cleanup --- handler.go | 2 +- handler_test.go | 30 +++++++++++++++--------------- input_definition_test.go | 18 +++++++++--------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/handler.go b/handler.go index 88f1e3fa6..4fabdf911 100644 --- a/handler.go +++ b/handler.go @@ -1655,7 +1655,7 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// InputJSONDataParser validate input json file and execute SetBit +// InputJSONDataParser validates input json file and executes SetBit. func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) if err != nil { diff --git a/handler_test.go b/handler_test.go index 33c537fa8..8cc734b23 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1121,7 +1121,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Test index not found + // Test index not found. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input-definition/input2", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { @@ -1200,7 +1200,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Eusure throwing error if request body is invalid + // Eusure throwing error if request body is invalid. jsonErrorBody := []byte(` { "frames":[{ @@ -1237,7 +1237,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { h.Holder = hldr.Holder h.Cluster = NewCluster(1) - // Test index not found + // Test index not found. w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusNotFound { @@ -1246,7 +1246,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Test input definition is deleted + // Test input definition is deleted. index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} @@ -1257,7 +1257,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { t.Fatal(err) } - // Test definition not found + // Test definition not found. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/foo", strings.NewReader(""))) if w.Code != http.StatusNotFound { @@ -1277,7 +1277,7 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { } } -// Ensure handler can get existing input definition +// Ensure handler can get existing input definition. func TestHandler_GetInputDefinition(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -1290,7 +1290,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}} def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}} - // Return error if index does not exist + // Return error if index does not exist. w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusNotFound { @@ -1299,7 +1299,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) } - // Return existing input definition + // Return existing input definition. index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) inputDef, err := index.CreateInputDefinition(&def) if err != nil { @@ -1319,7 +1319,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } - // Check non existant definition + // Check nonexistant definition. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader(""))) if w.Code != http.StatusNotFound { @@ -1430,7 +1430,7 @@ func TestHandler_CreateInput(t *testing.T) { h.Holder = hldr.Holder h.Cluster = NewCluster(1) - // Return error if index does not exist + // Return error if index does not exist. w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { @@ -1439,7 +1439,7 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) } - // Check non existant definition + // Check nonexistant definition. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { @@ -1454,13 +1454,13 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected body: %s", body) } - // Verify the bits set per frame + // Verify the bits set per frame. // f := index.Frame("cab-type") f0 := index.Frame("distance-miles") v0 := f0.View(pilosa.ViewStandard) fragment0 := v0.Fragment(0) - // Verify the distanceMiles Bit was set + // Verify the distanceMiles Bit was set. if a := fragment0.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{1}) { t.Fatalf("unexpected bits: %+v", a) } @@ -1469,12 +1469,12 @@ func TestHandler_CreateInput(t *testing.T) { v1 := f1.View(pilosa.ViewStandard) fragment1 := v1.Fragment(0) - // Verify the add-ons frame does not have a distanceMiles Bit set + // Verify the add-ons frame does not have a distanceMiles Bit set. // The Input process must respect the Action Frame assignments if a := fragment1.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{}) { t.Fatalf("unexpected bits: %+v", a) } - // Verify the withPet Bit was set + // Verify the withPet Bit was set. if a := fragment1.Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1}) { t.Fatalf("unexpected bits: %+v", a) } diff --git a/input_definition_test.go b/input_definition_test.go index 32219b68c..68ac11593 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -111,7 +111,7 @@ func TestActionValidation(t *testing.T) { info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}} err := info.Validate("id") if err != pilosa.ErrInputDefinitionAttrsRequired { - t.Fatalf("Expected frame required error, actual error: %s", err) + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err) } frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} @@ -126,8 +126,8 @@ func TestActionValidation(t *testing.T) { field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate("id") - if !strings.Contains(err.Error(), "invalid index or frame's name") { - t.Fatalf("Expected iinvalid index or frame's name error, actual error: %s", err) + if err != pilosa.ErrName { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err) } frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{RowLabel: "row"}} @@ -135,24 +135,24 @@ func TestActionValidation(t *testing.T) { field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate("id") - if !strings.Contains(err.Error(), "frame required") { - t.Fatalf("Expected frame required error, actual error: %s", err) + if err != pilosa.ErrFrameRequired { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) } action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate("test") - if !strings.Contains(err.Error(), "PrimaryKey field name does not match columnLabel") { - t.Fatalf("Expected PrimaryKey field name does not match columnLabel error, actual error: %s", err) + if err != pilosa.ErrInputDefinitionColumnLabel { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionColumnLabel, err) } action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate("id") - if !strings.Contains(err.Error(), "input-definition must contain one PrimaryKey") { - t.Fatalf("Expected input-definition must contain one PrimaryKey error, actual error: %s", err) + if err != pilosa.ErrInputDefinitionHasPrimaryKey { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionHasPrimaryKey, err) } action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}} From 7abab7b795106cea29d7d76fa5be1b35e4cbafcc Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 11:23:36 -0500 Subject: [PATCH 76/99] Error ErrInputDefinitionValueMap --- input_definition.go | 3 +-- input_definition_test.go | 4 ++-- pilosa.go | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/input_definition.go b/input_definition.go index 5f748f571..c0f468b55 100644 --- a/input_definition.go +++ b/input_definition.go @@ -15,7 +15,6 @@ package pilosa import ( - "errors" "fmt" "io/ioutil" "os" @@ -209,7 +208,7 @@ func (a *Action) Validate() error { switch a.ValueDestination { case InputMapping: if len(a.ValueMap) == 0 { - return errors.New("valueMap required for map") + return ErrInputDefinitionValueMap } } return nil diff --git a/input_definition_test.go b/input_definition_test.go index 68ac11593..c395b0630 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -167,8 +167,8 @@ func TestActionValidation(t *testing.T) { field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}} info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}} err = info.Validate("id") - if !strings.Contains(err.Error(), "valueMap required for map") { - t.Fatalf("Expected valueMap required for map error, actual error: %s", err) + if err != pilosa.ErrInputDefinitionValueMap { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionValueMap, err) } action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} diff --git a/pilosa.go b/pilosa.go index cb650e02d..228a27fd2 100644 --- a/pilosa.go +++ b/pilosa.go @@ -42,6 +42,7 @@ var ( ErrInputDefinitionColumnLabel = errors.New("PrimaryKey field name does not match columnLabel") ErrInputDefinitionNameRequired = errors.New("input-definition name required") ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") + ErrInputDefinitionValueMap = errors.New("valueMap required for map") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") From fb7e1091b5e2508a477e2d5356f3890602e88251 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 13:16:05 -0500 Subject: [PATCH 77/99] field definitions require an action --- handler_test.go | 4 ++-- input_definition.go | 16 ++++++++++------ input_definition_test.go | 8 ++++++++ pilosa.go | 1 + 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/handler_test.go b/handler_test.go index 8cc734b23..853d05cd0 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1319,7 +1319,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, string(expect)) } - // Check nonexistant definition. + // Check nonexistent definition. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader(""))) if w.Code != http.StatusNotFound { @@ -1439,7 +1439,7 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound) } - // Check nonexistant definition. + // Check nonexistent definition. w = httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { diff --git a/input_definition.go b/input_definition.go index c0f468b55..0e5375507 100644 --- a/input_definition.go +++ b/input_definition.go @@ -278,13 +278,9 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { // Validate columnLabel and duplicate primaryKey. for _, field := range i.Fields { - if field.PrimaryKey { - numPrimaryKey++ - if field.Name != columnLabel { - return ErrInputDefinitionColumnLabel - } - } + var actionCount int for _, action := range field.Actions { + actionCount++ if err := action.Validate(); err != nil { return err } @@ -299,6 +295,14 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { accountRowID[action.Frame] = convert(action.RowID) } } + if field.PrimaryKey { + numPrimaryKey++ + if field.Name != columnLabel { + return ErrInputDefinitionColumnLabel + } + } else if actionCount == 0 { + return ErrInputDefinitionActionRequired + } } if len(i.Fields) > 0 && numPrimaryKey == 0 { diff --git a/input_definition_test.go b/input_definition_test.go index c395b0630..9a7b6f706 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -180,6 +180,14 @@ func TestActionValidation(t *testing.T) { if !strings.Contains(err.Error(), "duplicate rowID with other field") { t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err) } + + field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true} + field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false} + info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}} + err = info.Validate("id") + if err != pilosa.ErrInputDefinitionActionRequired { + t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionActionRequired, err) + } } func TestHandleAction(t *testing.T) { diff --git a/pilosa.go b/pilosa.go index 228a27fd2..943b67a3a 100644 --- a/pilosa.go +++ b/pilosa.go @@ -43,6 +43,7 @@ var ( ErrInputDefinitionNameRequired = errors.New("input-definition name required") ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrInputDefinitionValueMap = errors.New("valueMap required for map") + ErrInputDefinitionActionRequired = errors.New("field definitions require an action") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") From 64ed6f78153ae0d3eac0ccc8613af02ff18b536b Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 13:36:04 -0500 Subject: [PATCH 78/99] action length test --- input_definition.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/input_definition.go b/input_definition.go index 0e5375507..d9cc543b5 100644 --- a/input_definition.go +++ b/input_definition.go @@ -278,9 +278,7 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { // Validate columnLabel and duplicate primaryKey. for _, field := range i.Fields { - var actionCount int for _, action := range field.Actions { - actionCount++ if err := action.Validate(); err != nil { return err } @@ -300,7 +298,7 @@ func (i *InputDefinitionInfo) Validate(columnLabel string) error { if field.Name != columnLabel { return ErrInputDefinitionColumnLabel } - } else if actionCount == 0 { + } else if len(field.Actions) == 0 { return ErrInputDefinitionActionRequired } } From 122bff142ce9b972c1fdc96b8457d749209dec1a Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 13:53:07 -0500 Subject: [PATCH 79/99] refactor validation comment. --- handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler.go b/handler.go index 4fabdf911..09f920920 100644 --- a/handler.go +++ b/handler.go @@ -1520,7 +1520,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - // validation definition before/after encode? + // Validation the input definition with the curent index's ColumnLabel. if err := req.Validate(index.ColumnLabel()); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return From a2ad3925d0ef617bf0b515a9c3fa877dc296eafa Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 15:07:40 -0500 Subject: [PATCH 80/99] refactor input definition tests to use test utility package --- handler_test.go | 75 ++++++++++++++++++++-------------------- index_test.go | 12 ++++--- input_definition_test.go | 3 +- 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/handler_test.go b/handler_test.go index b0f919d10..d28221d7c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "bytes" "context" + "encoding/json" "errors" "io/ioutil" "net/http" @@ -940,7 +941,7 @@ func TestHandler_Expvars(t *testing.T) { // Ensure handler can create a input definition. func TestHandler_CreateInputDefinition(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) inputBody := []byte(` @@ -973,11 +974,11 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } ] }`) - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -985,7 +986,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { } w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusConflict { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionExists.Error()+"\n" { @@ -994,7 +995,7 @@ func TestHandler_CreateInputDefinition(t *testing.T) { // Test index not found. w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input-definition/input2", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/foo/input-definition/input2", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { @@ -1005,12 +1006,12 @@ func TestHandler_CreateInputDefinition(t *testing.T) { // Ensure throwing error if there's duplicated primaryKey field. func TestHandler_DuplicatePrimaryKey(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) //Ensure throwing error if there's duplicated primaryKey field invalidPrimaryKey := []byte(` @@ -1036,7 +1037,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { }`) w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionDupePrimaryKey.Error()+"\n" { @@ -1064,7 +1065,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { }`) w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrInputDefinitionColumnLabel.Error()+"\n" { @@ -1091,7 +1092,7 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { }`) w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `unexpected EOF`+"\n" { @@ -1102,15 +1103,15 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { // Ensure handler can delete a input definition. func TestHandler_DeleteInputDefinition(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) // Test index not found. w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { @@ -1130,13 +1131,13 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { // Test definition not found. w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/foo", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/foo", strings.NewReader(""))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -1150,11 +1151,11 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { // Ensure handler can get existing input definition. func TestHandler_GetInputDefinition(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} @@ -1163,7 +1164,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { // Return error if index does not exist. w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { @@ -1183,7 +1184,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { } w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != string(expect)+"\n" { @@ -1192,7 +1193,7 @@ func TestHandler_GetInputDefinition(t *testing.T) { // Check nonexistent definition. w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader(""))) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader(""))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } @@ -1277,7 +1278,7 @@ var defaultBody = ` }` func TestHandler_CreateInput(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) @@ -1297,13 +1298,13 @@ func TestHandler_CreateInput(t *testing.T) { "distanceMiles": 8, "withPet": true }]`) - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) + h.Cluster = test.NewCluster(1) // Return error if index does not exist. w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/foo/input/input1", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/foo/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" { @@ -1312,13 +1313,13 @@ func TestHandler_CreateInput(t *testing.T) { // Check nonexistent definition. w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody))) if w.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d", w.Code) } w = httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -1353,7 +1354,7 @@ func TestHandler_CreateInput(t *testing.T) { } func TestInput_JSON(t *testing.T) { - hldr := MustOpenHolder() + hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) defBody := []byte(defaultBody) @@ -1405,14 +1406,14 @@ func TestInput_JSON(t *testing.T) { }]`, err: "Frame not found: foo"}, } - h := NewHandler() + h := test.NewHandler() h.Holder = hldr.Holder - h.Cluster = NewCluster(1) - for _, test := range tests { + h.Cluster = test.NewCluster(1) + for _, req := range tests { w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(test.json)))) - if body := w.Body.String(); body != test.err+"\n" { - t.Fatalf("Expect error: %s, actual: %s", test.err, body) + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json)))) + if body := w.Body.String(); body != req.err+"\n" { + t.Fatalf("Expect error: %s, actual: %s", req.err, body) } } } diff --git a/index_test.go b/index_test.go index 11fe59982..5188c9a59 100644 --- a/index_test.go +++ b/index_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "io/ioutil" "reflect" "strings" "testing" @@ -276,6 +277,7 @@ func TestIndex_SetTimeQuantum(t *testing.T) { t.Fatalf("unexpected quantum (reopen): %s", q) } } + // Ensure index can delete a frame. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") @@ -289,7 +291,7 @@ func TestIndex_InvalidName(t *testing.T) { } func TestIndex_CreateInputDefinition(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create Input Definition. @@ -309,7 +311,7 @@ func TestIndex_CreateInputDefinition(t *testing.T) { // Ensure create input definition handle correct error func TestIndex_CreateExistingInputDefinition(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() //Test input definition name is required @@ -336,7 +338,7 @@ func TestIndex_CreateExistingInputDefinition(t *testing.T) { // Ensure to delete existing input definition. func TestIndex_DeleteInputDefinition(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create Input Definition. @@ -367,7 +369,7 @@ func TestIndex_DeleteInputDefinition(t *testing.T) { // Ensure that frame in input definition will be created when server restart func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create Input Definition. @@ -390,7 +392,7 @@ func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) { func TestIndex_InputBits(t *testing.T) { var bits []*pilosa.Bit - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Set index time quantum. diff --git a/input_definition_test.go b/input_definition_test.go index 9a7b6f706..58bf9e0e5 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -22,10 +22,11 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/test" ) func TestInputDefinition_Open(t *testing.T) { - index := MustOpenIndex() + index := test.MustOpenIndex() defer index.Close() // Create Input Definition. From 867af19f9bbffd8f7da6af301b0bf6a38307c6ac Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Thu, 29 Jun 2017 15:52:57 -0500 Subject: [PATCH 81/99] removed duplicate test --- test/index.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/index.go b/test/index.go index 650aeb6bf..a751b3af6 100644 --- a/test/index.go +++ b/test/index.go @@ -3,7 +3,6 @@ package test import ( "io/ioutil" "os" - "testing" "github.com/pilosa/pilosa" ) @@ -77,15 +76,3 @@ func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*F } return &Frame{Frame: f}, nil } - -// Ensure index can delete a frame. -func TestIndex_InvalidName(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-index-") - if err != nil { - panic(err) - } - index, err := pilosa.NewIndex(path, "ABC") - if index != nil { - t.Fatalf("unexpected index name %s", index) - } -} From 514ff39177fe4277de720fb2fdfda9193c2551bc Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 30 Jun 2017 10:52:00 -0500 Subject: [PATCH 82/99] update input definition doc --- docs/getting-started.md | 118 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 96fd4e9e7..e5a28ef32 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -74,9 +74,123 @@ curl localhost:10101/index/repository/frame/language \ -d '{"options": {"rowLabel": "language_id", "inverseEnabled": true}}' ``` -#### Import Some Data -The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). Download the `stargazer.csv` and `language.csv` files in that repo. +#### Create the Schema with Input Definition + +Input Definition supports user to define a schema based on their data and allows them to provide data to Pilosa in +a more standard format like JSON. Once an Input Definition is created, we can send data to Pilosa in JSON a format that adheres to that definition and Pilosa will internally perform all +of the appropriate mutations. + +Before creating a schema, let's create the repository index first: + +``` +curl localhost:10101/index/repository \ + -X POST \ + -d '{"options": {"columnLabel": "repo_id"}}' +``` +Then we can send following input definition as JSON to Pilosa: + +``` +curl localhost:10101/index/repository/input-definition/stargazer \ + -X POST \ + -d '{ + "frames": [ + { + "name": "language", + "options": { + "inverseEnabled": true, + "timeQuantum": "YMD" + } + }, + { + "name": "stargazer", + "options": { + "inverseEnabled": true, + "timeQuantum": "YMD" + } + } + ], + "fields": [ + { + "name": "repo_id", + "primaryKey": true + }, + { + "actions": [ + { + "frame": "language", + "valueDestination": "mapping", + "valueMap": { + "C": 7, + "C#": 27, + "Go": 5, + "Java": 21, + "JavaScript": 13, + "Python": 17, + } + } + ], + "name": "language_id" + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "value-to-row" + } + ], + "name": "stargazer_id" + } + ] + }' +``` + +Instead of creating `stargazer` frame and `language` frame individually like above, we can create multiple frames in one input definition. +We can also set `repo_id` for multiple frames at the same time by providing actions. There are three options for valueDestination: + + - value-to-row: the value for this field is used as the `rowID` + - single-row-boolean: the value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. + - mapping: the value for this field is used to lookup a `rowID` in a map, a valueMap is required for this destination type + +The sample input definition in JSON format for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-defintion.txt` + +#### Import Some Data with Input Definition + +The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). + +If you import data using input-definition, download the `json-input.txt` file in that repo, then run the following request to input-definition created above: + +``` +curl localhost:10101/index/repository/input/stargazer \ + -X POST \ + -d '[ + { + "language_id": "Go", + "repo_id": 91720568, + "stargazer_id": 513114 + }, + { + "language_id": "Python", + "repo_id": 95122322 + }' + ] +``` + +As define in input definition, field name `language_id` map language to correspondent id in `valueMap` in `language` frame and field name `stargazer_id` is added to `stargazer` frame as rowID +The data input above is relevant to multiple `SetBit()` queries : + +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) + SetBit(frame="language", repo_id=91720568, language_id=5) + SetBit(frame="language", repo_id=95122322, language_id=17) + ' +``` + +#### Import Some Data with csv files + +If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo. ``` curl -O https://raw.githubusercontent.com/pilosa/getting-started/master/stargazer.csv From 31c1994d2a6ccfcfca6bad927e7da0520b90e100 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 30 Jun 2017 13:46:52 -0500 Subject: [PATCH 83/99] API docs for input definition --- docs/api-reference.md | 95 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 8c083d514..2a3c9686c 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -173,7 +173,7 @@ Response: ### Remove frame -`DELETE POST /index//frame/` +`DELETE /index//frame/` Removes the given frame. @@ -219,6 +219,99 @@ Response: {} ``` +### Create input definition + +`POST /index//input-definition/` + +Creates a input definition in the given index with the given name. + +The request payload is in JSON, and must contain the `frames` and `fields` field. `frames` is an array of a JSON specifying the frames to use used/created for this import. Each frame must contain a `name` and may contain the following options: + +* `rowLabel` (string): Row label of the frame. +* `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. +* `inverseEnabled` (boolean): Enables [the inverted view]({{< ref "data-model.md#inverse" >}}) for this frame if `true`. +* `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. +* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. + +The `fields` array contains series of json objects describing how to process each field in the JSON data. Each `field` object must contain a string `name` which maps to the source json field name. One field must be defined at the `primaryKey`. The `primarykey` source field must contain only non duplicated unsigned integer values, and the field name must equal the column label for the `Index`. The `primarykey` ID's will map directly to column ID's in Pilosa. + +* `name` (string): Maps the source data field to actions that process its value. +* `actions` (array): List of Actions that will process the field's value. + +The `action` describes how the field value will be processed. Each `action` may contain: + +* `frame` (string): The Frame that will contain this Action's set bits. +* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame. +* `valueDestination` (string): The mapping rule used for this data. + - `value-to-row` The value should be an integer and will map directly to a RowID. + - `single-row-boolean` If the value is true set a bit using the `rowid`. + - `mapping` Map the value to a RowID in the `valueMap`. +* `valueMap` (object): string and integer pairs used to map field values to row ID's. + +Request: +``` +curl localhost:10101/index/user/input-definition/stargazer-input \ + -X POST \ + -d '{"frames":[{"name": "language", "options": {"rowLabel": "language_id"}}], "fields":[{"name": "repo_id", "primaryKey":true}, {"name": "language_id", "actions":[{"frame": "language", "valueDestination": "mapping", "valueMap": {"Go":5,"Python":17,"C++":10}}]}]}' +``` + +Response: +``` +{} +``` + +### Get input definition + +`GET /index//input-definition/` + +Returns the input definition in JSON. + +Request: +``` +curl -XGET localhost:10101/index/user/input-definition/stargazer-input +``` + +Response: +``` +{"frames":[{"name":"language","options":{"rowLabel":"language_id"}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} +``` + +### Remove input definition + +`DELETE /index//input-definition/` + +Removes the given input definition. + +Request: +``` +curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input +``` + +Response: +``` +{} +``` + +### Process input data + +`POST /index//input/` + +Process JSON body with the given input definition and set bits. + +The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and other fields that will be handled by the actions in the input definition. + +Request: +``` +curl localhost:10101/index/user/input/stargazer-input \ + -X POST \ + -d '[{"language_id": "Go", "repo_id": 92274475, "stargazer_id": 513111}]' +``` + +Response: +``` +{} +``` + ### List hosts `GET /hosts` From 5d939f0a1f19f4f8975d898f8f42ba4013d34e8a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 30 Jun 2017 14:53:52 -0500 Subject: [PATCH 84/99] update doc with input-definition json file --- docs/getting-started.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index e5a28ef32..6aea6a274 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -88,7 +88,7 @@ curl localhost:10101/index/repository \ -X POST \ -d '{"options": {"columnLabel": "repo_id"}}' ``` -Then we can send following input definition as JSON to Pilosa: +Then we can send following input definition as JSON to Pilosa. The sample input-defintion schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-definition.json` file ``` curl localhost:10101/index/repository/input-definition/stargazer \ @@ -152,13 +152,12 @@ We can also set `repo_id` for multiple frames at the same time by providing acti - single-row-boolean: the value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - mapping: the value for this field is used to lookup a `rowID` in a map, a valueMap is required for this destination type -The sample input definition in JSON format for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-defintion.txt` #### Import Some Data with Input Definition The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). -If you import data using input-definition, download the `json-input.txt` file in that repo, then run the following request to input-definition created above: +If you import data using input-definition, download the `json-input.json` file in that repo, then run the following request to input-definition created above: ``` curl localhost:10101/index/repository/input/stargazer \ From 082b0ba7ebfe035d92d488c00b0f8d3608bb2231 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 30 Jun 2017 15:37:47 -0500 Subject: [PATCH 85/99] minor adjustments to the API docs for input-definition --- docs/api-reference.md | 55 ++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 2a3c9686c..9a82c02f6 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -223,9 +223,9 @@ Response: `POST /index//input-definition/` -Creates a input definition in the given index with the given name. +Creates an input definition in the given index with the given name. -The request payload is in JSON, and must contain the `frames` and `fields` field. `frames` is an array of a JSON specifying the frames to use used/created for this import. Each frame must contain a `name` and may contain the following options: +The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: * `rowLabel` (string): Row label of the frame. * `timeQuantum` (string): [Time Quantum]({{< ref "data-model.md#time-quantum" >}}) for this frame. @@ -233,26 +233,53 @@ The request payload is in JSON, and must contain the `frames` and `fields` field * `cacheType` (string): [ranked]({{< ref "data-model.md#ranked" >}}) or [LRU]({{< ref "data-model.md#lru" >}}) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -The `fields` array contains series of json objects describing how to process each field in the JSON data. Each `field` object must contain a string `name` which maps to the source json field name. One field must be defined at the `primaryKey`. The `primarykey` source field must contain only non duplicated unsigned integer values, and the field name must equal the column label for the `Index`. The `primarykey` ID's will map directly to column ID's in Pilosa. +The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field name must equal the column label for the `Index`, and its value must be an unsigned integer which maps directly to a columnID in Pilosa. -* `name` (string): Maps the source data field to actions that process its value. -* `actions` (array): List of Actions that will process the field's value. +* `name` (string): Maps the source data field to actions that process the field's corresponding value. +* `actions` (array): List of actions that will process the field's value. The `action` describes how the field value will be processed. Each `action` may contain: -* `frame` (string): The Frame that will contain this Action's set bits. +* `frame` (string): The Frame that will contain this action's set bits. * `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame. * `valueDestination` (string): The mapping rule used for this data. - - `value-to-row` The value should be an integer and will map directly to a RowID. - - `single-row-boolean` If the value is true set a bit using the `rowid`. - - `mapping` Map the value to a RowID in the `valueMap`. -* `valueMap` (object): string and integer pairs used to map field values to row ID's. + - `value-to-row`: The value should be an integer and will map directly to a RowID. + - `single-row-boolean`: If the value is true set a bit using the `rowid`. + - `mapping`: Map the value to a RowID in the `valueMap`. +* `valueMap` (object): string and integer pairs used to map field values to RowID's. Request: ``` curl localhost:10101/index/user/input-definition/stargazer-input \ -X POST \ - -d '{"frames":[{"name": "language", "options": {"rowLabel": "language_id"}}], "fields":[{"name": "repo_id", "primaryKey":true}, {"name": "language_id", "actions":[{"frame": "language", "valueDestination": "mapping", "valueMap": {"Go":5,"Python":17,"C++":10}}]}]}' + -d '{ + "frames":[ + { + "name": "language", + "options": {"rowLabel": "language_id"} + } + ], + "fields":[ + { + "name": "repo_id", + "primaryKey":true + }, + { + "name": "language_id", + "actions":[ + { + "frame": "language", + "valueDestination": "mapping", + "valueMap": { + "Go": 5, + "Python": 17, + "C++": 10 + } + } + ] + } + ] + }' ``` Response: @@ -264,7 +291,7 @@ Response: `GET /index//input-definition/` -Returns the input definition in JSON. +Returns the given input definition as JSON. Request: ``` @@ -296,9 +323,9 @@ Response: `POST /index//input/` -Process JSON body with the given input definition and set bits. +Processes the JSON payload using the given input definition. -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and other fields that will be handled by the actions in the input definition. +The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. Request: ``` From fa794f6dc9cba52c02cc3dbc72b70f67be1797b6 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 30 Jun 2017 15:59:54 -0500 Subject: [PATCH 86/99] adjustments to the getting-started docs for input-definitions --- docs/getting-started.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 6aea6a274..6fd5aafd0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -75,10 +75,10 @@ curl localhost:10101/index/repository/frame/language \ "inverseEnabled": true}}' ``` -#### Create the Schema with Input Definition +#### Create the Schema Using an Input Definition -Input Definition supports user to define a schema based on their data and allows them to provide data to Pilosa in -a more standard format like JSON. Once an Input Definition is created, we can send data to Pilosa in JSON a format that adheres to that definition and Pilosa will internally perform all +Input definitions allow users to define a schema based on their data and to provide data to Pilosa in +a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations. Before creating a schema, let's create the repository index first: @@ -88,7 +88,7 @@ curl localhost:10101/index/repository \ -X POST \ -d '{"options": {"columnLabel": "repo_id"}}' ``` -Then we can send following input definition as JSON to Pilosa. The sample input-defintion schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-definition.json` file +Then we can send the following input definition as JSON to Pilosa. The sample input defintion schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-definition.json` file ``` curl localhost:10101/index/repository/input-definition/stargazer \ @@ -145,19 +145,19 @@ curl localhost:10101/index/repository/input-definition/stargazer \ }' ``` -Instead of creating `stargazer` frame and `language` frame individually like above, we can create multiple frames in one input definition. -We can also set `repo_id` for multiple frames at the same time by providing actions. There are three options for valueDestination: +Instead of creating a `stargazer` frame and a `language` frame individually like above, we can create multiple frames in one input definition. +We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination: - - value-to-row: the value for this field is used as the `rowID` - - single-row-boolean: the value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - - mapping: the value for this field is used to lookup a `rowID` in a map, a valueMap is required for this destination type + - value-to-row: The value for this field is used as the `rowID`. + - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. + - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. -#### Import Some Data with Input Definition +#### Import Data Using an Input Definition The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). -If you import data using input-definition, download the `json-input.json` file in that repo, then run the following request to input-definition created above: +If you import data using an input definition, download the `json-input.json` file in that repo, then run the following request using the input definition created above: ``` curl localhost:10101/index/repository/input/stargazer \ @@ -175,8 +175,8 @@ curl localhost:10101/index/repository/input/stargazer \ ] ``` -As define in input definition, field name `language_id` map language to correspondent id in `valueMap` in `language` frame and field name `stargazer_id` is added to `stargazer` frame as rowID -The data input above is relevant to multiple `SetBit()` queries : +As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. +The data input above is equivalent to the following `SetBit()` operations: ``` curl localhost:10101/index/repository/query \ @@ -187,7 +187,7 @@ curl localhost:10101/index/repository/query \ ' ``` -#### Import Some Data with csv files +#### Import Data From CSV Files If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo. From 32b46932e19604202a3e9bb81860a038f4aca1c4 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 3 Jul 2017 10:32:20 -0500 Subject: [PATCH 87/99] remove redundant errors from conflict --- pilosa.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pilosa.go b/pilosa.go index 48dd3c488..bdce7a3d8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -36,12 +36,6 @@ var ( ErrFrameInverseDisabled = errors.New("frame inverse disabled") ErrColumnRowLabelEqual = errors.New("column and row labels cannot be equal") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") ErrInputDefinitionExists = errors.New("input-definition already exists") ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey") ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey") @@ -50,14 +44,8 @@ var ( ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required") ErrInputDefinitionValueMap = errors.New("valueMap required for map") ErrInputDefinitionActionRequired = errors.New("field definitions require an action") + ErrInputDefinitionNotFound = errors.New("input-definition not found") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") - ErrInputDefinitionNotFound = errors.New("input-definition not found") ErrFieldNotFound = errors.New("field not found") ErrFieldNameRequired = errors.New("field name required") ErrInvalidFieldType = errors.New("invalid field type") From a4975e430b86abcad2672d3dc1842eb80d2601a1 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 3 Jul 2017 11:21:44 -0500 Subject: [PATCH 88/99] fix potential divide by zero --- fragment_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment_test.go b/fragment_test.go index 74ee91eeb..50e169fe4 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -176,7 +176,7 @@ func TestFragment_SetFieldValue(t *testing.T) { if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool { // Limit bit depth & maximum values. bitDepth = (bitDepth % 62) + 1 - columnN = (columnN % 100) + columnN = (columnN % 100) + 1 for i := range values { values[i] = values[i] % (1 << bitDepth) } From 294d3ab05018d2f34878504fc8ea9a9d990acc4c Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 3 Jul 2017 11:22:53 -0500 Subject: [PATCH 89/99] remove stargazer_id from this example since it is not used --- docs/api-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 9a82c02f6..f87a8c66c 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -331,7 +331,7 @@ Request: ``` curl localhost:10101/index/user/input/stargazer-input \ -X POST \ - -d '[{"language_id": "Go", "repo_id": 92274475, "stargazer_id": 513111}]' + -d '[{"language_id": "Go", "repo_id": 92274475}]' ``` Response: From 0a22386a58ecf5b4a7d9ee506428e56e762461f3 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Wed, 5 Jul 2017 15:30:21 -0500 Subject: [PATCH 90/99] add action to handle timestamp --- index.go | 2 +- input_definition.go | 11 ++++++++++- input_definition_test.go | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/index.go b/index.go index f155bc991..16720630e 100644 --- a/index.go +++ b/index.go @@ -772,7 +772,7 @@ func (i *Index) InputBits(frame string, bits []*Bit) error { // Convert timestamps to time.Time. if bit.Timestamp > 0 { - t := time.Unix(0, bit.Timestamp) + t := time.Unix(bit.Timestamp, 0) timestamps[i] = &t } } diff --git a/input_definition.go b/input_definition.go index d9cc543b5..a79b9c53e 100644 --- a/input_definition.go +++ b/input_definition.go @@ -22,6 +22,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "time" ) // Action types. @@ -29,9 +30,10 @@ const ( InputMapping = "mapping" InputValueToRow = "value-to-row" InputSingleRowBool = "single-row-boolean" + InputSetTimestamp = "set-timestamp" ) -var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool} +var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool, InputSetTimestamp} // InputDefinition represents a container for the data input definition. type InputDefinition struct { @@ -367,6 +369,13 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value) } bit.RowID = uint64(v) + case InputSetTimestamp: + v, err := time.Parse(TimeFormat, value.(string)) + if err != nil { + return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", value) + } + bit.Timestamp = v.Unix() + default: return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) } diff --git a/input_definition_test.go b/input_definition_test.go index 58bf9e0e5..e5138bf6b 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -23,6 +23,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" + "time" ) func TestInputDefinition_Open(t *testing.T) { @@ -268,6 +269,22 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type string") } + action.ValueDestination = pilosa.InputSetTimestamp + value = "2017-03-20T19:35" + parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) + b, err = pilosa.HandleAction(action, value, colID) + if b == nil { + t.Fatalf("Expected return bit") + } else if b.Timestamp != parsedTime.Unix() { + t.Fatalf("Timestamp is not set correctly") + } + + value = "12345677" + b, err = pilosa.HandleAction(action, value, colID) + if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { + t.Fatal("Expect invalid timestamp format") + } + action.ValueDestination = "test" b, err = pilosa.HandleAction(action, value, colID) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { From ea3e71f78dabd3c3805fdbf86a60b9fd0804ae16 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 09:44:12 -0500 Subject: [PATCH 91/99] update adding timestamp --- handler.go | 17 ++++++++++++++++- handler_test.go | 10 ++++++++++ input_definition.go | 20 ++++++++++++-------- input_definition_test.go | 30 ++++++++++++++++-------------- 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/handler.go b/handler.go index 09f920920..ade37b0c3 100644 --- a/handler.go +++ b/handler.go @@ -1664,11 +1664,17 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, // if field in input data is not in defined definition, return error var columnLabel string validFields := make(map[string]bool) + timestampFrame := make(map[string]string) for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { columnLabel = field.Name } + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame] = field.Name + } + } } for key := range req { _, ok := validFields[key] @@ -1678,6 +1684,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } setBits := make(map[string][]*Bit) + for _, field := range inputDef.Fields() { // skip field that defined in definition but not in input data if _, ok := req[field.Name]; !ok { @@ -1692,9 +1699,17 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + var timestamp string for _, action := range field.Actions { frame := action.Frame - bit, err := HandleAction(action, req[field.Name], uint64(colValue)) + timeField, ok := timestampFrame[action.Frame] + if !ok { + timestamp = "" + } else { + timestamp = req[timeField].(string) + } + + bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } diff --git a/handler_test.go b/handler_test.go index d28221d7c..c1e752e32 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1271,6 +1271,16 @@ var defaultBody = ` "frame":"foo", "valueDestination":"value-to-row" + } + ] + }, + { + "name":"time_value", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"set_timestamp" + } ] } diff --git a/input_definition.go b/input_definition.go index a79b9c53e..ae8cc35c9 100644 --- a/input_definition.go +++ b/input_definition.go @@ -212,7 +212,10 @@ func (a *Action) Validate() error { if len(a.ValueMap) == 0 { return ErrInputDefinitionValueMap } + case InputSetTimestamp: + } + return nil } @@ -338,11 +341,17 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -// TODO handle Timestamps -func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { +func HandleAction(a Action, value interface{}, colID uint64, timestamp string) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID + if timestamp != "" { + v, err := time.Parse(TimeFormat, timestamp) + if err != nil { + return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", timestamp) + } + bit.Timestamp = v.Unix() + } switch a.ValueDestination { case InputMapping: @@ -370,12 +379,7 @@ func HandleAction(a Action, value interface{}, colID uint64) (*Bit, error) { } bit.RowID = uint64(v) case InputSetTimestamp: - v, err := time.Parse(TimeFormat, value.(string)) - if err != nil { - return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", value) - } - bit.Timestamp = v.Unix() - + break default: return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination) } diff --git a/input_definition_test.go b/input_definition_test.go index e5138bf6b..d6bb7ba38 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -197,9 +197,10 @@ func TestHandleAction(t *testing.T) { colID := uint64(0) rowID := uint64(100) action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} + timestamp := "" value = 1 - b, err := pilosa.HandleAction(action, value, colID) + b, err := pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected integer type is not handled by single-row-boolean") } else if !strings.Contains(err.Error(), "single-row-boolean value") { @@ -207,31 +208,31 @@ func TestHandleAction(t *testing.T) { } value = "1" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore strings, only accept boolean") } value = "t" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = float64(1) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "must equate to a Bool") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } value = false - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that do not equate to True") } value = true - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.ColumnID != 0 { t.Fatalf("Unexpected ColumnID %v", b.ColumnID) @@ -244,35 +245,35 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputValueToRow rowID = 101 value = float64(25.0) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { if b.RowID != 25 { t.Fatalf("Unexpected RowID %v", b.RowID) } } value = "25" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type float64") } action.ValueDestination = pilosa.InputMapping value = "test" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } value = 25 - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b != nil { t.Fatalf("Expected Ignore values that are not type string") } action.ValueDestination = pilosa.InputSetTimestamp - value = "2017-03-20T19:35" + timestamp = "2017-03-20T19:35" parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if b == nil { t.Fatalf("Expected return bit") } else if b.Timestamp != parsedTime.Unix() { @@ -280,13 +281,14 @@ func TestHandleAction(t *testing.T) { } value = "12345677" - b, err = pilosa.HandleAction(action, value, colID) + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { t.Fatal("Expect invalid timestamp format") } action.ValueDestination = "test" - b, err = pilosa.HandleAction(action, value, colID) + timestamp = "" + b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } From 0f9f860624085e4d90cbb85bac70ec4dc6bb4e04 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 12:10:43 -0500 Subject: [PATCH 92/99] update timestamp for whole frame --- handler.go | 12 +++++++++++- handler_test.go | 15 ++++++++++++--- input_definition.go | 2 +- input_definition_test.go | 8 +------- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/handler.go b/handler.go index ade37b0c3..c9868a3a9 100644 --- a/handler.go +++ b/handler.go @@ -1670,6 +1670,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if field.PrimaryKey { columnLabel = field.Name } + // finding frame that need to add timestamp for _, action := range field.Actions { if action.ValueDestination == InputSetTimestamp { timestampFrame[action.Frame] = field.Name @@ -1699,6 +1700,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) } + // Looking into timestampFrame map and set timestamp to the whole frame var timestamp string for _, action := range field.Actions { frame := action.Frame @@ -1706,7 +1708,15 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if !ok { timestamp = "" } else { - timestamp = req[timeField].(string) + tmstamp, ok := req[timeField] + if !ok { + timestamp = "" + } else { + timestamp, ok = tmstamp.(string) + if !ok { + return nil, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, having: %v", req[timeField]) + } + } } bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) diff --git a/handler_test.go b/handler_test.go index c1e752e32..34eff2424 100644 --- a/handler_test.go +++ b/handler_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" + "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" @@ -1279,7 +1280,7 @@ var defaultBody = ` "actions":[ { "frame":"add-ons", - "valueDestination":"set_timestamp" + "valueDestination":"set-timestamp" } ] @@ -1306,7 +1307,8 @@ func TestHandler_CreateInput(t *testing.T) { "id": 1, "cabType": "yellow", "distanceMiles": 8, - "withPet": true + "withPet": true, + "time_value": "2017-03-20T19:35" }]`) h := test.NewHandler() h.Holder = hldr.Holder @@ -1328,6 +1330,7 @@ func TestHandler_CreateInput(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } + // Test successfully ingest data w = httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1337,7 +1340,6 @@ func TestHandler_CreateInput(t *testing.T) { } // Verify the bits set per frame. - // f := index.Frame("cab-type") f0 := index.Frame("distance-miles") v0 := f0.View(pilosa.ViewStandard) fragment0 := v0.Fragment(0) @@ -1415,6 +1417,13 @@ func TestInput_JSON(t *testing.T) { "noFrame": 1 }]`, err: "Frame not found: foo"}, + {json: `[{ + "id": 1, + "cabType": "yellow", + "distanceMiles": 8, + "time_value": 12345 + }]`, + err: "set-timestamp value must be in time format: YYYY-MM-DD, having: 12345"}, } h := test.NewHandler() h.Holder = hldr.Holder diff --git a/input_definition.go b/input_definition.go index ae8cc35c9..d8aa7a90a 100644 --- a/input_definition.go +++ b/input_definition.go @@ -348,7 +348,7 @@ func HandleAction(a Action, value interface{}, colID uint64, timestamp string) ( if timestamp != "" { v, err := time.Parse(TimeFormat, timestamp) if err != nil { - return nil, fmt.Errorf("set-timestamp value for :%v must in time format: YYYY-MM-DD", timestamp) + return nil, err } bit.Timestamp = v.Unix() } diff --git a/input_definition_test.go b/input_definition_test.go index d6bb7ba38..7558f6fac 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -272,7 +272,7 @@ func TestHandleAction(t *testing.T) { action.ValueDestination = pilosa.InputSetTimestamp timestamp = "2017-03-20T19:35" - parsedTime, _ := time.Parse(pilosa.TimeFormat, value.(string)) + parsedTime, _ := time.Parse(pilosa.TimeFormat, timestamp) b, err = pilosa.HandleAction(action, value, colID, timestamp) if b == nil { t.Fatalf("Expected return bit") @@ -280,12 +280,6 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Timestamp is not set correctly") } - value = "12345677" - b, err = pilosa.HandleAction(action, value, colID, timestamp) - if !strings.Contains(err.Error(), "set-timestamp value for :12345677 must in time format: YYYY-MM-DD") { - t.Fatal("Expect invalid timestamp format") - } - action.ValueDestination = "test" timestamp = "" b, err = pilosa.HandleAction(action, value, colID, timestamp) From 541d839b4d7b9bb11112d2bcabd7538d357ea0c9 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 12:33:09 -0500 Subject: [PATCH 93/99] remove unused import --- handler_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index 34eff2424..21cf22037 100644 --- a/handler_test.go +++ b/handler_test.go @@ -26,7 +26,6 @@ import ( "strings" "testing" - "fmt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" From 2144de29d0701a6fe044d27e02916eea85e301bc Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 6 Jul 2017 15:15:26 -0500 Subject: [PATCH 94/99] update doc with input definition timestamp --- docs/getting-started.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 6fd5aafd0..8535a4ac2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -140,7 +140,16 @@ curl localhost:10101/index/repository/input-definition/stargazer \ } ], "name": "stargazer_id" - } + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "set-timestamp" + } + ], + "name": "time_value + } ] }' ``` @@ -151,7 +160,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel - value-to-row: The value for this field is used as the `rowID`. - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - + - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame #### Import Data Using an Input Definition @@ -167,6 +176,7 @@ curl localhost:10101/index/repository/input/stargazer \ "language_id": "Go", "repo_id": 91720568, "stargazer_id": 513114 + "time_value": "2017-05-18T20:40" }, { "language_id": "Python", @@ -182,6 +192,7 @@ The data input above is equivalent to the following `SetBit()` operations: curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) + 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") SetBit(frame="language", repo_id=91720568, language_id=5) SetBit(frame="language", repo_id=95122322, language_id=17) ' From 7333e27392aa65a7a0cdb06c72bb9d4cf228700e Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Jul 2017 09:16:02 -0500 Subject: [PATCH 95/99] refactor Input Data GetTimeStamp functionality into a method (#1) * refactor Input Data GetTimeStamp functionality into a method * Comment time format tests --- handler.go | 72 +++++++++++++++++++++++----------------- handler_test.go | 32 +++++++++++++++++- input_definition.go | 11 ++---- input_definition_test.go | 15 +-------- 4 files changed, 75 insertions(+), 55 deletions(-) diff --git a/handler.go b/handler.go index c9868a3a9..27ac356d7 100644 --- a/handler.go +++ b/handler.go @@ -1661,22 +1661,35 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if err != nil { return nil, err } - // if field in input data is not in defined definition, return error - var columnLabel string + // If field in input data is not in defined definition, return error. + var colValue uint64 validFields := make(map[string]bool) - timestampFrame := make(map[string]string) + timestampFrame := make(map[string]int64) for _, field := range inputDef.Fields() { validFields[field.Name] = true if field.PrimaryKey { - columnLabel = field.Name + columnLabel := field.Name + value, ok := req[columnLabel] + if !ok { + return nil, fmt.Errorf("columnLabel required") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) } - // finding frame that need to add timestamp + // Find frame that need to add timestamp. for _, action := range field.Actions { if action.ValueDestination == InputSetTimestamp { - timestampFrame[action.Frame] = field.Name + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } } } } + for key := range req { _, ok := validFields[key] if !ok { @@ -1691,35 +1704,12 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, if _, ok := req[field.Name]; !ok { continue } - value, ok := req[columnLabel] - if !ok { - return nil, fmt.Errorf("columnLabel required") - } - colValue, ok := value.(float64) - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } // Looking into timestampFrame map and set timestamp to the whole frame - var timestamp string for _, action := range field.Actions { frame := action.Frame - timeField, ok := timestampFrame[action.Frame] - if !ok { - timestamp = "" - } else { - tmstamp, ok := req[timeField] - if !ok { - timestamp = "" - } else { - timestamp, ok = tmstamp.(string) - if !ok { - return nil, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, having: %v", req[timeField]) - } - } - } - - bit, err := HandleAction(action, req[field.Name], uint64(colValue), timestamp) + timestamp := timestampFrame[action.Frame] + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) } @@ -1730,3 +1720,23 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, } return setBits, nil } + +// GetTimeStamp retrieves unix timestamp from Input data. +func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) { + tmstamp, ok := data[timeField] + if !ok { + return 0, nil + } + + timestamp, ok := tmstamp.(string) + if !ok { + return 0, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, has: %v", data[timeField]) + } + + v, err := time.Parse(TimeFormat, timestamp) + if err != nil { + return 0, err + } + + return v.Unix(), nil +} diff --git a/handler_test.go b/handler_test.go index 21cf22037..0609c22db 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1422,7 +1422,7 @@ func TestInput_JSON(t *testing.T) { "distanceMiles": 8, "time_value": 12345 }]`, - err: "set-timestamp value must be in time format: YYYY-MM-DD, having: 12345"}, + err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() h.Holder = hldr.Holder @@ -1446,3 +1446,33 @@ func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) def.Name = name return def, nil } + +func TestHandler_GetTimeStamp(t *testing.T) { + data := make(map[string]interface{}) + timeField := "time" + data["time"] = "2017-03-20T19:35" + val, err := pilosa.GetTimeStamp(data, timeField) + if val != 1490038500 { + t.Fatalf("Timestamp is not set correctly for %s", data["time"]) + } + + // Verify that an integer is not a valid time format. + data["int"] = 1490000000 + val, err = pilosa.GetTimeStamp(data, "int") + if !strings.Contains(err.Error(), "set-timestamp value must be in time format") { + t.Fatalf("Expected set-timestamp value must be in time format error, actual error: %s", err) + } + + // Verify reversing month and year is not valid time format. + data["time"] = "03-2017-20T19:35" + val, err = pilosa.GetTimeStamp(data, timeField) + if !strings.Contains(err.Error(), "cannot parse") { + t.Fatalf("Expected Timestamp is not set correctly, actual error: %s", err) + } + + // Handle time fields that do not exist. + val, err = pilosa.GetTimeStamp(data, "test") + if val != 0 { + t.Fatalf("Expected Ignore nonexistent fields") + } +} diff --git a/input_definition.go b/input_definition.go index d8aa7a90a..ab5af2bc4 100644 --- a/input_definition.go +++ b/input_definition.go @@ -22,7 +22,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "time" ) // Action types. @@ -341,17 +340,11 @@ func (i *InputDefinition) AddFrame(frame InputFrame) error { // HandleAction Process the input data with its action and return a bit to be imported later // Note: if the Bit should not be set then nil is returned with no error // From the JSON marshalling the possible types are: float64, boolean, string -func HandleAction(a Action, value interface{}, colID uint64, timestamp string) (*Bit, error) { +func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (*Bit, error) { var err error var bit Bit bit.ColumnID = colID - if timestamp != "" { - v, err := time.Parse(TimeFormat, timestamp) - if err != nil { - return nil, err - } - bit.Timestamp = v.Unix() - } + bit.Timestamp = timestamp switch a.ValueDestination { case InputMapping: diff --git a/input_definition_test.go b/input_definition_test.go index 7558f6fac..da6e91231 100644 --- a/input_definition_test.go +++ b/input_definition_test.go @@ -23,7 +23,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/test" - "time" ) func TestInputDefinition_Open(t *testing.T) { @@ -197,7 +196,7 @@ func TestHandleAction(t *testing.T) { colID := uint64(0) rowID := uint64(100) action := pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID} - timestamp := "" + timestamp := int64(0) value = 1 b, err := pilosa.HandleAction(action, value, colID, timestamp) @@ -270,21 +269,9 @@ func TestHandleAction(t *testing.T) { t.Fatalf("Expected Ignore values that are not type string") } - action.ValueDestination = pilosa.InputSetTimestamp - timestamp = "2017-03-20T19:35" - parsedTime, _ := time.Parse(pilosa.TimeFormat, timestamp) - b, err = pilosa.HandleAction(action, value, colID, timestamp) - if b == nil { - t.Fatalf("Expected return bit") - } else if b.Timestamp != parsedTime.Unix() { - t.Fatalf("Timestamp is not set correctly") - } - action.ValueDestination = "test" - timestamp = "" b, err = pilosa.HandleAction(action, value, colID, timestamp) if !strings.Contains(err.Error(), "Unrecognized Value Destination") { t.Fatalf("Expected Unrecognized Value Destination error, actual error: %s", err) } - } From 7b7f471a184faabf1bcf075dd3ad56316d2f6e7d Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 18 Jul 2017 10:16:10 -0500 Subject: [PATCH 96/99] skip null field when ingest data --- handler.go | 3 +++ handler_test.go | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index 87712feb0..7ebd673f7 100644 --- a/handler.go +++ b/handler.go @@ -1694,6 +1694,9 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, for _, action := range field.Actions { frame := action.Frame timestamp := timestampFrame[action.Frame] + if req[field.Name] == nil { + continue + } bit, err := HandleAction(action, req[field.Name], colValue, timestamp) if err != nil { return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) diff --git a/handler_test.go b/handler_test.go index 0609c22db..f2bb02f63 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1274,6 +1274,16 @@ var defaultBody = ` } ] }, + { + "name":"null_value", + "actions":[ + { + "frame":"add-ons", + "valueDestination":"value-to-row" + + } + ] + }, { "name":"time_value", "actions":[ @@ -1307,7 +1317,8 @@ func TestHandler_CreateInput(t *testing.T) { "cabType": "yellow", "distanceMiles": 8, "withPet": true, - "time_value": "2017-03-20T19:35" + "time_value": "2017-03-20T19:35", + "null_value": null }]`) h := test.NewHandler() h.Holder = hldr.Holder From ec4419ddf6c52331268951c4a989ea46ead67056 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Tue, 18 Jul 2017 11:34:24 -0500 Subject: [PATCH 97/99] add comment for skip null value --- handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/handler.go b/handler.go index 7ebd673f7..a69545c5f 100644 --- a/handler.go +++ b/handler.go @@ -1694,6 +1694,7 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, for _, action := range field.Actions { frame := action.Frame timestamp := timestampFrame[action.Frame] + // Skip input data field values that are set to null if req[field.Name] == nil { continue } From 79f8f70e1c0801e82476b59e2a6a1e97fee2e867 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 26 Jul 2017 15:49:19 -0500 Subject: [PATCH 98/99] separate input definition docs from getting started --- docs/getting-started.md | 127 ++------------------------------------ docs/input-definition.md | 128 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 123 deletions(-) create mode 100644 docs/input-definition.md diff --git a/docs/getting-started.md b/docs/getting-started.md index 8535a4ac2..9c069cbd0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -75,129 +75,6 @@ curl localhost:10101/index/repository/frame/language \ "inverseEnabled": true}}' ``` -#### Create the Schema Using an Input Definition - -Input definitions allow users to define a schema based on their data and to provide data to Pilosa in -a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all -of the appropriate mutations. - -Before creating a schema, let's create the repository index first: - -``` -curl localhost:10101/index/repository \ - -X POST \ - -d '{"options": {"columnLabel": "repo_id"}}' -``` -Then we can send the following input definition as JSON to Pilosa. The sample input defintion schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-definition.json` file - -``` -curl localhost:10101/index/repository/input-definition/stargazer \ - -X POST \ - -d '{ - "frames": [ - { - "name": "language", - "options": { - "inverseEnabled": true, - "timeQuantum": "YMD" - } - }, - { - "name": "stargazer", - "options": { - "inverseEnabled": true, - "timeQuantum": "YMD" - } - } - ], - "fields": [ - { - "name": "repo_id", - "primaryKey": true - }, - { - "actions": [ - { - "frame": "language", - "valueDestination": "mapping", - "valueMap": { - "C": 7, - "C#": 27, - "Go": 5, - "Java": 21, - "JavaScript": 13, - "Python": 17, - } - } - ], - "name": "language_id" - }, - { - "actions": [ - { - "frame": "stargazer", - "valueDestination": "value-to-row" - } - ], - "name": "stargazer_id" - }, - { - "actions": [ - { - "frame": "stargazer", - "valueDestination": "set-timestamp" - } - ], - "name": "time_value - } - ] - }' -``` - -Instead of creating a `stargazer` frame and a `language` frame individually like above, we can create multiple frames in one input definition. -We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination: - - - value-to-row: The value for this field is used as the `rowID`. - - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. - - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame - -#### Import Data Using an Input Definition - -The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). - -If you import data using an input definition, download the `json-input.json` file in that repo, then run the following request using the input definition created above: - -``` -curl localhost:10101/index/repository/input/stargazer \ - -X POST \ - -d '[ - { - "language_id": "Go", - "repo_id": 91720568, - "stargazer_id": 513114 - "time_value": "2017-05-18T20:40" - }, - { - "language_id": "Python", - "repo_id": 95122322 - }' - ] -``` - -As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. -The data input above is equivalent to the following `SetBit()` operations: - -``` -curl localhost:10101/index/repository/query \ - -X POST \ - -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) - 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") - SetBit(frame="language", repo_id=91720568, language_id=5) - SetBit(frame="language", repo_id=95122322, language_id=17) - ' -``` - #### Import Data From CSV Files If you import data using csv files and without input defintion, download the `stargazer.csv` and `language.csv` files in that repo. @@ -224,6 +101,10 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. +### Input Definition +Alternatively Pilosa can import JSON data using a definition file describing the schema and ETL rules to process the data. +[Input Definition](../input-definition/) + #### Make Some Queries
diff --git a/docs/input-definition.md b/docs/input-definition.md new file mode 100644 index 000000000..4b73c4991 --- /dev/null +++ b/docs/input-definition.md @@ -0,0 +1,128 @@ ++++ +title = "Input Definition" ++++ + +## Input Definition +This document builds on the data import concepts introduced in [Getting Started](../getting-started/). +Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data. + +#### Create the Schema Using an Input Definition + +Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations. + +Before creating a schema, let's create the repository index first: + +``` +curl localhost:10101/index/repository \ + -X POST \ + -d '{"options": {"columnLabel": "repo_id"}}' +``` +Then we can send the following input definition as JSON to Pilosa. The sample input defintion schema for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started), `input-definition.json` file + +``` +curl localhost:10101/index/repository/input-definition/stargazer \ + -X POST \ + -d '{ + "frames": [ + { + "name": "language", + "options": { + "inverseEnabled": true, + "timeQuantum": "YMD" + } + }, + { + "name": "stargazer", + "options": { + "inverseEnabled": true, + "timeQuantum": "YMD" + } + } + ], + "fields": [ + { + "name": "repo_id", + "primaryKey": true + }, + { + "actions": [ + { + "frame": "language", + "valueDestination": "mapping", + "valueMap": { + "C": 7, + "C#": 27, + "Go": 5, + "Java": 21, + "JavaScript": 13, + "Python": 17, + } + } + ], + "name": "language_id" + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "value-to-row" + } + ], + "name": "stargazer_id" + }, + { + "actions": [ + { + "frame": "stargazer", + "valueDestination": "set-timestamp" + } + ], + "name": "time_value + } + ] + }' +``` + +Instead of creating a `stargazer` frame and a `language` frame individually like in [Getting Started](../getting-started/), we can create multiple frames in one input definition. +We can also set `repo_id` for multiple frames at the same time by providing field actions. There are three options for valueDestination: + + - value-to-row: The value for this field is used as the `rowID`. + - single-row-boolean: The value must be a boolean, and this specifies `SetBit()` or `ClearBit()`, a `rowID` must be specified for this destination type. + - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. + - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame + +#### Import Data Using an Input Definition + +The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). + +If you import data using an input definition, download the `json-input.json` file in that repo, then run the following request using the input definition created above: + +``` +curl localhost:10101/index/repository/input/stargazer \ + -X POST \ + -d '[ + { + "language_id": "Go", + "repo_id": 91720568, + "stargazer_id": 513114 + "time_value": "2017-05-18T20:40" + }, + { + "language_id": "Python", + "repo_id": 95122322 + }' + ] +``` + +As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. +The data input above is equivalent to the following `SetBit()` operations: + +``` +curl localhost:10101/index/repository/query \ + -X POST \ + -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) + 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") + SetBit(frame="language", repo_id=91720568, language_id=5) + SetBit(frame="language", repo_id=95122322, language_id=17) + ' +``` \ No newline at end of file From 8dc75535ff39d8bad2025567075f82f2cbf90a66 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Wed, 26 Jul 2017 16:39:07 -0500 Subject: [PATCH 99/99] use link in sentence --- docs/getting-started.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 9c069cbd0..bdf9384b2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -102,8 +102,7 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that, both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out `language.txt` to see the mapping for languages. ### Input Definition -Alternatively Pilosa can import JSON data using a definition file describing the schema and ETL rules to process the data. -[Input Definition](../input-definition/) +Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. #### Make Some Queries