scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 15:21:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove input definition, add install-stringer to Makefile

Browse source 
also removes one line of unreachable code in cluster.go (unrelated)
This commit is contained in:
Matt Jaffee 2018-05-14 17:14:20 -05:00
parent d36e33fa52
commit 616545cd5c
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
19 changed files with 275 additions and 3492 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Makefile
View file

@ -123,7 +123,7 @@ require-protoc-gen-gofast:
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer
install-dep:
go get -u github.com/golang/dep/cmd/dep
@ -131,6 +131,9 @@ install-dep:
install-statik:
go get -u github.com/rakyll/statik
install-stringer:
go get -u golang.org/x/tools/cmd/stringer
install-protoc-gen-gofast:
go get -u github.com/gogo/protobuf/protoc-gen-gofast

239
api.go
View file

@ -23,7 +23,6 @@ import (
"io"
"io/ioutil"
"net/http"
"reflect"
"strconv"
"strings"
"time"
@ -511,121 +510,6 @@ func (api *API) Hosts(ctx context.Context) []*Node {
return api.Cluster.Nodes
}
// CreateInputDefinition is deprecated and will be removed. Do not use it.
func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error {
if err := api.validate(apiCreateInputDefinition); err != nil {
return errors.Wrap(err, "validating api method")
}
api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed.
Please open an issue if you need to continue using it.`)
// Find index.
index := api.Holder.Index(indexName)
if index == nil {
return ErrIndexNotFound
}
if err := inputDef.Validate(); err != nil {
return err
}
// Encode InputDefinition to its internal representation.
def := inputDef.Encode()
def.Name = inputDefName
// Create InputDefinition.
if _, err := index.CreateInputDefinition(def); err != nil {
return err
}
err := api.Broadcaster.SendSync(
&internal.CreateInputDefinitionMessage{
Index: indexName,
Definition: def,
})
if err != nil {
api.Logger.Printf("problem sending CreateInputDefinition message: %s", err)
}
return nil
}
// InputDefinition is deprecated and will be removed.
func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) {
if err := api.validate(apiInputDefinition); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
api.Logger.Printf(`InputDefinition is deprecated and will be removed.`)
// Find index.
index := api.Holder.Index(indexName)
if index == nil {
return nil, ErrIndexNotFound
}
inputDef, err := index.InputDefinition(inputDefName)
if err != nil {
return nil, err
}
return inputDef, nil
}
// DeleteInputDefinition is deprecated and will be removed.
func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error {
if err := api.validate(apiDeleteInputDefinition); err != nil {
return errors.Wrap(err, "validating api method")
}
api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.")
// Find index.
index := api.Holder.Index(indexName)
if index == nil {
return ErrIndexNotFound
}
// Delete input definition from the index.
if err := index.DeleteInputDefinition(inputDefName); err != nil {
return err
}
err := api.Broadcaster.SendSync(
&internal.DeleteInputDefinitionMessage{
Index: indexName,
Name: inputDefName,
})
if err != nil {
api.Logger.Printf("problem sending DeleteInputDefinition message: %s", err)
}
return nil
}
// WriteInput is deprecated and will be removed.
func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error {
if err := api.validate(apiWriteInput); err != nil {
return errors.Wrap(err, "validating api method")
}
api.Logger.Printf("WriteInput is deprecated and will be removed.")
// Find index.
index := api.Holder.Index(indexName)
if index == nil {
return ErrIndexNotFound
}
for _, req := range reqs {
bits, err := api.inputJSONDataParser(req.(map[string]interface{}), index, inputDefName)
if err != nil {
return err
}
for fr, bs := range bits {
if err := index.InputBits(fr, bs); err != nil {
return err
}
}
}
return nil
}
// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
func (api *API) RecalculateCaches(ctx context.Context) error {
if err := api.validate(apiRecalculateCaches); err != nil {
@ -977,75 +861,6 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I
return index, frame, nil
}
// inputJSONDataParser validates input json file and executes SetBit. Deprecated - remove with input definition stuff.
func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) {
inputDef, err := index.InputDefinition(name)
if err != nil {
return nil, err
}
// If field in input data is not in defined definition, return error.
var colValue uint64
validFields := make(map[string]bool)
timestampFrame := make(map[string]int64)
for _, field := range inputDef.Fields() {
validFields[field.Name] = true
if field.PrimaryKey {
value, ok := req[field.Name]
if !ok {
return nil, fmt.Errorf("primary key does not exist")
}
rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float
if !ok {
return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value))
}
colValue = uint64(rawValue)
}
// Find frame that need to add timestamp.
for _, action := range field.Actions {
if action.ValueDestination == InputSetTimestamp {
timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name)
if err != nil {
return nil, err
}
}
}
}
for key := range req {
_, ok := validFields[key]
if !ok {
return nil, fmt.Errorf("field not found: %s", key)
}
}
setBits := make(map[string][]*Bit)
for _, field := range inputDef.Fields() {
// skip field that defined in definition but not in input data
if _, ok := req[field.Name]; !ok {
continue
}
// Looking into timestampFrame map and set timestamp to the whole frame
for _, action := range field.Actions {
frame := action.Frame
timestamp := timestampFrame[action.Frame]
// Skip input data field values that are set to null
if req[field.Name] == nil {
continue
}
bit, err := HandleAction(action, req[field.Name], colValue, timestamp)
if err != nil {
return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err)
}
if bit != nil {
setBits[frame] = append(setBits[frame], bit)
}
}
}
return setBits, nil
}
// SetCoordinator makes a new Node the cluster coordinator.
func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) {
if err := api.validate(apiSetCoordinator); err != nil {
@ -1136,11 +951,9 @@ const (
apiCreateField
apiCreateFrame
apiCreateIndex
apiCreateInputDefinition
apiDeleteField
apiDeleteFrame
apiDeleteIndex
apiDeleteInputDefinition
apiDeleteView
apiExportCSV
apiFields
@ -1152,7 +965,6 @@ const (
apiImportValue
apiIndex
apiIndexAttrDiff
apiInputDefinition
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
@ -1171,7 +983,6 @@ const (
apiUnmarshalFragment
//apiVersion // not implemented
apiViews
apiWriteInput
)
var methodsCommon = map[apiMethod]struct{}{
@ -1185,31 +996,27 @@ var methodsResizing = map[apiMethod]struct{}{
}
var methodsNormal = map[apiMethod]struct{}{
apiCreateField: struct{}{},
apiCreateFrame: struct{}{},
apiCreateIndex: struct{}{},
apiCreateInputDefinition: struct{}{},
apiDeleteField: struct{}{},
apiDeleteFrame: struct{}{},
apiDeleteIndex: struct{}{},
apiDeleteInputDefinition: struct{}{},
apiDeleteView: struct{}{},
apiExportCSV: struct{}{},
apiFields: struct{}{},
apiFragmentBlockData: struct{}{},
apiFragmentBlocks: struct{}{},
apiFrameAttrDiff: struct{}{},
apiImport: struct{}{},
apiImportValue: struct{}{},
apiIndex: struct{}{},
apiIndexAttrDiff: struct{}{},
apiInputDefinition: struct{}{},
apiQuery: struct{}{},
apiRecalculateCaches: struct{}{},
apiRemoveNode: struct{}{},
apiRestoreFrame: struct{}{},
apiSliceNodes: struct{}{},
apiUnmarshalFragment: struct{}{},
apiViews: struct{}{},
apiWriteInput: struct{}{},
apiCreateField: struct{}{},
apiCreateFrame: struct{}{},
apiCreateIndex: struct{}{},
apiDeleteField: struct{}{},
apiDeleteFrame: struct{}{},
apiDeleteIndex: struct{}{},
apiDeleteView: struct{}{},
apiExportCSV: struct{}{},
apiFields: struct{}{},
apiFragmentBlockData: struct{}{},
apiFragmentBlocks: struct{}{},
apiFrameAttrDiff: struct{}{},
apiImport: struct{}{},
apiImportValue: struct{}{},
apiIndex: struct{}{},
apiIndexAttrDiff: struct{}{},
apiQuery: struct{}{},
apiRecalculateCaches: struct{}{},
apiRemoveNode: struct{}{},
apiRestoreFrame: struct{}{},
apiSliceNodes: struct{}{},
apiUnmarshalFragment: struct{}{},
apiViews: struct{}{},
}

8
apimethod_string.go
View file

@ -2,15 +2,15 @@
package pilosa
import "fmt"
import "strconv"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return fmt.Sprintf("apiMethod(%d)", i)
return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

10
broadcast.go
View file

@ -129,8 +129,6 @@ const (
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateInputDefinition
MessageTypeDeleteInputDefinition
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
@ -163,10 +161,6 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
case *internal.CreateInputDefinitionMessage:
typ = MessageTypeCreateInputDefinition
case *internal.DeleteInputDefinitionMessage:
typ = MessageTypeDeleteInputDefinition
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
case *internal.ResizeInstruction:
@ -217,10 +211,6 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateInputDefinition:
m = &internal.CreateInputDefinitionMessage{}
case MessageTypeDeleteInputDefinition:
m = &internal.DeleteInputDefinitionMessage{}
case MessageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:

2
cluster.go
View file

@ -1728,8 +1728,6 @@ func (c *Cluster) nodeJoin(node *Node) error {
// know that it can proceed with opening its Holder.
return c.sendTo(node, c.Status())
}
return nil
}
// If the cluster already contains the node, just send it the cluster status.

149
handler.go
View file

@ -154,10 +154,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/view/{view}", handler.handleDeleteView).Methods("DELETE")
router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
@ -1342,131 +1338,6 @@ func errorString(err error) string {
return err.Error()
}
// handlePostInputDefinition handles POST /input-definition request.
func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
inputDefName := mux.Vars(r)["input-definition"]
// Decode request.
var req InputDefinitionInfo
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req); err != nil {
switch err {
case ErrIndexNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case ErrInputDefinitionExists:
http.Error(w, err.Error(), http.StatusConflict)
case ErrInputDefinitionAttrsRequired:
fallthrough
case ErrInputDefinitionNameRequired:
fallthrough
case ErrInputDefinitionActionRequired:
fallthrough
case ErrInputDefinitionHasPrimaryKey:
fallthrough
case ErrInputDefinitionDupePrimaryKey:
http.Error(w, err.Error(), http.StatusBadRequest)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
// handleGetInputDefinition handles GET /input-definition request.
func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
inputDefName := mux.Vars(r)["input-definition"]
inputDef, err := h.API.InputDefinition(r.Context(), indexName, inputDefName)
if err != nil {
switch err {
case nil:
break
case ErrIndexNotFound:
fallthrough
case ErrInputDefinitionNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
if err = json.NewEncoder(w).Encode(InputDefinitionInfo{
Frames: inputDef.frames,
Fields: inputDef.fields,
}); err != nil {
h.Logger.Printf("write status response error: %s", err)
}
}
// handleDeleteInputDefinition handles DELETE /input-definition request.
func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
inputDefName := mux.Vars(r)["input-definition"]
if err := h.API.DeleteInputDefinition(r.Context(), indexName, inputDefName); err != nil {
switch err {
case nil:
break
case ErrIndexNotFound:
fallthrough
case ErrInputDefinitionNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusNotFound)
}
return
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type defaultInputDefinitionResponse struct{}
func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
inputDefName := mux.Vars(r)["input-definition"]
// Decode request.
var reqs []interface{}
err := json.NewDecoder(r.Body).Decode(&reqs)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err = h.API.WriteInput(r.Context(), indexName, inputDefName, reqs); err != nil {
switch err {
case nil:
break
case ErrIndexNotFound:
fallthrough
case ErrInputDefinitionNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusBadRequest)
}
return
}
if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req setCoordinatorRequest
@ -1577,26 +1448,6 @@ func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request
w.WriteHeader(http.StatusNoContent)
}
// GetTimeStamp retrieves unix timestamp from Input data.
func GetTimeStamp(data map[string]interface{}, timeField string) (int64, error) {
tmstamp, ok := data[timeField]
if !ok {
return 0, nil
}
timestamp, ok := tmstamp.(string)
if !ok {
return 0, fmt.Errorf("set-timestamp value must be in time format: YYYY-MM-DD, has: %v", data[timeField])
}
v, err := time.Parse(TimeFormat, timestamp)
if err != nil {
return 0, errors.Wrap(err, "parsing timestamp")
}
return v.Unix(), nil
}
func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {

482
handler_test.go
View file

@ -1222,275 +1222,6 @@ func TestHandler_Expvars(t *testing.T) {
}
}
// Ensure handler can create a input definition.
func TestHandler_CreateInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
inputBody := []byte(`
{
"frames":[{
"name":"event-time",
"options":{
"timeQuantum": "YMD",
"inverseEnabled": false,
"cacheType": "ranked"
}
}],
"fields": [
{
"name": "columnID",
"primaryKey": true
},
{
"name": "cabType",
"actions": [
{
"frame": "cab-type",
"valueDestination": "mapping",
"valueMap": {
"Green": 1,
"Yellow": 2
}
}
]
}
]
}`)
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusConflict {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrInputDefinitionExists.Error()+"\n" {
t.Fatalf("unexpected body: %s", body)
}
// Test index not found.
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/foo/input-definition/input2", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure throwing error if there's duplicated primaryKey field.
func TestHandler_DuplicatePrimaryKey(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
//Ensure throwing error if there's duplicated primaryKey field
invalidPrimaryKey := []byte(`
{
"frames":[{
"name":"event-time",
"options":{
"timeQuantum": "YMD",
"inverseEnabled": false,
"cacheType": "ranked"
}
}],
"fields": [
{
"name": "columnID",
"primaryKey": true
},
{
"name": "columnID",
"primaryKey": true
}
]
}`)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input2", bytes.NewBuffer(invalidPrimaryKey)))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrInputDefinitionDupePrimaryKey.Error()+"\n" {
t.Fatalf("unexpected body: %s", body)
}
// Ensure throwing error if there's no primary key
hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
unmatchColumnBody := []byte(`
{
"frames":[{
"name":"event-time",
"options":{
"timeQuantum": "YMD",
"inverseEnabled": false,
"cacheType": "ranked"
}
}],
"fields": [
{
"name": "foo",
"actions": [
{
"frame": "cab-type",
"valueDestination": "mapping",
"valueMap": {
"Green": 1,
"Yellow": 2
}
}
]
}
]
}`)
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i1/input-definition/input1", bytes.NewBuffer(unmatchColumnBody)))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrInputDefinitionHasPrimaryKey.Error()+"\n" {
t.Fatalf("unexpected body: %s", body)
}
// Eusure throwing error if request body is invalid.
jsonErrorBody := []byte(`
{
"frames":[{
"name":"event-time",
"options":{
"timeQuantum": "YMD",
"inverseEnabled": false,
"cacheType": "ranked"
}
}],
"fields": [
{
"name": "columnID",
"primaryKey": true
}`)
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(jsonErrorBody)))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `unexpected EOF`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure handler can delete a input definition.
func TestHandler_DeleteInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Test index not found.
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader("")))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" {
t.Fatalf("unexpected body: %s", body)
}
// Test input definition is deleted.
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
_, err := index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
// Test definition not found.
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/foo", strings.NewReader("")))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
}
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/input-definition/test", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
_, err = index.InputDefinition("test")
if err != pilosa.ErrInputDefinitionNotFound {
t.Fatal(err)
}
}
// Ensure handler can get existing input definition.
func TestHandler_GetInputDefinition(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
// Return error if index does not exist.
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader("")))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" {
t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound)
}
// Return existing input definition.
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
inputDef, err := index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
response := &pilosa.InputDefinitionInfo{Frames: inputDef.Frames(), Fields: inputDef.Fields()}
expect, err := json.Marshal(response)
if err != nil {
t.Fatal(err)
}
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/test", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != string(expect)+"\n" {
t.Fatalf("unexpected body: %s, expect: %s", body, string(expect))
}
// Check nonexistent definition.
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/input-definition/foo", strings.NewReader("")))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
var defaultBody = `
{
"frames":[
@ -1589,219 +1320,6 @@ var defaultBody = `
]
}`
func TestHandler_CreateInput(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
defBody := []byte(defaultBody)
def, err := EncodeInputDef("input1", defBody)
if err != nil {
t.Fatal(err)
}
_, err = index.CreateInputDefinition(def)
if err != nil {
t.Fatal(err)
}
inputBody := []byte(`
[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"withPet": true,
"time_value": "2017-03-20T19:35",
"null_value": null
}]`)
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
// Return error if index does not exist.
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/foo/input/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != pilosa.ErrIndexNotFound.Error()+"\n" {
t.Fatalf("unexpected body: %s, expect: %s", body, pilosa.ErrIndexNotFound)
}
// Check nonexistent definition.
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input2", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusNotFound {
t.Fatalf("unexpected status code: %d", w.Code)
}
// Test successfully ingest data
w = httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer(inputBody)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
// Verify the bits set per frame.
f0 := index.Frame("distance-miles")
v0 := f0.View(pilosa.ViewStandard)
fragment0 := v0.Fragment(0)
// Verify the distanceMiles Bit was set.
if a := fragment0.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{1}) {
t.Fatalf("unexpected bits: %+v", a)
}
f1 := index.Frame("add-ons")
v1 := f1.View(pilosa.ViewStandard)
fragment1 := v1.Fragment(0)
// Verify the add-ons frame does not have a distanceMiles Bit set.
// The Input process must respect the Action Frame assignments
if a := fragment1.Row(8).Bits(); !reflect.DeepEqual(a, []uint64{}) {
t.Fatalf("unexpected bits: %+v", a)
}
// Verify the withPet Bit was set.
if a := fragment1.Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
func TestInput_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
defBody := []byte(defaultBody)
def, err := EncodeInputDef("input1", defBody)
if err != nil {
t.Fatal(err)
}
_, err = index.CreateInputDefinition(def)
if err != nil {
t.Fatal(err)
}
tests := []struct {
json string
err string
}{
{json: `[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"nofield": true
}]`,
err: "field not found: nofield"},
{json: `[{
"id": "abc",
"cabType": "yellow",
"distanceMiles": 8,
"withPet": true
}]`,
err: "float64 require, got value:abc, type: string"},
{json: `[{
"cabType": "yellow",
"distanceMiles": 8,
"withPet": true
}]`,
err: "primary key does not exist"},
{json: `[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"withPet": true
}`,
err: "unexpected EOF"},
{json: `[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"noFrame": 1
}]`,
err: "Frame not found: foo"},
{json: `[{
"id": 1,
"cabType": "yellow",
"distanceMiles": 8,
"time_value": 12345
}]`,
err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"},
}
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
for _, req := range tests {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json))))
if body := w.Body.String(); body != req.err+"\n" {
t.Fatalf("Expect error: %s, actual: %s", req.err, body)
}
}
}
func EncodeInputDef(name string, body []byte) (*internal.InputDefinition, error) {
var req pilosa.InputDefinitionInfo
err := json.Unmarshal(body, &req)
if err != nil {
return nil, err
}
def := req.Encode()
def.Name = name
return def, nil
}
func TestHandler_GetTimeStamp(t *testing.T) {
data := make(map[string]interface{})
timeField := "time"
data["time"] = "2017-03-20T19:35"
val, err := pilosa.GetTimeStamp(data, timeField)
if val != 1490038500 {
t.Fatalf("Timestamp is not set correctly for %s", data["time"])
}
// Verify that an integer is not a valid time format.
data["int"] = 1490000000
val, err = pilosa.GetTimeStamp(data, "int")
if !strings.Contains(err.Error(), "set-timestamp value must be in time format") {
t.Fatalf("Expected set-timestamp value must be in time format error, actual error: %s", err)
}
// Verify reversing month and year is not valid time format.
data["time"] = "03-2017-20T19:35"
val, err = pilosa.GetTimeStamp(data, timeField)
if !strings.Contains(err.Error(), "cannot parse") {
t.Fatalf("Expected Timestamp is not set correctly, actual error: %s", err)
}
// Handle time fields that do not exist.
val, err = pilosa.GetTimeStamp(data, "test")
if val != 0 {
t.Fatalf("Expected Ignore nonexistent fields")
}
}
// Ensure handler can delete a view.
func TestHandler_DeleteView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
viewName := pilosa.ViewStandard + "_2017"
hldr.MustCreateFragmentIfNotExists("i0", "f0", viewName, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD")
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Frame("f0").View(viewName); f != nil {
t.Fatal("expected nil view")
}
}
func MustReadAll(r io.Reader) []byte {
buf, err := ioutil.ReadAll(r)
if err != nil {

1
holder.go
View file

@ -262,7 +262,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
}
}
}
// TODO: Create inputDefinitions that don't exist.
}
return nil
}

196
index.go
View file

@ -28,11 +28,6 @@ import (
"github.com/pkg/errors"
)
// Default index settings.
const (
InputDefinitionDir = ".input-definitions"
)
// Index represents a container for frames.
type Index struct {
mu sync.RWMutex
@ -51,9 +46,6 @@ type Index struct {
// Column attribute storage and cache.
columnAttrStore AttrStore
// InputDefinitions by name.
inputDefinitions map[string]*InputDefinition
broadcaster Broadcaster
Stats StatsClient
@ -68,10 +60,9 @@ func NewIndex(path, name string) (*Index, error) {
}
return &Index{
path: path,
name: name,
frames: make(map[string]*Frame),
inputDefinitions: make(map[string]*InputDefinition),
path: path,
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
@ -125,10 +116,6 @@ func (i *Index) Open() error {
return errors.Wrap(err, "opening attrstore")
}
if err := i.openInputDefinitions(); err != nil {
return err
}
return nil
}
@ -146,7 +133,7 @@ func (i *Index) openFrames() error {
}
for _, fi := range fis {
if !fi.IsDir() || fi.Name() == InputDefinitionDir {
if !fi.IsDir() {
continue
}
@ -276,11 +263,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
// InputDefinitionPath returns the path to the input definition directory for the index.
func (i *Index) InputDefinitionPath() string {
return filepath.Join(i.path, InputDefinitionDir)
}
// Frame returns a frame in the index by name.
func (i *Index) Frame(name string) *Frame {
i.mu.RLock()
@ -288,20 +270,8 @@ func (i *Index) Frame(name string) *Frame {
return i.frame(name)
}
// InputDefinition returns an input definition in the index by name.
func (i *Index) InputDefinition(name string) (*InputDefinition, error) {
i.mu.Lock()
defer i.mu.Unlock()
if inputDef, ok := i.inputDefinitions[name]; ok {
return inputDef, nil
}
return nil, ErrInputDefinitionNotFound
}
func (i *Index) frame(name string) *Frame { return i.frames[name] }
func (i *Index) inputDefinition(name string) *InputDefinition { return i.inputDefinitions[name] }
// Frames returns a list of all frames in the index.
func (i *Index) Frames() []*Frame {
i.mu.RLock()
@ -316,20 +286,6 @@ func (i *Index) Frames() []*Frame {
return a
}
// InputDefinitions returns a list of all inputDefinitions in the index.
func (i *Index) InputDefinitions() []*InputDefinition {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*InputDefinition, 0, len(i.inputDefinitions))
for _, d := range i.inputDefinitions {
a = append(a, d)
}
//sort.Sort(inputDefintionSlice(a)) // TODO
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
@ -493,9 +449,8 @@ func EncodeIndexes(a []*Index) []*internal.Index {
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Frames: encodeFrames(d.Frames()),
InputDefinitions: encodeInputDefinitions(d.InputDefinitions()),
Name: d.name,
Frames: encodeFrames(d.Frames()),
}
}
@ -531,142 +486,3 @@ type importValueData struct {
ColumnIDs []uint64
Values []int64
}
// CreateInputDefinition creates a new input definition.
func (i *Index) CreateInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) {
// Ensure input definition doesn't already exist.
if i.inputDefinitions[pb.Name] != nil {
return nil, ErrInputDefinitionExists
}
return i.createInputDefinition(pb)
}
func (i *Index) createInputDefinition(pb *internal.InputDefinition) (*InputDefinition, error) {
if pb.Name == "" {
return nil, ErrInputDefinitionNameRequired
}
for _, fr := range pb.Frames {
opt := FrameOptions{
// Deprecating row labels per #810. So, setting the default row label here.
InverseEnabled: fr.Meta.InverseEnabled,
CacheType: fr.Meta.CacheType,
CacheSize: fr.Meta.CacheSize,
TimeQuantum: TimeQuantum(fr.Meta.TimeQuantum),
}
_, err := i.CreateFrame(fr.Name, opt)
if err == ErrFrameExists {
continue
} else if err != nil {
return nil, err
}
}
// Initialize input definition.
inputDef, err := i.newInputDefinition(pb.Name)
if err != nil {
return nil, err
}
if err = inputDef.LoadDefinition(pb); err != nil {
return nil, err
}
if err = inputDef.saveMeta(); err != nil {
return nil, err
}
i.inputDefinitions[pb.Name] = inputDef
return inputDef, nil
}
func (i *Index) newInputDefinition(name string) (*InputDefinition, error) {
inputDef, err := NewInputDefinition(i.InputDefinitionPath(), i.name, name)
if err != nil {
return nil, err
}
return inputDef, nil
}
// DeleteInputDefinition removes an input definition from the index.
func (i *Index) DeleteInputDefinition(name string) error {
// Fail if input definition doesn't exist.
_, err := i.InputDefinition(name)
if err != nil {
return err
}
i.mu.Lock()
defer i.mu.Unlock()
// Delete input definition file.
if err := os.Remove(filepath.Join(i.InputDefinitionPath(), name)); err != nil {
return err
}
// Remove reference.
delete(i.inputDefinitions, name)
return nil
}
// openInputDefinitions opens and initializes the input definitions inside the index.
func (i *Index) openInputDefinitions() error {
inputDef, err := os.Open(i.InputDefinitionPath())
if os.IsNotExist(err) {
return nil
} else if err != nil {
return err
}
defer inputDef.Close()
inputFiles, err := inputDef.Readdir(0)
for _, file := range inputFiles {
input, err := i.newInputDefinition(file.Name())
if err != nil {
return err
}
input.Open()
i.inputDefinitions[file.Name()] = input
// Create frame if it doesn't exist.
for _, fr := range input.frames {
_, err := i.CreateFrame(fr.Name, fr.Options)
if err == ErrFrameExists {
continue
} else if err != nil {
return nil
}
}
}
return nil
}
// InputBits Process the []Bit though the Frame import process
func (i *Index) InputBits(frame string, bits []*Bit) error {
var rowIDs, columnIDs []uint64
var timestamps []*time.Time
f := i.Frame(frame)
if f == nil {
return fmt.Errorf("Frame not found: %s", frame)
}
for i, bit := range bits {
if bit == nil {
continue
}
rowIDs = append(rowIDs, bit.RowID)
columnIDs = append(columnIDs, bit.ColumnID)
// Convert timestamps to time.Time.
if bit.Timestamp > 0 {
// Don't create a full timestamps slice unless
// at least one bit contains a timestamp.
if len(timestamps) == 0 {
timestamps = make([]*time.Time, len(bits))
}
t := time.Unix(bit.Timestamp, 0)
timestamps[i] = &t
}
}
return f.Import(rowIDs, columnIDs, timestamps)
}

137
index_test.go
View file

@ -17,11 +17,9 @@ package pilosa_test
import (
"io/ioutil"
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/test"
)
@ -255,138 +253,3 @@ func TestIndex_InvalidName(t *testing.T) {
t.Fatalf("unexpected index name %v", index)
}
}
func TestIndex_CreateInputDefinition(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
field := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&field}}
inputDef, err := index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
} else if inputDef.Frames()[0].Name != frames.Name {
t.Fatalf("unexpected input definition frames %v", inputDef.Frames())
} else if inputDef.Fields()[0].Name != field.Name {
t.Fatalf("unexpected input definition actions %v", inputDef.Fields())
}
}
// Ensure create input definition handle correct error
func TestIndex_CreateExistingInputDefinition(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
//Test input definition name is required
def := internal.InputDefinition{Name: "", Frames: []*internal.Frame{}, Fields: []*internal.InputDefinitionField{}}
_, err := index.CreateInputDefinition(&def)
if err != pilosa.ErrInputDefinitionNameRequired {
t.Fatal(err)
}
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
_, err = index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
_, err = index.CreateInputDefinition(&def)
if err != pilosa.ErrInputDefinitionExists {
t.Fatal(err)
}
}
// Ensure to delete existing input definition.
func TestIndex_DeleteInputDefinition(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
_, err := index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
_, err = index.InputDefinition("test")
if err != nil {
t.Fatal(err)
}
err = index.DeleteInputDefinition("test")
if err != nil {
t.Fatal(err)
}
_, err = index.InputDefinition("test")
if err != pilosa.ErrInputDefinitionNotFound {
t.Fatal(err)
}
}
// Ensure that frame in input definition will be created when server restart
func TestIndex_CreateFrameWhenOpenInputDefinition(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
input, err := index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
input.AddFrame(pilosa.InputFrame{Name: "f1"})
index.Reopen()
if index.Frame("f1") == nil {
t.Fatal("Frame does not created when open index")
}
}
func TestIndex_InputBits(t *testing.T) {
var bits []*pilosa.Bit
index := test.MustOpenIndex()
defer index.Close()
err := index.InputBits("f", bits)
if !strings.Contains(err.Error(), "Frame not found") {
t.Fatalf("Expected Frame not found error, actual error: %s", err)
}
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil {
t.Fatal(err)
}
bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 0})
bits = append(bits, &pilosa.Bit{RowID: 0, ColumnID: 1})
bits = append(bits, &pilosa.Bit{RowID: 2, ColumnID: 2, Timestamp: 1})
bits = append(bits, nil)
err = index.InputBits("f", bits)
if err != nil {
t.Fatal(err)
}
f := index.Frame("f")
v := f.View(pilosa.ViewStandard)
fragment := v.Fragment(0)
// Verify the Bits were set
if a := fragment.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{0, 1}) {
t.Fatalf("unexpected bits: %+v", a)
}
}

420
input_definition.go
View file

@ -1,420 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// Action types.
const (
InputMapping = "mapping"
InputValueToRow = "value-to-row"
InputSingleRowBool = "single-row-boolean"
InputSetTimestamp = "set-timestamp"
)
var validValueDestination = []string{InputMapping, InputValueToRow, InputSingleRowBool, InputSetTimestamp}
// InputDefinition represents a container for the data input definition.
type InputDefinition struct {
name string
path string
index string
frames []InputFrame
fields []InputDefinitionField
}
// NewInputDefinition returns a new instance of InputDefinition.
func NewInputDefinition(path, index, name string) (*InputDefinition, error) {
err := ValidateName(name)
if err != nil {
return nil, err
}
return &InputDefinition{
path: path,
index: index,
name: name,
}, nil
}
// Frames returns frames of the input definition was initialized with.
func (i *InputDefinition) Frames() []InputFrame { return i.frames }
// Fields returns fields of the input definition was initialized with.
func (i *InputDefinition) Fields() []InputDefinitionField { return i.fields }
// Open opens and initializes the InputDefinition from file.
func (i *InputDefinition) Open() error {
if err := func() error {
if err := os.MkdirAll(i.path, 0777); err != nil {
return err
}
if err := i.loadMeta(); err != nil {
return err
}
return nil
}(); err != nil {
return err
}
return nil
}
// LoadDefinition loads the protobuf format of a definition.
func (i *InputDefinition) LoadDefinition(pb *internal.InputDefinition) error {
// Copy metadata fields.
i.name = pb.Name
for _, fr := range pb.Frames {
inputFrame := InputFrame{
Name: fr.Name,
Options: *decodeFrameOptions(fr.Meta),
}
i.frames = append(i.frames, inputFrame)
}
primaryKeyGiven := false
for _, field := range pb.Fields {
var actions []Action
for _, action := range field.InputDefinitionActions {
actions = append(actions, Action{
Frame: action.Frame,
ValueDestination: action.ValueDestination,
ValueMap: action.ValueMap,
RowID: &action.RowID,
})
}
if field.PrimaryKey {
primaryKeyGiven = true
}
inputField := InputDefinitionField{
Name: field.Name,
PrimaryKey: field.PrimaryKey,
Actions: actions,
}
i.fields = append(i.fields, inputField)
}
if len(pb.Fields) > 0 && !primaryKeyGiven {
return ErrInputDefinitionHasPrimaryKey
}
return nil
}
func (i *InputDefinition) loadMeta() error {
var pb internal.InputDefinition
buf, err := ioutil.ReadFile(filepath.Join(i.path, i.name))
if err != nil {
return err
}
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
return i.LoadDefinition(&pb)
}
// saveMeta writes meta data for the input definition file.
func (i *InputDefinition) saveMeta() error {
if err := os.MkdirAll(i.path, 0777); err != nil {
return err
}
var frames []*internal.Frame
for _, fr := range i.frames {
frames = append(frames, fr.Encode())
}
var fields []*internal.InputDefinitionField
for _, field := range i.fields {
fields = append(fields, field.Encode())
}
// Marshal input definition.
buf, err := proto.Marshal(&internal.InputDefinition{
Name: i.name,
Frames: frames,
Fields: fields,
})
if err != nil {
return err
}
// Write to meta file.
if err := ioutil.WriteFile(filepath.Join(i.path, i.name), buf, 0666); err != nil {
return err
}
return nil
}
// InputDefinitionField descripes a single field mapping in the InputDefinition.
type InputDefinitionField struct {
Name string `json:"name,omitempty"`
PrimaryKey bool `json:"primaryKey,omitempty"`
Actions []Action `json:"actions,omitempty"`
}
// Encode converts InputDefinitionField into its internal representation.
func (o *InputDefinitionField) Encode() *internal.InputDefinitionField {
var actions []*internal.InputDefinitionAction
for _, action := range o.Actions {
actions = append(actions, action.Encode())
}
return &internal.InputDefinitionField{
Name: o.Name,
PrimaryKey: o.PrimaryKey,
InputDefinitionActions: actions,
}
}
// Action describes the mapping method for the field in the InputDefinition.
type Action struct {
Frame string `json:"frame,omitempty"`
ValueDestination string `json:"valueDestination,omitempty"`
ValueMap map[string]uint64 `json:"valueMap,omitempty"`
RowID *uint64 `json:"rowID,omitempty"`
}
// Validate ensures the input definition action conforms to our specification.
func (a *Action) Validate() error {
if a.Frame == "" {
return ErrFrameRequired
}
if !foundItem(validValueDestination, a.ValueDestination) {
return fmt.Errorf("invalid ValueDestination: %s", a.ValueDestination)
}
switch a.ValueDestination {
case InputMapping:
if len(a.ValueMap) == 0 {
return ErrInputDefinitionValueMap
}
}
return nil
}
// Encode converts Action into its internal representation.
func (a *Action) Encode() *internal.InputDefinitionAction {
return &internal.InputDefinitionAction{
Frame: a.Frame,
ValueDestination: a.ValueDestination,
ValueMap: a.ValueMap,
RowID: convert(a.RowID),
}
}
// convert pointer to uint64.
func convert(x *uint64) uint64 {
if x != nil {
return *x
}
return 0
}
// InputFrame defines the frame used in the input definition.
type InputFrame struct {
Name string `json:"name,omitempty"`
Options FrameOptions `json:"options,omitempty"`
}
// Validate the InputFrame data.
func (i *InputFrame) Validate() error {
if err := ValidateName(i.Name); err != nil {
return err
}
// TODO frame option validation
return nil
}
// Encode converts InputFrame into its internal representation.
func (i *InputFrame) Encode() *internal.Frame {
return &internal.Frame{
Name: i.Name,
Meta: i.Options.Encode(),
}
}
// InputDefinitionInfo represents the json message format needed to create an InputDefinition.
type InputDefinitionInfo struct {
Frames []InputFrame `json:"frames"`
Fields []InputDefinitionField `json:"fields"`
}
// Validate the InputDefinitionInfo data.
func (i *InputDefinitionInfo) Validate() error {
numPrimaryKey := 0
accountRowID := make(map[string]uint64)
if len(i.Frames) == 0 || len(i.Fields) == 0 {
return ErrInputDefinitionAttrsRequired
}
for _, frame := range i.Frames {
if err := frame.Validate(); err != nil {
return err
}
}
// Validate duplicate primaryKey.
for _, field := range i.Fields {
if field.Name == "" {
return ErrInputDefinitionNameRequired
}
for _, action := range field.Actions {
if err := action.Validate(); err != nil {
return err
}
if action.ValueDestination == InputSingleRowBool {
if action.RowID == nil {
return fmt.Errorf("rowID required for single-row-boolean Field %s", field.Name)
}
val, ok := accountRowID[action.Frame]
if ok && val == convert(action.RowID) {
return fmt.Errorf("duplicate rowID with other field: %v", action.RowID)
}
accountRowID[action.Frame] = convert(action.RowID)
}
}
if field.PrimaryKey {
numPrimaryKey++
} else if len(field.Actions) == 0 {
return ErrInputDefinitionActionRequired
}
}
if len(i.Fields) > 0 && numPrimaryKey == 0 {
return ErrInputDefinitionHasPrimaryKey
}
if numPrimaryKey > 1 {
return ErrInputDefinitionDupePrimaryKey
}
return nil
}
// Encode converts InputDefinitionInfo into its internal representation.
func (i *InputDefinitionInfo) Encode() *internal.InputDefinition {
var def internal.InputDefinition
for _, f := range i.Frames {
def.Frames = append(def.Frames, f.Encode())
}
for _, f := range i.Fields {
def.Fields = append(def.Fields, f.Encode())
}
return &def
}
// encodeInputDefinitions converts a into its internal representation.
func encodeInputDefinitions(a []*InputDefinition) []*internal.InputDefinition {
other := make([]*internal.InputDefinition, len(a))
for i := range a {
other[i] = encodeInputDefinition(a[i])
}
return other
}
// encodeInputDefinition converts i into its internal representation.
func encodeInputDefinition(i *InputDefinition) *internal.InputDefinition {
//fo := f.options()
return &internal.InputDefinition{
Name: i.name,
Frames: encodeInputFrames(i.frames),
Fields: encodeInputDefinitionFields(i.fields),
}
}
// encodeInputFrames converts a into its internal representation.
func encodeInputFrames(a []InputFrame) []*internal.Frame {
other := make([]*internal.Frame, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// encodeInputDefinitionFields converts a into its internal representation.
func encodeInputDefinitionFields(a []InputDefinitionField) []*internal.InputDefinitionField {
other := make([]*internal.InputDefinitionField, len(a))
for i := range a {
other[i] = a[i].Encode()
}
return other
}
// AddFrame manually add frame to input definition.
func (i *InputDefinition) AddFrame(frame InputFrame) error {
i.frames = append(i.frames, frame)
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// HandleAction Process the input data with its action and return a bit to be imported later
// Note: if the Bit should not be set then nil is returned with no error
// From the JSON marshalling the possible types are: float64, boolean, string
func HandleAction(a Action, value interface{}, colID uint64, timestamp int64) (*Bit, error) {
var err error
var bit Bit
bit.ColumnID = colID
bit.Timestamp = timestamp
switch a.ValueDestination {
case InputMapping:
v, ok := value.(string)
if !ok {
return nil, fmt.Errorf("Mapping value must be a string %v", value)
}
bit.RowID, ok = a.ValueMap[v]
if !ok {
return nil, fmt.Errorf("Value %s does not exist in definition map", v)
}
case InputSingleRowBool:
v, ok := value.(bool)
if !ok {
return nil, fmt.Errorf("single-row-boolean value %v must equate to a Bool", value)
}
if v == false { // False returns a nil error and nil bit.
return nil, err
}
bit.RowID = *a.RowID
case InputValueToRow:
v, ok := value.(float64)
if !ok {
return nil, fmt.Errorf("value-to-row value must equate to an integer %v", value)
}
bit.RowID = uint64(v)
case InputSetTimestamp:
// InputSetTimestamp action is used in the InputJSONDataParser Handler to append a timestamp to all bits in the frame.
// There are no individual rowID's to set, and the action is a no-op at this step
return nil, nil
default:
return nil, fmt.Errorf("Unrecognized Value Destination: %s in Action", a.ValueDestination)
}
return &bit, err
}

253
input_definition_test.go
View file

@ -1,253 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"encoding/json"
"testing"
"strings"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/test"
)
func TestInputDefinition_Open(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create Input Definition.
frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{}}
action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}}
fields := internal.InputDefinitionField{Name: "id", PrimaryKey: true, InputDefinitionActions: []*internal.InputDefinitionAction{&action}}
def := internal.InputDefinition{Name: "^", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
inputDef, err := index.CreateInputDefinition(&def)
if !strings.Contains(err.Error(), "invalid index or frame's name") {
t.Fatalf("Expected Invalid name error, actual error: %s", err)
}
def = internal.InputDefinition{Name: "test", Frames: []*internal.Frame{&frames}, Fields: []*internal.InputDefinitionField{&fields}}
inputDef, err = index.CreateInputDefinition(&def)
if err != nil {
t.Fatal(err)
}
err = inputDef.Open()
if err != nil {
t.Fatal(err)
}
}
// Verify the InputDefinition Encoding to the internal format
func TestInputDefinition_Encoding(t *testing.T) {
inputBody := []byte(`
{
"frames":[{
"name":"event-time",
"options":{
"timeQuantum": "YMD",
"inverseEnabled": true,
"cacheType": "ranked"
}
}],
"fields": [
{
"name": "id",
"primaryKey": true
},
{
"name": "cabType",
"actions": [
{
"frame": "cab-type",
"valueDestination": "mapping",
"valueMap": {
"Green": 1,
"Yellow": 2
}
}
]
}
]
}`)
var def pilosa.InputDefinitionInfo
err := json.Unmarshal(inputBody, &def)
if err != nil {
t.Fatal(err)
}
internalDef := def.Encode()
if internalDef.Frames[0].Name != "event-time" {
t.Fatalf("unexpected frame: %v", internalDef)
} else if internalDef.Frames[0].Meta.CacheType != "ranked" {
t.Fatalf("unexpected frame meta data: %v", internalDef)
} else if len(internalDef.Fields) != 2 {
t.Fatalf("unexpected number of Fields: %d", len(internalDef.Fields))
} else if len(internalDef.Fields[1].InputDefinitionActions) != 1 {
t.Fatalf("unexpected number of Actions: %v", internalDef.Fields[1].InputDefinitionActions)
} else if internalDef.Fields[1].InputDefinitionActions[0].ValueDestination != "mapping" {
t.Fatalf("unexpected ValueDestination: %v", internalDef.Fields[1].InputDefinitionActions[0])
}
}
// Test The Action validation cases
func TestActionValidation(t *testing.T) {
rowID := uint64(100)
action := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, ValueMap: map[string]uint64{"Green": 1}}
field := pilosa.InputDefinitionField{Name: "id", PrimaryKey: false, Actions: []pilosa.Action{action}}
info := pilosa.InputDefinitionInfo{Fields: []pilosa.InputDefinitionField{field}}
err := info.Validate()
if err != pilosa.ErrInputDefinitionAttrsRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionAttrsRequired, err)
}
frame := pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if !strings.Contains(err.Error(), "rowID required for single-row-boolean") {
t.Fatalf("Expected rowID required for single-row-boolean error, actual error: %s", err)
}
frame = pilosa.InputFrame{Name: "^", Options: pilosa.FrameOptions{}}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if err != pilosa.ErrName {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrName, err)
}
frame = pilosa.InputFrame{Name: "f", Options: pilosa.FrameOptions{}}
action = pilosa.Action{ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if err != pilosa.ErrFrameRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err)
}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "x", PrimaryKey: false, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if err != pilosa.ErrInputDefinitionHasPrimaryKey {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionHasPrimaryKey, err)
}
action = pilosa.Action{Frame: "f", ValueDestination: "value-to-ROW", ValueMap: map[string]uint64{"Green": 1}}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if !strings.Contains(err.Error(), "invalid ValueDestination") {
t.Fatalf("Expected invalid ValueDestination error, actual error: %s", err)
}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputMapping, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field}}
err = info.Validate()
if err != pilosa.ErrInputDefinitionValueMap {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionValueMap, err)
}
action = pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false, Actions: []pilosa.Action{action}}
action1 := pilosa.Action{Frame: "f", ValueDestination: pilosa.InputSingleRowBool, RowID: &rowID}
field1 := pilosa.InputDefinitionField{Name: "id", PrimaryKey: true, Actions: []pilosa.Action{action1}}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}}
err = info.Validate()
if !strings.Contains(err.Error(), "duplicate rowID with other field") {
t.Fatalf("Expected duplicate rowID with other field error, actual error: %s", err)
}
field = pilosa.InputDefinitionField{Name: "id", PrimaryKey: true}
field1 = pilosa.InputDefinitionField{Name: "test", PrimaryKey: false}
info = pilosa.InputDefinitionInfo{Frames: []pilosa.InputFrame{frame}, Fields: []pilosa.InputDefinitionField{field, field1}}
err = info.Validate()
if err != pilosa.ErrInputDefinitionActionRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrInputDefinitionActionRequired, err)
}
}
func TestHandleAction(t *testing.T) {
var value interface{}
colID := uint64(0)
rowID := uint64(100)
action := pilosa.Action{RowID: &rowID}
timestamp := int64(0)
tests := []struct {
action string
name string
value interface{}
expected uint64
err string
}{
{name: "integer single-row-bool", action: pilosa.InputSingleRowBool, value: 1, err: "single-row-boolean value"},
{name: "string single-row-bool", action: pilosa.InputSingleRowBool, value: "1", err: "single-row-boolean value 1 must equate to a Bool"},
{name: "string value-to-row", action: pilosa.InputValueToRow, value: "25", err: "value-to-row value must equate to an integer"},
{name: "string mapping", action: pilosa.InputMapping, value: "test", err: "Value test does not exist in definition map"},
{name: "int mapping", action: pilosa.InputMapping, value: 25, err: "Mapping value must be a string"},
{name: "invalid action", action: "test", value: true, err: "Unrecognized Value Destination"},
}
for _, r := range tests {
t.Run(r.name, func(t *testing.T) {
action.ValueDestination = r.action
_, err := pilosa.HandleAction(action, r.value, colID, timestamp)
if !strings.Contains(err.Error(), r.err) {
t.Fatalf("Expect err: %s, actual: %s", r.err, err.Error())
}
})
}
value = true
action.ValueDestination = pilosa.InputSingleRowBool
b, err := pilosa.HandleAction(action, value, colID, timestamp)
if err != nil {
t.Fatalf("err with HandleAction: %v", err)
}
if b != nil {
if b.ColumnID != 0 {
t.Fatalf("Unexpected ColumnID %v", b.ColumnID)
}
if b.RowID != 100 {
t.Fatalf("Unexpected rowID %v", b.RowID)
}
}
action.ValueDestination = pilosa.InputValueToRow
rowID = 101
value = float64(25.0)
b, _ = pilosa.HandleAction(action, value, colID, timestamp)
if b != nil {
if b.RowID != 25 {
t.Fatalf("Unexpected RowID %v", b.RowID)
}
}
action.ValueDestination = pilosa.InputSetTimestamp
t.Run("nil bit", func(t *testing.T) {
b, err = pilosa.HandleAction(action, value, colID, timestamp)
if err != nil {
t.Fatalf("err with HandleAction: %v", err)
}
if b != nil {
t.Fatalf("Expected nil bit is set")
}
})
}

1743
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

31
internal/private.proto
View file

@ -91,37 +91,6 @@ message Schema {
message Index {
string Name = 1;
repeated Frame Frames = 4;
repeated InputDefinition InputDefinitions = 6;
}
message InputDefinition {
string Name = 1;
repeated Frame Frames = 2;
repeated InputDefinitionField Fields = 3;
}
message InputDefinitionField {
string Name = 1;
bool PrimaryKey = 2;
repeated InputDefinitionAction InputDefinitionActions = 3;
}
message InputDefinitionAction {
string Frame = 1;
string ValueDestination = 2;
map<string, uint64> ValueMap = 3;
uint64 RowID = 4;
}
message CreateInputDefinitionMessage {
string Index = 1;
InputDefinition Definition = 3;
}
message DeleteInputDefinitionMessage {
string Index = 1;
string Name = 2;
}
message URI {

35
internal/public.pb.go
View file

@ -1,6 +1,5 @@
// Code generated by protoc-gen-gogo.
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: public.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import binary "encoding/binary"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
@ -807,7 +808,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue))))
binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
i += 8
}
return i, nil
}
@ -1249,24 +1251,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Public(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Public(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -2351,15 +2335,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if (iNdEx + 8) > l {
return io.ErrUnexpectedEOF
}
v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
v = uint64(dAtA[iNdEx-8])
v |= uint64(dAtA[iNdEx-7]) << 8
v |= uint64(dAtA[iNdEx-6]) << 16
v |= uint64(dAtA[iNdEx-5]) << 24
v |= uint64(dAtA[iNdEx-4]) << 32
v |= uint64(dAtA[iNdEx-3]) << 40
v |= uint64(dAtA[iNdEx-2]) << 48
v |= uint64(dAtA[iNdEx-1]) << 56
m.FloatValue = float64(math.Float64frombits(v))
default:
iNdEx = preIndex

9
pilosa.go
View file

@ -37,15 +37,6 @@ var (
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrInputDefinitionExists = errors.New("input-definition already exists")
ErrInputDefinitionHasPrimaryKey = errors.New("input-definition must contain one PrimaryKey")
ErrInputDefinitionDupePrimaryKey = errors.New("input-definition can only contain one PrimaryKey")
ErrInputDefinitionNameRequired = errors.New("input-definition name required")
ErrInputDefinitionAttrsRequired = errors.New("frames and fields are required")
ErrInputDefinitionValueMap = errors.New("valueMap required for map")
ErrInputDefinitionActionRequired = errors.New("field definitions require an action")
ErrInputDefinitionNotFound = errors.New("input-definition not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")

12
server.go
View file

@ -476,18 +476,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err := f.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateInputDefinitionMessage:
idx := s.Holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
idx.CreateInputDefinition(obj.Definition)
case *internal.DeleteInputDefinitionMessage:
idx := s.Holder.Index(obj.Index)
err := idx.DeleteInputDefinition(obj.Name)
if err != nil {
return err
}
case *internal.CreateViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {

26
server/cluster_test.go
View file

@ -118,32 +118,6 @@ func TestMain_SendReceiveMessage(t *testing.T) {
if maxSlices1["i"] != 2 {
t.Fatalf("unexpected maxSlice on node1: %d", maxSlices1["i"])
}
// Write input definition to the first node.
if _, err := m0.CreateDefinition("i", "test", `{
"frames": [{"name": "event-time",
"options": {
"cacheType": "ranked",
"timeQuantum": "YMD"
}}],
"fields": [{"name": "col",
"primaryKey": true
}]}
`); err != nil {
t.Fatal(err)
}
// We have to wait for the broadcast message to be sent before checking state.
time.Sleep(1 * time.Second)
frame0 := m0.Server.Holder.Frame("i", "event-time")
if frame0 == nil {
t.Fatal("frame not found")
}
frame1 := m1.Server.Holder.Frame("i", "event-time")
if frame1 == nil {
t.Fatal("frame not found")
}
}
// Ensure that an empty node comes up in a NORMAL state.

9
test/pilosa.go
View file

@ -255,15 +255,6 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) {
return resp.Body, nil
}
// CreateDefinition.
func (m *Main) CreateDefinition(index, def, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/input-definition/%s", index, def), query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
}
func (m *Main) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
if resp.StatusCode != 204 {