From 1e6d0a433effc81a2d7e7952664cfa0e78e9890d Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 19 Jun 2018 16:19:36 +0300 Subject: [PATCH 1/8] Updated getting started section for latest develop --- docs/getting-started.md | 49 ++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 276b0bff9..7f5759844 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -34,14 +34,15 @@ Let's make sure Pilosa is running: curl localhost:10101/status ``` ``` response -{"state":"NORMAL","nodes":[{"id":"18eb5546-5a1a-4ba4-9c52-b53fbe22317e","uri":{"scheme":"http","host":"localhost","port":10101}}]} +{"state":"NORMAL","nodes":[{"id":"91715a50-7d50-4c54-9a03-873801da1cd1","uri":{"scheme":"http","host":"localhost","port +":10101},"isCoordinator":true}],"localID":"91715a50-7d50-4c54-9a03-873801da1cd1"} ``` ### Sample Project In order to better understand Pilosa's capabilities, we will create a sample project called "Star Trace" containing information about 1,000 popular Github repositories which have "go" in their name. The Star Trace index will include data points such as programming language, tags, and stargazers—people who have starred a project. -Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Frames. So the "repository" index might have a "languages" frame as well as a "tags" frame. You can learn more about indexes and frames in the [Data Model](../data-model/) section of the documentation. +Although Pilosa doesn't keep the data in a tabular format, we still use the terms "columns" and "rows" when describing the data model. We put the primary objects in columns, and the properties of those objects in rows. For example, the Star Trace project will contain an index called "repository" which contains columns representing Github repositories, and rows representing properties like programming languages and tags. We can better organize the rows by grouping them into sets called Fields. So the "repository" index might have a "languages" field as well as a "tags" field. You can learn more about indexes and fields in the [Data Model](../data-model/) section of the documentation. #### Create the Schema @@ -55,7 +56,7 @@ curl localhost:10101/schema {"indexes":null} ``` -Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: +Before we can import data or run queries, we need to create our indexes and the fields within them. Let's create the repository index first: ``` request curl localhost:10101/index/repository -X POST ``` @@ -63,27 +64,29 @@ curl localhost:10101/index/repository -X POST {} ``` -Let's create the `stargazer` frame which has user IDs of stargazers as its rows: +Let's create the `stargazer` field which has user IDs of stargazers as its rows: ``` request -curl localhost:10101/index/repository/frame/stargazer \ +curl localhost:10101/index/repository/field/stargazer \ -X POST \ - -d '{"options": {"timeQuantum": "YMD"}}' + -d '{"options": {"type": "time", "timeQuantum": "YMD"}}' ``` ``` response {} ``` -Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. +Since our data contains time stamps for the time users starred repos, we set the field type to `time`. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. -Next up is the `language` frame, which will contain IDs for programming languages: +Next up is the `language` field, which will contain IDs for programming languages: ``` request -curl localhost:10101/index/repository/frame/language \ +curl localhost:10101/index/repository/field/language \ -X POST ``` ``` response {} ``` +The `language` is a `set` field, but since the default field type is `set`, we didn't specify it in field options. + #### Import Data From CSV Files Download the `stargazer.csv` and `language.csv` files here: @@ -116,14 +119,14 @@ Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(frame="stargazer", row=14)' + -d 'Bitmap(field="stargazer", row=14)' ``` ``` response { "results":[ { "attrs":{}, - "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + "columns":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] } ] } @@ -133,7 +136,7 @@ What are the top 5 languages in the sample data: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'TopN(frame="language", n=5)' + -d 'TopN(field="language", n=5)' ``` ``` response { @@ -154,8 +157,8 @@ Which repositories were starred by user 14 and 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19) + Bitmap(field="stargazer", row=14), + Bitmap(field="stargazer", row=19) )' ``` ``` response @@ -163,7 +166,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[2,3,362,396,416,461,464,466,470,486] + "columns":[2,3,362,396,416,461,464,466,470,486] } ] } @@ -174,8 +177,8 @@ Which repositories were starred by user 14 or 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Union( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19) + Bitmap(field="stargazer", row=14), + Bitmap(field="stargazer", row=19) )' ``` ``` response @@ -183,7 +186,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + "columns":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] } ] } @@ -194,9 +197,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", row=14), - Bitmap(frame="stargazer", row=19), - Bitmap(frame="language", row=1) + Bitmap(field="stargazer", row=14), + Bitmap(field="stargazer", row=19), + Bitmap(field="language", row=1) )' ``` ``` response @@ -204,7 +207,7 @@ curl localhost:10101/index/repository/query \ "results":[ { "attrs":{}, - "bits":[2,362,416,461] + "columns":[2,362,416,461] } ] } @@ -214,7 +217,7 @@ Set user 99999 as a stargazer for repository 77777: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", col=77777, row=99999)' + -d 'SetBit(field="stargazer", col=77777, row=99999)' ``` ``` response {"results":[true]} From 80fadb5d2169f8a3e51ab3f12f38d2af6607f367 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 19 Jun 2018 16:59:13 +0300 Subject: [PATCH 2/8] Updated client libraries --- docs/client-libraries.md | 93 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/docs/client-libraries.md b/docs/client-libraries.md index f445139b0..a492cf9b5 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -46,16 +46,16 @@ func main() { panic(err) } - // We need to refer to indexes and frames before we can use them in a query. + // We need to refer to indexes and fields before we can use them in a query. repository, _ := schema.Index("repository") - stargazer, _ := repository.Frame("stargazer") - language, _ := repository.Frame("language") + stargazer, _ := repository.Field("stargazer") + language, _ := repository.Field("language") var response *pilosa.QueryResponse // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Bitmap(14)) - fmt.Println("User 14 starred: ", response.Result().Bitmap().Bits) + response, _ = client.Query(stargazer.Row(14)) + fmt.Println("User 14 starred: ", response.Result().Row().Columns) // What are the top 5 languages in the sample data? response, err = client.Query(language.TopN(5)) @@ -68,26 +68,26 @@ func main() { // Which repositories were starred by both user 14 and 19: response, _ = client.Query( repository.Intersect( - stargazer.Bitmap(14), - stargazer.Bitmap(19))) - fmt.Println("Both user 14 and 19 starred:", response.Result().Bitmap().Bits) + stargazer.Row(14), + stargazer.Row(19))) + fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns) // Which repositories were starred by user 14 or 19: response, _ = client.Query( repository.Union( - stargazer.Bitmap(14), - stargazer.Bitmap(19))) - fmt.Println("User 14 or 19 starred:", response.Result().Bitmap().Bits) + stargazer.Row(14), + stargazer.Row(19))) + fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns) // Which repositories were starred by user 14 or 19 and were written in language 1: response, _ = client.Query( repository.Intersect( repository.Union( - stargazer.Bitmap(14), - stargazer.Bitmap(19), + stargazer.Row(14), + stargazer.Row(19), ), - language.Bitmap(1))) - fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Bitmap().Bits) + language.Row(1))) + fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) // Set user 99999 as a stargazer for repository 77777? client.Query(stargazer.SetBit(99999, 77777)) @@ -112,6 +112,7 @@ We are going to use the index you have created in the [Getting Started](../getti Error handling has been omitted in the example below for brevity. ```python +from __future__ import print_function from pilosa import Index, Client, PilosaError, TimeQuantum # We will just use the default client which assumes the server is at http://localhost:10101 @@ -122,8 +123,8 @@ client = Client() # and the stargazer data should be imported. # See the Getting Started repository: https://github.com/pilosa/getting-started/ -# Let's create Index and Frame objects, which will contain the settings -# for the corresponding indexes and frames. +# Let's create Index and Field objects, which will contain the settings +# for the corresponding indexes and fields. try: schema = client.schema() except PilosaError as e: @@ -132,13 +133,13 @@ except PilosaError as e: # We will just terminate the program in this case. raise SystemExit(e) -# We need to refer to indexes and frames before we can use them in a query. +# We need to refer to indexes and fields before we can use them in a query. repository = schema.index("repository") -stargazer = repository.frame("stargazer") -language = repository.frame("language") +stargazer = repository.field("stargazer") +language = repository.field("language") # Which repositories did user 8 star: -repository_ids = client.query(stargazer.bitmap(14)).result.bitmap.bits +repository_ids = client.query(stargazer.row(14)).result.row.columns print("User 8 starred: ", repository_ids) # What are the top 5 languages in the sample data: @@ -147,29 +148,29 @@ print("Top 5 languages: ", [item.id for item in top_languages]) # Which repositories were starred by both user 14 and 19: query = repository.intersect( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ) -mutually_starred = client.query(query).result.bitmap.bits +mutually_starred = client.query(query).result.row.columns print("Both user 14 and 19 starred:", mutually_starred) # Which repositories were starred by user 14 or 19: query = repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ) -either_starred = client.query(query).result.bitmap.bits +either_starred = client.query(query).result.row.columns print("User 14 or 19 starred:", either_starred) # Which repositories were starred by user 14 or 19 and were written in language 1: query = repository.intersect( repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ), - language.bitmap(1) + language.row(1) ) -mutually_starred = client.query(query).result.bitmap.bits +mutually_starred = client.query(query).result.row.columns print("User 14 or 19 starred, written in language 1:", mutually_starred) # Set user 99999 as a stargazer for repository 77777 @@ -218,10 +219,10 @@ public class StarTrace { throw new RuntimeException(ex); } - // We need to refer to indexes and frames before we can use them in a query. + // We need to refer to indexes and fields before we can use them in a query. Index repository = schema.index("repository"); - Frame stargazer = repository.frame("stargazer"); - Frame language = repository.frame("language"); + Field stargazer = repository.field("stargazer"); + Field language = repository.field("language"); QueryResponse response; QueryResult result; @@ -229,8 +230,8 @@ public class StarTrace { List repositoryIDs; // Which repositories did user 14 star: - response = client.query(stargazer.bitmap(14)); - repositoryIDs = response.getResult().getBitmap().getBits(); + response = client.query(stargazer.row(14)); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 starred: " + repositoryIDs); // What are the top 5 languages in the sample data: @@ -245,32 +246,32 @@ public class StarTrace { // Which repositories were starred by both user 14 and 19: query = repository.intersect( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("Both user 14 and 19 starred: " + repositoryIDs); // Which repositories were starred by user 14 or 19: query = repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 or 19 starred: " + repositoryIDs); // Which repositories were starred by user 14 or 19 and were written in language 1: query = repository.intersect( repository.union( - stargazer.bitmap(14), - stargazer.bitmap(19) + stargazer.row(14), + stargazer.row(19) ), - language.bitmap(1) + language.row(1) ); response = client.query(query); - repositoryIDs = response.getResult().getBitmap().getBits(); + repositoryIDs = response.getResult().getRow().getColumns(); System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); // Set user 99999 as a stargazer for repository 77777: From 8f3189c1b3b38e551f9404abd8252d4280b6bf41 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 21 Jun 2018 21:09:06 -0600 Subject: [PATCH 3/8] Translation fixes, error checking. --- Gopkg.lock | 6 -- executor.go | 19 ++++++ executor_test.go | 147 +++++++++++++++++++++++++++++++++++++++-------- server.go | 10 ++-- statik/statik.go | 10 ---- 5 files changed, 147 insertions(+), 45 deletions(-) delete mode 100644 statik/statik.go diff --git a/Gopkg.lock b/Gopkg.lock index b3fc9be2b..d3a12ef8a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -197,12 +197,6 @@ revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" -[[projects]] - name = "github.com/rakyll/statik" - packages = ["fs"] - revision = "fd36b3595eb2ec8da4b8153b107f7ea08504899d" - version = "v0.1.1" - [[projects]] name = "github.com/satori/go.uuid" packages = ["."] diff --git a/executor.go b/executor.go index c4abe0bd0..e59802851 100644 --- a/executor.go +++ b/executor.go @@ -1593,6 +1593,9 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { // Translate column key. if idx.Keys() { + if c.Args["col"] != nil && !isString(c.Args["col"]) { + return errors.New("'col' value must be a string when index 'keys' option enabled") + } if value := callArgString(c, "col"); value != "" { ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) if err != nil { @@ -1600,12 +1603,19 @@ func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { } c.Args["col"] = ids[0] } + } else { + if isString(c.Args["col"]) { + return errors.New("string 'col' value not allowed unless index 'keys' option enabled") + } } // Translate row key, if field is specified & key exists. if fieldName := callArgString(c, "field"); fieldName != "" { field := idx.Field(fieldName) if field.Keys() { + if c.Args["row"] != nil && !isString(c.Args["row"]) { + return errors.New("'row' value must be a string when field 'keys' option enabled") + } if value := callArgString(c, "row"); value != "" { ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) if err != nil { @@ -1613,6 +1623,10 @@ func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { } c.Args["row"] = ids[0] } + } else { + if isString(c.Args["row"]) { + return errors.New("string 'row' value not allowed unless field 'keys' option enabled") + } } } @@ -1782,3 +1796,8 @@ func callArgString(call *pql.Call, key string) string { s, _ := value.(string) return s } + +func isString(v interface{}) bool { + _, ok := v.(string) + return ok +} diff --git a/executor_test.go b/executor_test.go index 164133ecb..d0816b35e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -265,35 +265,132 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_SetBit(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + t.Run("ID", func(t *testing.T) { + t.Run("OK", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() - // set a bit so the view gets created. - hldr.SetBit("i", "f", 1, 0) + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if n := hldr.Row("i", "f", 11).Count(); n != 0 { - t.Fatalf("unexpected bitmap count: %d", n) - } + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if n := hldr.Row("i", "f", 11).Count(); n != 0 { + t.Fatalf("unexpected bitmap count: %d", n) + } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { - t.Fatal(err) - } else { - if !res[0].(bool) { - t.Fatalf("expected column changed") - } - } + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { + t.Fatal(err) + } else { + if !res[0].(bool) { + t.Fatalf("expected column changed") + } + } - if n := hldr.Row("i", "f", 11).Count(); n != 1 { - t.Fatalf("unexpected bitmap count: %d", n) - } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { - t.Fatal(err) - } else { - if res[0].(bool) { - t.Fatalf("expected column unchanged") - } - } + if n := hldr.Row("i", "f", 11).Count(); n != 1 { + t.Fatalf("unexpected bitmap count: %d", n) + } + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { + t.Fatal(err) + } else { + if res[0].(bool) { + t.Fatalf("expected column unchanged") + } + } + }) + + t.Run("ErrInvalidColValueType", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col="foo")`), nil, nil); err == nil || err.Error() != `string 'col' value not allowed unless index 'keys' option enabled` { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidRowValueType", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row="bar", col=2)`), nil, nil); err == nil || err.Error() != `string 'row' value not allowed unless field 'keys' option enabled` { + t.Fatal(err) + } + }) + }) + + t.Run("Keys", func(t *testing.T) { + t.Run("OK", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if n := hldr.Row("i", "f", 11).Count(); n != 0 { + t.Fatalf("unexpected bitmap count: %d", n) + } + + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col="foo")`), nil, nil); err != nil { + t.Fatal(err) + } else { + if !res[0].(bool) { + t.Fatalf("expected column changed") + } + } + + if n := hldr.Row("i", "f", 11).Count(); n != 1 { + t.Fatalf("unexpected bitmap count: %d", n) + } + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col="foo")`), nil, nil); err != nil { + t.Fatal(err) + } else { + if res[0].(bool) { + t.Fatalf("expected column unchanged") + } + } + }) + + t.Run("ErrInvalidColValueType", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col=2)`), nil, nil); err == nil || err.Error() != `'col' value must be a string when index 'keys' option enabled` { + t.Fatal(err) + } + }) + + t.Run("ErrInvalidRowValueType", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col=2)`), nil, nil); err == nil || err.Error() != `'row' value must be a string when field 'keys' option enabled` { + t.Fatal(err) + } + }) + }) } // Ensure a SetValue() query can be executed. diff --git a/server.go b/server.go index b6c4e7996..93aaecc9e 100644 --- a/server.go +++ b/server.go @@ -258,11 +258,8 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Initialize translation database. s.translateFile = NewTranslateFile() - s.translateFile.Path = filepath.Join(path, "keys") + s.translateFile.Path = filepath.Join(path, ".keys") s.translateFile.PrimaryTranslateStore = s.primaryTranslateStore - if err := s.translateFile.Open(); err != nil { - return nil, err - } // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { @@ -312,6 +309,11 @@ func (s *Server) Open() error { log.Println(errors.Wrap(err, "logging startup")) } + // Initialize id-key storage. + if err := s.translateFile.Open(); err != nil { + return err + } + // Cluster settings. s.Cluster.Broadcaster = s s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest diff --git a/statik/statik.go b/statik/statik.go deleted file mode 100644 index 54ef98b8f..000000000 --- a/statik/statik.go +++ /dev/null @@ -1,10 +0,0 @@ -package statik - -import ( - "github.com/rakyll/statik/fs" -) - -func init() { - data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00assets/chevron-down.png\x89PNG\x0d\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\xc8\x00\x00\x00\xc8\x08\x06\x00\x00\x00\xadX\xae\x9e\x00\x00\x0e\x0eIDATx\xda\xed\xddy\x90\x14\xd5\x1d\x07\xf0\xc7\xce\xd5=}\xce\xec\x1c;\xb33\xb3;3{\xc0\x9e\xec\x01\xcb.\xbb\xec1\xbb\xa8A\xa3$h\xc5#\x1e \xb95\xa5Dc*\x95C<\"\xa5\xd1T\x02\xc6Jb\x89\xe6\x1f\xe3\x91hb\"\xa8\x89g\x8c\xa6\"\xc6\x8ax!`\x8c ^ \x88r\xaf\xc9\xef\x07\xa31D\x84\x85\xdd\x9973\xdfOUWQ\xcbL\xf7{\xef\xf7\xeb\xee7\xef\xf5!\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\x8d\x93\x16\xc3\x8cu\xfby\xa1\x7f\xeb\xb48\xd0,p\x10\x8e\xfd\xf2\xc6\xc8\xe6RQ\xd1\\V\xbc\xd3\xa1X\x0b\xed\x9a\xcc\xb7yq(\xf6\x99n_\xba\x89\xfe\xcf\x83\x1c\x80\x03\xf0p\x8eP\xae\xcc\x7f?o\xca<\xd6\xe78\x978\xa7\x8a\xa5\x92\x15j\xb0a\x9eQ9m\xb9Z^\xb7\xdeW;\xf2o^\xd4@\xfdsf|\xc62oE\xdb1\xf4\x19\x1fr\x01\xf6\xe3\xe3\xdc\xa0\x1c\xb9\x96r\xe5\x85\x0f\xf2\xa6\xbc\xee\xa5\xbd\xb9D9\xc5\xb9U\xe8\x95\xacR}\xa9\xaf\x1bU=\xcf\xbc_\xc1\xfd\x17\xb3\xba\xef/\xde\xc0\xe4\x85\xf4\xd90r\x02\xb2\xc2\x9c\x13\x9c\x1b\x07\xca\x1b\xce)\xd5\x97\xbe\x90s\xac\x10+\xa8*v\xb2\xca\xa1\x96/;P\x05\xf7_\x9cZ\xf8\"::T\xa2\xcbU\xda]*\xce\x01\xce\x85C\xcd\x1b\xca\xb1\xa5\x8a/\x95\xa0\xef*\x85RIEx\xb4\xd9\xde\xd0\x94{\x0f\xb5\x92\xef/\xde\x8a\xe6[\x85jt\xd3:\xdc\xc8\x95\x92\xe3\xa6\xd8\xf7p\x0e\x8c9o(\xd7(\xe7F\ne'9\xdf\xa8\x9a\xf9\x8a]32:\xd6\x8a\xf2w\xcc\xea\xde\xf5\xb4\x8e3\x91/%\xe7,\x8e\xfd\xe1\xe6\x0d\xe7\x1c\xad\xe3<\x99+\x18sx\xfd\xd7\xd9\xe9\xc1\xcdv\xcd\xf0\x9e\xb1V\xf2\xbf\x95\x1d\xdeC\xeb\xd8\xe4\xd4\x02W\xd2:#\xc8\x9b\xa2\x17\xa1X_\xc51\x1f\x87\xbc\xd9\xcc9\xc8\xb9([%\x8f\xd3\xc2\xcd\xbf\xb1\x92\xb36\x1dn\x05\xf7_\xacd\xff\x9bz\xb8\xf5\x97\xb4\xee\xa3\x91CE\xebh\x8e1\xc7z\xfc\xf2f\xd6&\xceEZ\xf7\xb12T\xb0\x8c\x96\x0b\x8d\xea\x9e\xa7\xecd\xff\xd6\xf1\xaa\xe4\x07G\x85T\xff\xdbf\xf5\xcc'\xb9\xdb\x96\xdd\x16\x14\x07\x8e\xe5\"\x8e-\xc7x\xdc\xf3\x86r\x91s\x92\xb6qA\xbe\xf2f\x12-\x8dn\xbb\xfab\xbd\xb2}-\x9d\xde\xde\x1b\xefJ~\xe8\xd4\xf9\x1emc\x0do\x8b\xb6\xd9\x94\xdd6\x14&\x8e]s6o^\xccA\xde\xac\xcd\xe6Mc.\xf3\x86\xf7\xc8\x11\xad\xa2e\xb9\x11\x9f\xbea\xa2*\xf8\x7f\xe3\xde\xb4-\xda\xe6\x8d\xb4\xed\x13\x04.Q)D|y\xc8\\\x8a\xe1My\xc8\x9b\xe5\x9c\xb3\xb9:\x9b\xd4z\xc3M+\xa9\xdf\xf8N\xae*\xf9\xa1\xfe\xe5\xbbj\xa8\xe1a\xe12\xe6S9\xca\x91s\x05\xa3\x9ccF\xb1{\x84c\x98\xfb\xbc\xe9\x7f\xc7[\xd1\xb4\x82\xca\x91\xceEeO\xceu\x05\xf7_\x94@\xed:\x8f]\xcdCz\xad\xa2\x08/\\+\xb2\xb3F+\xc7\x8ac\x96\xef\xbc\xa1\xb2\x9c\x94\x8bJ\x9f\x91\xef\x8a\xf2\xa2G\xdb7\xba\xcd\x04\x9f:3\x02\xb3\xef2\xe2\x98\x0cs\x8c8V2\xe4\x0c\x95\xe7\xb4\\T\xfdR*[\x1dr\xfb\x88\xd5s[r\x9b\xca\x12_=\xd6\xf9\x00\xe7\x1c\xe7^\xc1\x1ca<\xbe\xf41z\xb4\xfdf:\xfdJ1\x14l\xa7\x07\xff\xa9'\xba~.\x9cJ\xaf\xc0\xec\xfb\xe1\xf5\x10\xa8\xed\xb8\x0d\xb9-%\xe9Fo\xe7\x1c\xe3\\\x93\xb1Ku(\xda\x8dD\xcfO\xadT\xff\x1aY\x8e6\xdeH\xcb\x1djp\xf2\xa7\x05n\xc6\x1a\x8b\x08\xb7\x19\xb7\x9d4\xbd\x82T\xff\x0b\x9c[\x9cc\x85\xde\xb8 \xad\xb2c\x91Y\xdd\xfb'Y\x1a\xd7\x8cM_\xa5\x84\x1b.\xe2\xee\x02r\xff\xa0&s[q\x9bI\x13?\xca%\xce)\xce\xadbid\x8f\xd3\xe3\x9fm\xa5\x06\x1e\xb3k2[\xe48=\x0fn\xf1X\x89\x1f\xbb\xb50\xff\xa8\xf3b?\xf8?^n\x1bj\xa3\xeb\xb8\xad\xe4\x18t\xc9l\xe1\x1c\xe2\\\x12Ez\xfd]\xbd\x91\xec\xbb\xc5J\x0fm\x90\xe5h\xa4\xf8k\xeeq\x9a\x91\xa3\xa8l~\xec\x13\x1f\xf0s\x9b(\xfe\xf4\xbd\x12\xcdmm\xe0\xdc)\x85\x81\x16\x9f\x1an9\x97\x8e\x06odg\xbf\xf3>\xebj\xc4\xa7\xaf+s\x1b\x17e\x87\x82K\xf9\x8eE\xae\xbb\x8f\xdb\x82\xdbD\x92\xab#\xf8\x92\x917\xd4p\xeb9\xa56T?\x85*\xbe\xd5JgFe\xd8I\xf6^\x0e=\xc9\xc9\x0f\x890Kx\x071\xa9\x0d~%\xc9\x1c\xd6{Vjh\x94s\x84\x7f\x07\x95b0\x1c\x1e#Y\xef\xb1\x93?SC\x0d\xb2\xfc.\xd9A\xdd\x8a\x87\xa9l\xc7\x8b\xd2\xba*\x98\xebz<\xd5\xfd\x11n\x03\x19b\xc19\xe11\xa2?\xe1\x1c\x11%|\x85\xb6C\xa8\xfe\xb8C\xf1\x9d\xea\xd4C\xaf\xc8ry\xb4\x11\xebz\x8e\xca\xb6\x84\x16\xb5\x04b\xc0u\\\xc2u\x96\xe5\xf6\x05\xce\x05\xca\x89S(7b\x02\xb7/\xec\xed\xf7Z\xc2\xe9\x99\xed\xd2\xc3\xbf7\xabfn\xcb\xff\xe9}h\xb7\x91\x98\xb1\x81\x82t\x13\x95\x8dG\xb9\x8a\xf1\xf2y\xaeS3\xd7\x91\xeb\xcau\xce\xfb\xf0-\xc5\x9er\xe0>\xca\x85\x91\xbd9\x81'\xd8\xfc\x0f~\x1eo\x1fu\xb9\xae\xf2\x86\x9b\xa4\x18\xe52b\xd3\xb6\xa8\x81\xfa;\xa9\\\xa7\x17\xd9P0\xd7\xe5t\xae\x1b\xd7Q\x8a \\\x8a9\xc7\x9es@\xe0V\x85\x8f=\xaa\xd5\xba\x8c\xd8\x02\xc5\x9fz^\x86\x1f\xeftT\xe3;\xd2VQ\xb9.+\x92\xa1`\xae\xc3e\\'\xae\x9b\x0c?\xc6)\xd6\xcfr\xcc9\xf6\x027\xbb\x1d\x12>\x82,\xd0\"\xad+(\x88\x9b\xa4\xf8\xd1X\x9e\xde\xa8\xf8R\x97;\xfd\xe9\x1eQ\x98\x93T\x8a\x12\xe9\xea\xe5:p]\xa4\x98\x11\xa7\xd8r\x8c\xa9l\xf3q\xd68\xbc\xb3IF\x0dMY\xaaWv\xbe,\xc5U\xa3\x95\x1d\xdb\xf4X\xe7\xedf\xa2\xfb\xf4l\x1f\xb9PX\xf4;\xe3\x0c39\xeb\x0e\xae\x83\x1cm\xd9\xf92\xc7\x96\xca6\x84\xb3\xc6\x91\xa9t*\xfe\xefP\x80\x9f\xb1\xd2C;$\x18\xe5\x1a\xb5\xd3C/\xe9\xd1\xf6/ 5\x1a\x97|\x94\xc5\xc1e\xe4\xb2r\x99%\x99\x11\xdf\xc1\xb1\xa4\x98~K\xe0\x8e\xcfq\x1d\xe9\x9aO\x0d\xfb\x90\x95\xec\x7fK\x9a\xfb\x10\xa2m\x17k\xc1\x86\xa9\x92\x0e\x07\xab\\6=\xd2\xb6X\x9a\xcbE(v\x1cC*\xdbY\x18\xa1\x9a\x18\xd3\x94\xf2\xba\xe5\xd4\xd0\x1b\xe5\xb9\xb2t\xe6\n\x8f?9[\xb2Q.\xaf\xc7_}\x14\x95m\xa5<;\xc7\xc0\x9b\x14\xbb\x1b8\x86H\xe3\x89\xee6\x08\xf1M_:\xf3\x8e];\xbc[\x8e\xcb\"Fv\n\xb7>O\x88j%\xcfG\xc6I{\xcb\xe0\xd6O\xe42I\xd16\x1c#\x8a\x15\x95\xed\x1b\x02\x93~9\xdcI\x1c\x8e\xe3\x8d\xf8\xf4?\xcbr\xdd\x10-\xbb\xd4P\xe3\xd5\"\x87O\xef\xfb\x08I5\xd8x\x0d\x97E\x96\xeb\xdb(F\x8fS\xac\x8e\xc3\xce\x91\x87aK\xbe_A-\xaf[&K7\xc2\xae\xc9\xbcmT\xf5\xf2\xb5\\#yh\x8f\xd9\xbcm.\x83,\xedA\xb1Y\xea\xd6\xc3\xfcN\x17\x05\xe9\x9a\xaf\xf9\x125V\xe9\xb1\x13_\xb5\xd2\x83\xafK\xb2\x93l\xb7R\x03k\xca\xbc\xe5_\x16\xb9\xb92\xd8\xe4m\xf16y\xdb\x92\\\xf4\xf9:\xc5\xe4\\\xe1\x8d\xf3(\x15\xee\xff\x97@P 5\x9f\xaa'\xbaWH\xf3\xe3\xbd\xaa\xf7E-\xdcr\x0d\x95\xadc\x02\xeb\xdd\xc9\xdb\xe0m\xc9Ro\xa3\xaa\xfbn\x8e\x05\xc7\x04i)\x17C\x89\xb4\xf6j\x91\xf6k$\x1a\xd6|\xd5H\xcc\xf8-\x95m\xde\x04\xd4w\x9e\x11\x9f\xf1;\xde\x864O4\x8c\xb4_\xcd1\xe0X \x1d\xa5\x9d/\xf1\xa4\x8dD\xcf\x12:\xcd\xaf\x91\xa4\xcb\xb5K\x8b\xb4>\xe62bgS\xf9\xe2\xe3P\xc78\xadk!\xaf\x93\x1f\xce,I\x97\x8a\x9f0\xb2\x84\xda>\x85\xf9\x8d\x02\xe9r\xe9\x89\x19\xe7\xd0\xd1u\xa5<\x93\x8a\xed\xcf{\x83\x0d|\xc1c\xf7a&\x11\x7f\xa7\x9b\xd6q9\xad\xeb\x05\x89\xce\x92+\xf5\xaa\xee\xaf\x08\x89\x9e\x83\x0b\x878\xca\xa5T4\x0eZ\xa9\xfe\xdb\xcdD\xcf\xdbv:3*\xc1\xe3j6\xa9\xe5\xb5\xb7 \xa7\x87\xdf\xfd>\x96\x89E/\x7f\x87\xbf\xcb\xeb\xc8\xffC\xf82\xa3Fl\xda[Vj\xe06\xa5\xa2i\x00\xa3T\x85-\xa2\xf8jn\xd2+;\x9e\xa5\xe4\x92b\xf2\xccm'V\xb9\xed\xd8\xdcl\x97k\xd2A\xce\x1aq\xfe,}\xe7II\x9eI\xb5\x93\xdb\xd2\xa5\x85\xaf\xa7\xb2\x85\x91^\xc5\x81\xdf\x11\xb1P\xf1\xa7\x1f5\xaaz\xa4\x18\n\xf5V4\xff\xc3\xa9\x07/\xa1rM\x11\x1f=\x89\xc6\x7f\x9bB\x9f\xb9\x94?+\xc7\x08U\xcf6j\xc3?R\xb9\x16d\xdb\x14\x8a\x08_R]/\x1c\xca\x83vr`\xab\x0c].\xea\xfe\xed\x98\xe4R\xff N\x97Z\xd1z\x85\x11\xefZ]*;\x07\xd5\xf5i5\xd2\xfa=\x81\xdba\xe1\x10\xc5\x14\x7f\xf2l\xbd\xb2\x83\xdf\xe2\xba\xa7Xw\x0c\xae\x9b\x1e\xebxH\xf1\xa7\xf8\xb9T1\x84\x1d\xc6j\x8eY5\xf3\x1e+5\xf0j\xd1u\xa9\xa8N\xfcP\x07\xaa\xe31\x083\x1c\x89\x16o\xb4\xe3\x87\xd4G_[<;\xc7\xac\x17\xb5\xca\x0e\x9e\xf8kFxa<\x98.#r\xb6\x9d\x1a\\O\xdd\x92\x1d\x05\xdc\xa5\xda\xc1up\x19Q\xeeR\x99\x08+\x8c\xb7\x99fr\xd6\xe3\xbe\x9a\xcc\xb6\x82\xdbA\xa8\xcc\\v\xb1\xef~\x14\x80 \x93\xd0\"m\xdf/\xb8\xc9\xbfH\x1b\xbfZ \x8e\xf0\xc1D+\x13\x81z\xc3\xe1\x0d|\xb2Pv\x0e*\xeb\x1c.\xb3\xc0C\xa2!\x87\xbcZ\xb0\xe1h\xfa\xf1~\x9f\xbc\x93\x7f\xfd\xf7j\xa1&\xd9\x1e\x8b\n%D\xf1VL\x9b\xa6\xc7g\\\xa2E;\xb6\xcb\xb2cx\xc3M[\xf5x\xf7bodz\xa7(\xccw\x99@\x11q(\xbe\x86\x84\xc7W\xfb\x05\xb7\x9dXMg\x94]y\xbc\\d\x97\xdb\x8e\xff\x8d\x9fz\xa2\xfa\x9bd\x7f5\x03\x94\x18\xbe\x99\xe8\x145P\xf3\x0b=\xda\x96\xf3\xf7\xfe\xe9\x95m\x9bi\xdb7S\x19>C\x8b\x86p\x80\x8c\xf8E\xa4\xddn+~\xb9\x1a\x9c\x92\xb3\xd9w\xda\xd6\x06\xda\xe6\xa5\xb4\xed.\x81W\x99A\x01\xf0 Q\xb6X\xaf\xec|\xd2\xac\xee\x9b\xb09\x13\xab\xba\xef]\xda\xc6*\xda\xd6wi\x9b6\x9a\x1d\n\xedl\xf2Yo\xb0\xf1\xd7f\xa2{\xdc\xdf\x8aE\xeb\xdc\xe4\x0d5\xf2+\xaaO\xcbn\x0b\xa0\xe0\xf0\xbcC\xa3K\x0b\xde`V\xf5\xbcj\xa73{\xc6\xe1 #{x]\xb4N~.U\x83\xc0\xdc\x06\x14\x01\xfe]p\x99Y5s\xed\x91\xbc\x88\x94\xbe\xbb\x9d\xd7A\xeb\xe2\xdba\xf1Z\x01(*|o\xf7\xb1j\xb0\x9e_r\xb3k\xec\x17\x1afv\xa9\xc1\xc9\x0f\xd2:\xe6\xa0)\xa1X9\x84\xd0\x83en\xed\xc6\xb1\xee \xf4\x9d\xeb\xf9\xbb\x02s\x1bP\x02g\x92r\xa7\xea_d$\xba\x0f\xfa\xe2\x1b\xfa\xcc\x1a\x97\xea?\x8f\xbf#\xf0\x84\x11(!\x01\x8f\xafj\x8e7\xd8p\xd7\x01/\x19 5\xdeI\x9f\xf9Dv\xe7\x00(9\x9aK\xabl\xd5BM\x8b= 0) {\n output_string += `
\n
\n Just getting started? Try this:
\n :create index test
\n :use test
\n :create frame foo
\n SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit\n `\n }\n }\n }\n }\n\n\n var markup =`\n
\n
\n
\n
\n
Input
\n       \n Source: ${res.indexname}\n
\n
\n ${res.input}\n
\n
\n
\n
\n
output
\n       \n ${res.querytime_ms} ms\n
\n
\n ${output_string}\n
\n
Expand
\n \n
\n
\n
\n
\n \n Export raw .csv\n
\n
\n
\n `\n node.innerHTML = markup;\n this.output.insertBefore(node, this.output.firstChild);\n\n // Expand when overflow\n var element = this.output.firstChild.getElementsByClassName(result_class)[0];\n var expand = this.output.firstChild.getElementsByClassName(\"expand\")[0];\n if (element.clientHeight < element.scrollHeight) {\n expand.style.display = 'block';\n } else {\n expand.style.display = 'none';\n }\n expand.onclick = function () {\n element.style.height = element.scrollHeight + \"px\";\n expand.style.display = 'none';\n return false;\n };\n }\n\n populate_index_dropdown() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/schema')\n var select = document.getElementById('index-dropdown')\n\n xhr.onload = function() {\n var schema = JSON.parse(xhr.responseText)\n for(var i=0; i 0) {\n select.value = 1;\n }\n }\n xhr.send(null)\n }\n\n}\n\nfunction populate_version() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/version')\n var node = document.getElementById('server-version')\n\n xhr.onload = function() {\n var version = JSON.parse(xhr.responseText)['version']\n var version_major_minor = /(v\\d+\\.\\d+)/.exec(version)[0]\n var doc_link = document.getElementById('nav-documentation')\n doc_link.onclick = function() {\n window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')\n }\n node.innerHTML = version\n }\n xhr.send(null)\n}\n\nfunction handle_nav_click(e) {\n // e.id = \"nav-xxx\"\n name = e.id.substring(4)\n set_active_pane_by_name(name)\n window.location.hash = name\n}\n\nfunction set_active_pane_by_name(name) {\n // toggle the nav buttons\n document.getElementsByClassName(\"nav-active\")[0].classList.remove(\"nav-active\")\n document.getElementById(\"nav-\" + name).classList.add(\"nav-active\")\n\n // toggle the main interface content divs\n document.getElementsByClassName(\"interface-active\")[0].classList.remove(\"interface-active\")\n document.getElementById('interface-' + name).classList.add(\"interface-active\")\n\n // hack hack\n switch(name) {\n case \"cluster\":\n update_cluster_status()\n break\n case \"documentation\":\n open_external_docs()\n break\n }\n}\n\n\nfunction update_cluster_status() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/status')\n status_node = document.getElementById('status')\n xhr.onload = function() {\n var status = JSON.parse(xhr.responseText)\n render_status(status)\n }\n xhr.send(null)\n}\n\nfunction render_status(status) {\n // render node table\n var nodes_div = document.getElementById(\"status-nodes\")\n while (nodes_div.firstChild) {\n nodes_div.removeChild(nodes_div.firstChild);\n }\n\n var nodes = status[\"status\"][\"Nodes\"]\n table = document.createElement(\"table\")\n tbody = document.createElement(\"tbody\")\n table.appendChild(tbody)\n var caption = document.createElement(\"caption\")\n caption.innerHTML = \"(\" + nodes.length + \")\"\n table.appendChild(caption)\n\n var header = document.createElement('tr')\n markup = `Host\n State`\n header.innerHTML = markup\n tbody.appendChild(header)\n for(var n=0; n${nodes[n][\"Host\"]}\n ${nodes[n][\"State\"]}`\n row.innerHTML = markup\n tbody.appendChild(row)\n }\n nodes_div.appendChild(table)\n\n // render index tables\n var indexes_div = document.getElementById(\"status-indexes\")\n while (indexes_div.firstChild) {\n indexes_div.removeChild(indexes_div.firstChild);\n }\n\n var indexes = nodes[0][\"Indexes\"] // TODO currently comes from only node 0\n for(var n=0; nName\n Cache Type\n Cache Size`\n header.innerHTML = markup\n tbody.appendChild(header)\n\n var frames = indexes[n][\"Frames\"]\n if(frames) {\n for(var m=0; m${frames[m][\"Name\"]}\n ${frames[m][\"Meta\"][\"CacheType\"]}\n ${frames[m][\"Meta\"][\"CacheSize\"]}`\n tbody.appendChild(row)\n }\n }\n indexes_div.appendChild(table)\n }\n\n // render slice tables\n // TODO enable when Slices element is present in status response\n /*\n var slices_div = document.getElementById(\"status-slices\")\n data = \"\"\n for(var n=0; n\"\n }\n }\n slices_div.innerHTML = data\n */\n\n}\n\nfunction open_external_docs() {\n window.open(\"https://www.pilosa.com/docs\");\n}\n\nfunction check_anchor_uri() {\n var pane_names = {\"console\": 0, \"cluster\": 0, \"documentation\": 0}\n var anchor = window.location.hash.substr(1);\n if(anchor in pane_names) {\n set_active_pane_by_name(anchor)\n }\n}\n\nDate.prototype.today = function () {\n return this.getFullYear() +\"/\"+ (((this.getMonth()+1) < 10)?\"0\":\"\") + (this.getMonth()+1) +\"/\"+ ((this.getDate() < 10)?\"0\":\"\") + this.getDate();\n}\n\nDate.prototype.timeNow = function () {\n return ((this.getHours() < 10)?\"0\":\"\") + this.getHours() +\":\"+ ((this.getMinutes() < 10)?\"0\":\"\") + this.getMinutes() +\":\"+ ((this.getSeconds() < 10)?\"0\":\"\") + this.getSeconds();\n}\n\npopulate_version()\n\n\nclass Autocompleter {\n constructor(input, output) {\n this.input = input\n this.output = output\n this.keyword_map = this.static_keywords\n this.init_dynamic_keywords()\n }\n\n get static_keywords() {\n return {\n // keyword: length of substring that comes after cursor\n \"SetBit()\": 1,\n \"ClearBit()\": 1,\n \"SetRowAttrs()\": 1,\n \"SetColumnAttrs()\": 1,\n \"Bitmap()\": 1,\n \"Union()\": 1,\n \"Intersect()\": 1,\n \"Difference()\": 1,\n \"Count()\": 1,\n \"Range()\": 1,\n \"TopN()\": 1,\n \"frame=\": 0,\n }\n }\n\n complete() {\n var completer = this\n // extract word fragment ending at cursor. a word fragment:\n // - starts with last nonalpha character before cursor (or beginning of string)\n // - ends at cursor\n var word_start = completer.input.selectionEnd-1\n while(word_start>0) {\n var c = completer.input.value.charCodeAt(word_start)\n if(!((c>64 && c<91) || (c>96 && c<123))) {\n word_start++\n break\n }\n word_start--\n }\n var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)\n\n // check for keyword match and insert if exactly one match\n var matches = []\n for(var keyword in this.keyword_map) {\n if(keyword.startsWith(input_word)){\n matches.push(keyword)\n }\n }\n if(matches.length > 1) {\n // completer.output.innerHTML = whatever\n }\n\n if(matches.length == 1) {\n // completer.output.innerHTML = \"\"\n var cursor_pos = completer.input.selectionEnd\n var completion = matches[0].substring(input_word.length)\n var before = completer.input.value.substring(0, cursor_pos)\n var after = completer.input.value.substring(cursor_pos)\n completer.input.value = before + completion + after\n var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]\n completer.input.setSelectionRange(new_pos, new_pos)\n }\n }\n\n init_dynamic_keywords() {\n // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list\n }\n\n add_keyword() {\n // call when index or frame created in webui\n }\n\n remove_keyword() {\n // call when index or frame deleted in webui\n // issue: if e.g. multiple indexes have same frame, removing one removes all.\n // solution: maintain count. requires more elaborate representation of keywords.\n }\n}\n\nvar input = document.getElementById('query')\nvar output = document.getElementById('outputs')\nvar button = document.getElementById('query-btn')\nvar autocomplete_output = document.getElementById('autocomplete-container')\n\nautocompleter = new Autocompleter(input, autocomplete_output)\nrepl = new REPL(input, output, button, autocompleter)\nrepl.populate_index_dropdown()\nrepl.bind_events()\n\ninput.focus()\n\ncheck_anchor_uri()\n\nfunction isJSON(str) {\n try {\n JSON.parse(str)\n } catch (e) {\n return false\n }\n return true\n}\n\nfunction parse_query(query, indexname) {\n var keys = query.replace(/\\s+/g, \" \").split(\" \");\n var command = keys[0];\n var command_type = keys[1];\n var command_name = keys[2];\n var option_str = keys.slice(3, keys.length)\n var options = parse_options(option_str);\n if (command !== \":use\") {\n if (!command_name){\n return {}\n }\n }\n\n var parsed_query = {};\n parsed_query[\"command\"] = command.substr(1, command.length);\n parsed_query[\"command_name\"] = command_name;\n switch (command) {\n case \":create\":\n parsed_query[\"request\"] = \"POST\";\n if(Object.keys(options).length === 0) {\n parsed_query[\"data\"] = \"\";\n } else {\n var opts = {\"options\":{}};\n for (var o in options) {\n opts.options[o] = options[o]\n }\n parsed_query[\"data\"] = JSON.stringify(opts);\n }\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n break\n }\n break;\n case \":delete\":\n parsed_query[\"request\"] = \"DELETE\";\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n }\n break;\n case \":use\":\n parsed_query[\"command_name\"] = keys[1];\n break;\n default:\n return {}\n }\n return parsed_query;\n}\n\nfunction parse_options(option_str) {\n var int_keys = [\"cacheSize\"];\n var bool_keys = [\"inverseEnabled\"];\n var options = {};\n for (var i = 0; i < option_str.length; i++) {\n var parts = option_str[i].split('=');\n if (int_keys.indexOf(parts[0]) !== -1 ){\n options[parts[0]] = Number(parts[1])\n } else if (bool_keys.indexOf(parts[0]) !== -1){\n options[parts[0]] = (parts[1] == \"true\")\n } else {\n options[parts[0]] = parts[1]\n }\n }\n return options;\n}PK\x07\x08\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-cluster-active.svgnav_cluster_1\nPK\x07\x08\xc1J\xead \x02\x00\x00 \x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-cluster.svgnav_cluster_1PK\x07\x08\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-console-active.svgnav_consolePK\x07\x08\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-console.svgnav_console\nPK\x07\x08\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00assets/nav-documentation-active.svgdocumentation\nPK\x07\x08\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00assets/nav-documentation.svgdocumentationPK\x07\x08\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00assets/nav_item1.svgnav_item1PK\x07\x08+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xa0~\xe6J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00assets/style.css*{\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n}\nbody{\n font-family: sans-serif;\n background-color: #fbfcfd;\n margin: 0;\n color: #102445;\n}\nh2{\n margin-bottom: 30px;\n}\n\nh5{\n text-transform: uppercase;\n letter-spacing: 2px;\n line-height: 1.21;\n margin: 0;\n}\na{\n line-height: 1.38;\n letter-spacing: 0.2px;\n text-decoration: none;\n color: #102445;\n}\n\n\na:hover{\n color: #1db598;\n}\n\ntextarea{\n width: 100%;\n margin-bottom: 10px;\n border-radius: 2px;\n background-color: #fbfcfd;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n outline: none;\n padding: 30px;\n}\n\n\nselect{\n /*-webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n background: url(\"img/chevron-down.png\") no-repeat calc(100% - 10px) !important;*/\n border-radius: 3px;\n background-color: #fbfcfd;\n width: 187px;\n height: 50px;\n border: solid 1.5px #e4eff4;\n font-size: 18px;\n font-weight: bold;\n line-height: 1.39;\n letter-spacing: 0.2px;\n color: #102445;\n padding: 10.5px;\n\n}\n\nbutton{\n width: 165px;\n height: 50px;\n border-radius: 3px;\n background-color: #1db598;\n outline: none;\n border: none;\n font-size: 16px;\n color: white;\n}\n\nem{\n font-style: normal;\n opacity: 0.5;\n font-size: 14px;\n font-weight: 500;\n letter-spacing: 0.2px;\n color: #102445;\n}\n\n.header{\n height: 92px;\n display: flex;\n align-items: center;\n justify-content: space-between;\n width: 90%;\n margin: auto;\n}\n\n.container{\n display: flex;\n height:100%;\n min-height: 100vh;\n}\n.nav{\n color: white;\n display: flex;\n flex-direction: column;\n width: 150px;\n background: #3c5f8d;\n}\n\n.nav-item{\n height:150px;\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n border-bottom: 3px solid #2a4871;\n cursor: pointer;\n}\n\n.nav-active{\n background: #f2f7f9;\n font-weight: bold;\n color: #1db598;\n}\n\n.nav-item > .nav-image {\n display: flex;\n}\n\n.nav-item > .nav-image-active {\n display: none;\n}\n\n.nav-active > .nav-image {\n display: none;\n}\n\n.nav-active > .nav-image-active {\n display: flex;\n}\n\n\n.interface{\n display: none;\n flex: 1;\n flex-direction: column;\n align-items: center;\n background: #f2f7f9;\n}\n\n.interface-active{\n display: flex;\n}\n\n.query{\n margin-bottom: 30px;\n}\n.query,\n.output-container,\n.status-container{\n width: 75%;\n}\n\n.output{\n margin-bottom: 30px;\n}\n\n.input-controls{\n display: flex;\n justify-content: flex-end;\n}\n\n.tabs{\n display: flex;\n background: #eaf2f6;\n}\n.active-tab{\n background: white;\n font-weight: bold;\n color: #1db598;\n\n}\n\n.tab{\n height:60px;\n width: 100px;\n border-top-right-radius: 5px;\n display: flex;\n align-items: center;\n justify-content: center;\n visibility: visible;\n cursor: pointer;\n\n}\n\n.pane{\n background: white;\n padding: 30px;\n display: none;\n}\n\n.active{\n display: block;\n}\n\n.result-io-header{\n display: flex;\n align-items: center;\n margin-bottom: 15px;\n}\n\n.result-input,\n.result-output,\n.result-error{\n height: 60px;\n border-radius: 2px;\n background-color: #fafafa;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n color: #102445;\n padding: 15px;\n margin-bottom: 15px;\n word-break: break-all;\n overflow-wrap: break-word;\n overflow:hidden;\n}\n\n\n.result-output{\n background-color: #edf9f7;\n border-left: solid 4px #1db598;\n}\n\n.result-error{\n background-color: #fbf1f0;\n border-left: solid 4px #fa3035;\n color: #fa3035;\n}\n\n.raw{\n height: 253px;\n display: flex;\n align-items: center;\n justify-content: center;\n}\n\n\n.result-table > table {\n border-left: solid 4px #1db598;\n}\n\ntable{\n border: solid 0.5px #e0e0e0;\n width: 100%;\n margin-bottom: 30px;\n /*color:#3c5f8d;*/\n}\ncaption{\n text-align:left;\n font-size: 16px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n text-align: left;\n}\nth{\n font-size: 14px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n color: #102445;\n text-transform: uppercase;\n text-align: left;\n padding: 21px 30px;\n background-color: white;\n}\ntr{\n border: solid 0.5px #e0e0e0;\n background-color: white;\n}\ntr:nth-child(even) {\n background-color: #f2f7f9;\n}\ntd{\n padding: 21px 30px;\n}\n\n.expand {\n text-align: center;\n}\n\n.query h2 {\n display: inline-block;\n}\n\n.query-tooltip {\n position: relative;\n display: inline;\n color: #000;\n margin-left: 5px;\n}\n\n.query-tooltip:hover {\n color: #000;\n}\n\n.query-tooltip-content {\n background-color: rgb(250, 250, 250);\n border: solid 1.5px #e4eff4;\n color: #102445;\n border-radius: 2px;\n padding: 15px;\n margin-bottom: 15px;\n\n position: absolute;\n left: 80px;\n top: -30px;\n z-index: 1;\n}\n\n.query-tooltip-container {\n position: relative;\n visibility: hidden;\n}\n\n.query-tooltip:hover+.query-tooltip-container{\n visibility: visible;\n}\n\n.code{\n font-family: monospace;\n}\n\nPK\x07\x08\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00cz\xbfJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00index.html\n\n\n \n \n \n \n Pilosa WebUI\n \n\n\n
\n \"\"\n
\n
\n
\n
\n
\n \"\"\n \"\"\n Console\n
\n
\n \"\"\n \"\"\n Cluster Admin\n
\n
\n \"\"\n \"\"\n Documentation\n
\n
\n
\n\n
\n

Query

\n ?\n
\n
\n
PQL
\n
\n SetBit(frame=foo, rowID=0, columnID=0)
\n ClearBit(frame=foo, rowID=0, columnID=0)
\n SetRowAttrs(frame=foo, rowID=0, color=\"blue\")
\n SetColumnAttrs(frame=foo, columnID=0, shape=\"circle\")
\n Bitmap(frame=foo, rowID=0)
\n Range(frame=foo, rowID=0, start=\"2010-01\", end=\"2017-03\")
\n Count(<BITMAP_CALL>)
\n TopN([BITMAP_CALL], frame=foo, n=20)
\n Union([BITMAP_CALL, ...])
\n Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
\n Difference(<BITMAP_CALL>, <BITMAP_CALL>)\n
\n
\n
Special commands
\n
\n :create index test [columnLabel=column]
\n :use test
\n :create frame foo [rowLabel=row]
\n :delete index test
\n :delete frame foo\n
\n
\n <tab>: autocomplete
\n <up>/<down>: history
\n
\n
\n \n
\n
\n \n    \n \n
\n
\n
\n\n
\n

Output

\n
\n \n
\n
\n\n
\n\n
\n
\n

Nodes

\n
\n
\n
\n
\n

Indexes

\n
\n
\n
\n
\n \n
\n\n
\n\n
\n docs!\n
\n\n
\n \n\n\nPK\x07\x08\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3JJ\x1c\xff\xa8G\x0e\x00\x00G\x0e\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00assets/chevron-down.pngPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x96\x84jK\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x8c\x0e\x00\x00assets/main.jsPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc1J\xead \x02\x00\x00 \x02\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x92W\x00\x00assets/nav-cluster-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xe6Y\x00\x00assets/nav-cluster.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81/\\\x00\x00assets/nav-console-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x1a^\x00\x00assets/nav-console.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xfc_\x00\x00assets/nav-documentation-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x819b\x00\x00assets/nav-documentation.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81kd\x00\x00assets/nav_item1.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xa0~\xe6J\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81Of\x00\x00assets/style.cssPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00cz\xbfJ\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xcay\x00\x00index.htmlPK\x05\x06\x00\x00\x00\x00\x0b\x00\x0b\x00\xf2\x02\x00\x00\xf1\x89\x00\x00\x00\x00" - fs.Register(data) -} From bb33411c5de17e82a97e252aeeefb74894e7c899 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 25 Jun 2018 12:40:46 -0500 Subject: [PATCH 4/8] remove debugging print statement --- executor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor.go b/executor.go index a1402b066..610696022 100644 --- a/executor.go +++ b/executor.go @@ -1614,7 +1614,6 @@ func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { if err != nil { return err } - fmt.Printf("translated %s to %d in field %s\n", value, ids[0], fieldName) c.Args[colKey] = ids[0] } } else { From ee37152cd5d020767045a18ac1e77b686a0b7a30 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 25 Jun 2018 17:05:36 -0500 Subject: [PATCH 5/8] consolidate gossipEventReceiver into gossip member set pilosa.Server now implements StatusHandler and EventReceiver and needs only start a gossip memberset. A gossip member set now takes a server as an argument explicitly and the maze of handlers and receivers and the starting sequence is somewhat simplified. Server now trivially implements EventHandler by passing the call along to its Cluster object which has the actual implementation. This means that less things will need to refer to cluster. --- broadcast_test.go | 27 ++++++++++++++------------- cluster.go | 5 ----- gossip/gossip.go | 32 +++++++++++++++++--------------- server.go | 28 +++++++++++++--------------- server/server.go | 5 ----- test/pilosa.go | 4 ---- 6 files changed, 44 insertions(+), 57 deletions(-) diff --git a/broadcast_test.go b/broadcast_test.go index 970a249cb..23bd962a7 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -56,6 +56,7 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { + t.Skip("broadcast receiver") path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) @@ -67,24 +68,24 @@ func TestBroadcast_BroadcastReceiver(t *testing.T) { if err != nil { t.Fatalf("setting up server: %v", err) } - s := com.Server + // s := com.Server - sbr := NewSimpleBroadcastReceiver() - sbh := NewSimpleBroadcastHandler() + // sbr := NewSimpleBroadcastReceiver() + // sbh := NewSimpleBroadcastHandler() - s.BroadcastReceiver = sbr - s.BroadcastReceiver.Start(sbh) + // s.BroadcastReceiver = sbr + // s.BroadcastReceiver.Start(sbh) - msg := &internal.DeleteIndexMessage{ - Index: "i", - } + // msg := &internal.DeleteIndexMessage{ + // Index: "i", + // } - s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) + // s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) - // Make sure the message received is what was sentd - if !reflect.DeepEqual(sbh.receivedMessage, msg) { - t.Fatalf("unexpected message: %s", sbh.receivedMessage) - } + // // Make sure the message received is what was sentd + // if !reflect.DeepEqual(sbh.receivedMessage, msg) { + // t.Fatalf("unexpected message: %s", sbh.receivedMessage) + // } } type SimpleBroadcastReceiver struct { diff --git a/cluster.go b/cluster.go index 957fe20fd..ae34bf03c 100644 --- a/cluster.go +++ b/cluster.go @@ -886,11 +886,6 @@ func (c *Cluster) open() error { return errors.Wrap(err, "adding local node") } - // Start the EventReceiver. - if err := c.EventReceiver.Start(c); err != nil { - return fmt.Errorf("starting EventReceiver: %v", err) - } - // Open MemberSet communication. if err := c.MemberSet.Open(c.Node); err != nil { return fmt.Errorf("opening MemberSet: %v", err) diff --git a/gossip/gossip.go b/gossip/gossip.go index 9154e9fc7..4749c9c58 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -33,7 +33,6 @@ import ( ) // Ensure GossipMemberSet implements interfaces. -var _ pilosa.BroadcastReceiver = &GossipMemberSet{} var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. @@ -45,19 +44,15 @@ type GossipMemberSet struct { broadcasts *memberlist.TransmitLimitedQueue - statusHandler pilosa.StatusHandler - config *gossipConfig + pserver *pilosa.Server + config *gossipConfig Logger pilosa.Logger logger *log.Logger transport *Transport -} -// Start implements the BroadcastReceiver interface and sets the BroadcastHandler. -func (g *GossipMemberSet) Start(h pilosa.BroadcastHandler) error { - g.handler = h - return nil + gossipEventReceiver *GossipEventReceiver } // GetBindAddr returns the gossip bind address based on config and auto bind port. @@ -69,13 +64,16 @@ func (g *GossipMemberSet) GetBindAddr() string { // Open implements the MemberSet interface to start network activity. func (g *GossipMemberSet) Open(n *pilosa.Node) error { + err := g.gossipEventReceiver.Start(g.pserver) + if err != nil { + return errors.Wrap(err, "starting event delegate") + } if g.handler == nil { return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") } g.node = n - err := error(nil) g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() @@ -166,7 +164,7 @@ func WithLogger(logger *log.Logger) GossipMemberSetOption { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { +func NewGossipMemberSet(name string, host string, cfg Config, s *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { g := &GossipMemberSet{ Logger: pilosa.NopLogger, } @@ -177,6 +175,10 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe return nil, errors.Wrap(err, "executing option") } } + ger := NewGossipEventReceiver(g.logger) + g.gossipEventReceiver = ger + + g.handler = s if g.transport == nil { port, err := strconv.Atoi(cfg.Port) @@ -232,7 +234,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe gossipSeeds: cfg.Seeds, } - g.statusHandler = sh + g.pserver = s return g, nil } @@ -270,7 +272,7 @@ func (g *GossipMemberSet) GetBroadcasts(overhead, limit int) [][]byte { // LocalState implementation of the memberlist.Delegate interface // sends this Node's state data. func (g *GossipMemberSet) LocalState(join bool) []byte { - pb, err := g.statusHandler.LocalStatus() + pb, err := g.pserver.LocalStatus() if err != nil { g.Logger.Printf("error getting local state, err=%s", err) return []byte{} @@ -294,7 +296,7 @@ func (g *GossipMemberSet) MergeRemoteState(buf []byte, join bool) { g.Logger.Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.statusHandler.HandleRemoteStatus(&pb) + err := g.pserver.HandleRemoteStatus(&pb) if err != nil { g.Logger.Printf("merge state error: %s", err) } @@ -309,11 +311,11 @@ type GossipEventReceiver struct { ch chan memberlist.NodeEvent eventHandler pilosa.EventHandler - Logger pilosa.Logger + Logger *log.Logger } // NewGossipEventReceiver returns a new instance of GossipEventReceiver. -func NewGossipEventReceiver(logger pilosa.Logger) *GossipEventReceiver { +func NewGossipEventReceiver(logger *log.Logger) *GossipEventReceiver { return &GossipEventReceiver{ ch: make(chan memberlist.NodeEvent, 1), Logger: logger, diff --git a/server.go b/server.go index 84a09a01d..dc21d2bc2 100644 --- a/server.go +++ b/server.go @@ -61,10 +61,9 @@ type Server struct { clusterDisabled bool // External - BroadcastReceiver BroadcastReceiver - systemInfo SystemInfo - gcNotifier GCNotifier - logger Logger + systemInfo SystemInfo + gcNotifier GCNotifier + logger Logger NodeID string URI URI @@ -207,12 +206,11 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ - closing: make(chan struct{}), - Cluster: NewCluster(), - holder: NewHolder(), - BroadcastReceiver: NopBroadcastReceiver, - diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), - systemInfo: NewNopSystemInfo(), + closing: make(chan struct{}), + Cluster: NewCluster(), + holder: NewHolder(), + diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), + systemInfo: NewNopSystemInfo(), gcNotifier: NopGCNotifier, @@ -297,11 +295,6 @@ func (s *Server) Open() error { // Initialize Holder. s.holder.Broadcaster = s - // Start the BroadcastReceiver. - if err := s.BroadcastReceiver.Start(s); err != nil { - return fmt.Errorf("starting BroadcastReceiver: %v", err) - } - // Open Cluster management. if err := s.Cluster.open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) @@ -711,6 +704,11 @@ func (s *Server) monitorRuntime() { } } +// ReceiveEvent implement EventHandler +func (s *Server) ReceiveEvent(e *NodeEvent) error { + return s.Cluster.ReceiveEvent(e) +} + // countOpenFiles on operating systems that support lsof. func countOpenFiles() (int, error) { switch runtime.GOOS { diff --git a/server/server.go b/server/server.go index ef9f46ca3..25076d95c 100644 --- a/server/server.go +++ b/server/server.go @@ -292,13 +292,10 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Node.IsCoordinator = true } - gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) - m.Server.Cluster.EventReceiver = gossipEventReceiver gossipMemberSet, err := gossip.NewGossipMemberSet( m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, - gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport), @@ -306,9 +303,7 @@ func (m *Command) SetupNetworking() error { if err != nil { return errors.Wrap(err, "getting memberset") } - gossipMemberSet.Logger = m.logger m.Server.Cluster.MemberSet = gossipMemberSet - m.Server.BroadcastReceiver = gossipMemberSet return nil } diff --git a/test/pilosa.go b/test/pilosa.go index 0f5cdd0d5..cba038e09 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -223,10 +223,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( return seed, err } - if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { - return seed, err - } - m.Server.Cluster.Static = false go func() { From 7190fe71e5222010f693cede7b01cba2af6338f3 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 26 Jun 2018 11:23:31 -0500 Subject: [PATCH 6/8] fix comment --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index dc21d2bc2..d6a4dde38 100644 --- a/server.go +++ b/server.go @@ -704,7 +704,7 @@ func (s *Server) monitorRuntime() { } } -// ReceiveEvent implement EventHandler +// ReceiveEvent implements the EventHandler interface. func (s *Server) ReceiveEvent(e *NodeEvent) error { return s.Cluster.ReceiveEvent(e) } From bcb6942c80173bf0ed1a3523451795f21182eb3f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 26 Jun 2018 14:36:05 -0500 Subject: [PATCH 7/8] continue simplifying memberset and pilosa setup since the gossip MemberSet has access to Server, it wasn't really necessary to pass it a Node object when calling Open on it from Cluster. The end goal is to have it be removed from Cluster entirely, and have it be Opened externally, and this is a step toward that. Exposing Node method on Server doesn't really expose any more than was already there as the same info can be gotten from LocalStatus with a bit of type casting. I figured adding the method was a little cleaner, and we could collapse all the functionality when the dust has settled. The Cluster.open method has been broken into two parts - one of which happens earlier (at NewServer time), and the other will eventually just be "waiting to make sure we've joined the cluster". Right now it's calling Memberset.Open, and then waiting to make sure the cluster has been joined. --- broadcast.go | 4 ++-- cluster.go | 19 +++++++++++++++---- cluster_internal_test.go | 7 ++++--- gossip/gossip.go | 7 ++----- server.go | 35 +++++++++++++++++++++++++++-------- server/server.go | 13 +++++++------ test/pilosa.go | 17 +++++++---------- 7 files changed, 64 insertions(+), 38 deletions(-) diff --git a/broadcast.go b/broadcast.go index 9b894fea2..d102fda22 100644 --- a/broadcast.go +++ b/broadcast.go @@ -27,7 +27,7 @@ import ( type MemberSet interface { // Open starts any network activity implemented by the MemberSet // Node is the local node, used for membership broadcasts. - Open(n *Node) error + Open() error } // StaticMemberSet represents a basic MemberSet for testing. @@ -43,7 +43,7 @@ func NewStaticMemberSet(nodes []*Node) *StaticMemberSet { } // Open implements the MemberSet interface to start network activity, but for a static MemberSet it does nothing. -func (s *StaticMemberSet) Open(n *Node) error { +func (s *StaticMemberSet) Open() error { return nil } diff --git a/cluster.go b/cluster.go index ae34bf03c..75c29ceb8 100644 --- a/cluster.go +++ b/cluster.go @@ -861,7 +861,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { return int(b) } -func (c *Cluster) open() error { +func (c *Cluster) setup() error { // Cluster always comes up in state STARTING until cluster membership is determined. c.state = ClusterStateStarting @@ -876,7 +876,7 @@ func (c *Cluster) open() error { if c.isCoordinator() { err := c.considerTopology() if err != nil { - return fmt.Errorf("considerTopology: %v", err) + return errors.Wrap(err, "considerTopology") } } @@ -885,10 +885,21 @@ func (c *Cluster) open() error { if err != nil { return errors.Wrap(err, "adding local node") } + return nil +} +func (c *Cluster) open() error { + err := c.setup() + if err != nil { + return errors.Wrap(err, "setting up cluster") + } + return c.waitForStarted() +} + +func (c *Cluster) waitForStarted() error { // Open MemberSet communication. - if err := c.MemberSet.Open(c.Node); err != nil { - return fmt.Errorf("opening MemberSet: %v", err) + if err := c.MemberSet.Open(); err != nil { + return errors.Wrap(err, "opening MemberSet") } // If not coordinator then wait for ClusterStatus from coordinator. diff --git a/cluster_internal_test.go b/cluster_internal_test.go index bdd51c047..86e027600 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -25,6 +25,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" ) // Ensure that fragCombos creates the correct fragment mapping. @@ -591,10 +592,10 @@ func TestCluster_ResizeStates(t *testing.T) { tc.WriteTopology(node.Path, top) // Open TestCluster. - expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" + expected := "coordinator node0 is not in topology: [some-other-host]" err := tc.Open() - if err == nil || err.Error() != expected { - t.Errorf("did not receive expected error: %s", expected) + if err == nil || errors.Cause(err).Error() != expected { + t.Errorf("did not receive expected error, got: %s", errors.Cause(err).Error()) } // Close TestCluster. diff --git a/gossip/gossip.go b/gossip/gossip.go index 4749c9c58..d67215e85 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -38,7 +38,6 @@ var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. type GossipMemberSet struct { mu sync.RWMutex - node *pilosa.Node memberlist *memberlist.Memberlist handler pilosa.BroadcastHandler @@ -63,7 +62,7 @@ func (g *GossipMemberSet) GetBindAddr() string { } // Open implements the MemberSet interface to start network activity. -func (g *GossipMemberSet) Open(n *pilosa.Node) error { +func (g *GossipMemberSet) Open() error { err := g.gossipEventReceiver.Start(g.pserver) if err != nil { return errors.Wrap(err, "starting event delegate") @@ -72,8 +71,6 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error { return fmt.Errorf("must call Start(pilosa.BroadcastHandler) before calling Open()") } - g.node = n - g.mu.Lock() g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() @@ -241,7 +238,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, s *pilosa.Server, // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { - buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) + buf, err := proto.Marshal(pilosa.EncodeNode(g.pserver.Node())) if err != nil { g.Logger.Printf("marshal message error: %s", err) return []byte{} diff --git a/server.go b/server.go index 74bedab5b..27c56be6e 100644 --- a/server.go +++ b/server.go @@ -71,6 +71,7 @@ type Server struct { metricInterval time.Duration diagnosticInterval time.Duration maxWritesPerRequest int + isCoordinator bool primaryTranslateStore TranslateStore @@ -203,6 +204,13 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +func OptServerIsCoordinator(is bool) ServerOption { + return func(s *Server) error { + s.isCoordinator = is + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ @@ -249,6 +257,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Get or create NodeID. s.NodeID = s.LoadNodeID() + if s.isCoordinator { + s.Cluster.Coordinator = s.NodeID + } + // Set Cluster Node. node := &Node{ ID: s.NodeID, @@ -271,6 +283,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Cluster = s.Cluster s.executor.TranslateStore = s.translateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest + s.Cluster.Broadcaster = s + s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest + s.holder.Broadcaster = s + + err = s.Cluster.setup() + if err != nil { + return nil, errors.Wrap(err, "setting up cluster") + } return s, nil } @@ -290,15 +310,8 @@ func (s *Server) Open() error { return err } - // Cluster settings. - s.Cluster.Broadcaster = s - s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest - - // Initialize Holder. - s.holder.Broadcaster = s - // Open Cluster management. - if err := s.Cluster.open(); err != nil { + if err := s.Cluster.waitForStarted(); err != nil { return fmt.Errorf("opening Cluster: %v", err) } @@ -529,6 +542,12 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error { return s.defaultClient.SendMessage(context.Background(), &to.URI, pb) } +// Node returns the pilosa.Node object. It is used by membership protocols to +// get this node's name(ID), location(URI), and coordinator status. +func (s *Server) Node() *Node { + return s.Cluster.Node +} + // Server implements StatusHandler. // LocalStatus is used to periodically sync information // between nodes. Under normal conditions, nodes should diff --git a/server/server.go b/server/server.go index a6bc1d995..8b958d314 100644 --- a/server/server.go +++ b/server/server.go @@ -247,6 +247,12 @@ func (m *Command) SetupServer() error { primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) } + // Set Coordinator. + coordinatorOpt := pilosa.OptServerIsCoordinator(false) + if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { + coordinatorOpt = pilosa.OptServerIsCoordinator(true) + } + serverOptions := []pilosa.ServerOption{ pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), @@ -265,6 +271,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + coordinatorOpt, } serverOptions = append(serverOptions, m.serverOptions...) @@ -313,12 +320,6 @@ func (m *Command) SetupNetworking() error { } } - // Set Coordinator. - if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { - m.Server.Cluster.Coordinator = m.Server.NodeID - m.Server.Cluster.Node.IsCoordinator = true - } - gossipMemberSet, err := gossip.NewGossipMemberSet( m.Server.NodeID, m.Server.URI.Host(), diff --git a/test/pilosa.go b/test/pilosa.go index 47b132508..2ae9ae9b0 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -196,13 +196,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( - SetupNetworking (does the gossip or static stuff) - calls NewTransport - Open server - calls OpenListener */ - - // SetupServer - err = m.SetupServer() - if err != nil { - return seed, err - } - // Open gossip transport to use in SetupServer. transport, err := gossip.NewTransport(host, bindPort, nil) if err != nil { @@ -215,17 +208,21 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( } else { m.Config.Gossip.Seeds = []string{transport.URI.String()} } - seed = transport.URI.String() + // SetupServer + m.Config.Cluster.Disabled = false + err = m.SetupServer() + if err != nil { + return seed, err + } + // SetupNetworking err = m.SetupNetworking() if err != nil { return seed, err } - m.Server.Cluster.Static = false - go func() { err := m.Handler.Serve() if err != nil { From 8d908a89cefc13eb0543d4661ce5eff5a35a937a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 27 Jun 2018 11:36:02 -0500 Subject: [PATCH 8/8] Fix TestTranslateStore_Reader tests --- api.go | 2 +- http/translator_test.go | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/api.go b/api.go index 98fd5d717..109307b7a 100644 --- a/api.go +++ b/api.go @@ -48,7 +48,7 @@ type APIOption func(*API) error func OptAPIServer(s *Server) APIOption { return func(a *API) error { a.server = s - a.TranslateStore = s.translateFile + a.TranslateStore = s.primaryTranslateStore a.Holder = s.holder a.Broadcaster = s a.Cluster = s.Cluster diff --git a/http/translator_test.go b/http/translator_test.go index 15aef62a4..d317944e4 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "io/ioutil" + gohttp "net/http" "testing" "time" @@ -15,8 +16,6 @@ import ( ) func TestTranslateStore_Reader(t *testing.T) { - t.Skip() // Until test.NewServer() works - // Ensure client can connect and stream the translate store data. t.Run("OK", func(t *testing.T) { t.Run("ServerDisconnect", func(t *testing.T) { @@ -46,15 +45,30 @@ func TestTranslateStore_Reader(t *testing.T) { // Setup handler on test server. var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { - if off != 100 { - t.Fatalf("unexpected off: %d", off) + // Check context to make sure this is the call we are looking for. + // (Something else calls ReaderFunc on server startup) + if ctx.Value(gohttp.ServerContextKey) != nil { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil } - return &mrc, nil + mrc2 := mock.ReadCloser{ + ReadFunc: func(p []byte) (int, error) { + return 0, io.EOF + }, + CloseFunc: func() error { + return nil + }, + } + return &mrc2, nil } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0] + defer main.Close() // Connect to server and stream all available data. @@ -128,6 +142,7 @@ func TestTranslateStore_Reader(t *testing.T) { opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0] + defer main.Close() _, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0) if err != pilosa.ErrNotImplemented {