From 060254e0d60b9a50aa757c75b94e1ecc43e6e433 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 15 Jun 2018 16:44:56 -0600 Subject: [PATCH] Key-to-ID Translation This commit adds id-to-key translation to make it easier for users to provide non-integer identifiers for rows & columns. --- Gopkg.lock | 14 +- api.go | 13 + ctl/server.go | 3 + executor.go | 112 +++++ executor_test.go | 126 ++++- field.go | 14 + holder.go | 8 +- http/handler.go | 52 ++ http/translator.go | 87 ++++ http/translator_test.go | 134 +++++ index.go | 22 +- inmem/translator.go | 215 ++++++++ inmem/translator_test.go | 132 +++++ internal/private.pb.go | 349 +++++++------ internal/private.proto | 2 + internal/public.pb.go | 35 +- mock/mock.go | 14 + mock/translator.go | 38 ++ pilosa.go | 3 + pql/ast.go | 33 ++ row.go | 10 + server.go | 31 +- server/config.go | 5 + server/server.go | 7 + statik/statik.go | 10 + test/executor.go | 2 + translate.go | 1006 ++++++++++++++++++++++++++++++++++++++ translate_test.go | 565 +++++++++++++++++++++ 28 files changed, 2829 insertions(+), 213 deletions(-) create mode 100644 http/translator.go create mode 100644 http/translator_test.go create mode 100644 inmem/translator.go create mode 100644 inmem/translator_test.go create mode 100644 mock/mock.go create mode 100644 mock/translator.go create mode 100644 statik/statik.go create mode 100644 translate.go create mode 100644 translate_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 0b4a8e9ea..b3fc9be2b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -70,6 +70,18 @@ packages = ["proto"] revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/cmpopts", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] name = "github.com/gorilla/context" packages = ["."] @@ -304,6 +316,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4" + inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" solver-name = "gps-cdcl" solver-version = 1 diff --git a/api.go b/api.go index ab4948588..7adff1d1b 100644 --- a/api.go +++ b/api.go @@ -44,6 +44,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster + TranslateStore TranslateStore Logger Logger } @@ -124,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return resp, errors.Wrap(err, "reading column attrs") } + + // Translate column attributes, if necessary. + if api.TranslateStore != nil { + for _, col := range resp.ColumnAttrSets { + v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + resp.ColumnAttrSets = columnAttrSets } return resp, nil diff --git a/ctl/server.go b/ctl/server.go index a4c272a46..ad1ac5df5 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + // Translation + flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.") + // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") diff --git a/executor.go b/executor.go index fd07bd82a..a99753023 100644 --- a/executor.go +++ b/executor.go @@ -50,6 +50,9 @@ type Executor struct { // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int + + // Stores key/id translation data. + TranslateStore TranslateStore } // ExecutorOption is a functional option type for pilosa.Executor @@ -83,6 +86,11 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return nil, ErrIndexRequired } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { return nil, ErrTooManyWrites @@ -93,6 +101,29 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic opt = &ExecOptions{} } + // Translate query keys to ids, if necessary. + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } + } + + results, err := e.execute(ctx, index, q, slices, opt) + if err != nil { + return nil, err + } + + // Translate response objects from ids to keys, if necessary. + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) @@ -1559,6 +1590,78 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } } +func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { + // Translate column key. + if idx.Keys() { + if value := callArgString(c, "col"); value != "" { + ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) + if err != nil { + return err + } + c.Args["col"] = ids[0] + } + } + + // Translate row key, if field is specified & key exists. + if fieldName := callArgString(c, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + if value := callArgString(c, "row"); value != "" { + ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) + if err != nil { + return err + } + c.Args["row"] = ids[0] + } + } + } + + // Translate child calls. + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return err + } + } + + return nil +} + +func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { + switch result := result.(type) { + case *Row: + if idx.Keys() { + other := &Row{Attrs: result.Attrs} + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + key, err := e.TranslateStore.TranslateColumnToString(index, col) + if err != nil { + return nil, err + } + other.Keys = append(other.Keys, key) + } + } + return other, nil + } + + case []Pair: + if fieldName := callArgString(call, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + other := make([]Pair, len(result)) + for i := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) + if err != nil { + return nil, err + } + other[i] = Pair{Key: key, Count: result[i].Count} + } + return other, nil + } + } + } + return result, nil +} + // errSliceUnavailable is a marker error if no nodes are available. var errSliceUnavailable = errors.New("slice unavailable") @@ -1670,3 +1773,12 @@ func (vc *ValCount) Larger(other ValCount) ValCount { Count: vc.Count, } } + +func callArgString(call *pql.Call, key string) string { + value, ok := call.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} diff --git a/executor_test.go b/executor_test.go index 3d0d9854e..164133ecb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -22,6 +22,8 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -101,6 +103,35 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set bits. + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ + `SetBit(field=f, row="bar", col="foo")`+"\n"+ + `SetBit(field=f, row="baz", col="foo")`+"\n"+ + `SetBit(field=f, row="bar", col="bat")`+"\n"+ + `SetBit(field=f, row="bbb", col="aaa")`+"\n", + ), nil, nil); err != nil { + t.Fatal(err) + } + + if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(results, []interface{}{ + &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { + t.Fatal(diff) + } + }) } // Ensure a difference query can be executed. @@ -383,36 +414,36 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + t.Run("ID", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(field=f, row=0, col=0) - SetBit(field=f, row=0, col=1) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(field=f, row=10, col=0) - SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(field=other, row=0, col=0) - `), nil, nil); err != nil { - t.Fatal(err) - } + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=1) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(field=f, row=10, col=0) + SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=other, row=0, col=0) + `), nil, nil); err != nil { + t.Fatal(err) + } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - t.Run("Standard", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ @@ -422,6 +453,46 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row="foo", col="a") + SetBit(field=f, row="foo", col="b") + SetBit(field=f, row="foo", col="c") + SetBit(field=f, row="foo", col="d") + SetBit(field=f, row="foo", col="e") + SetBit(field=f, row="bar", col="a") + SetBit(field=f, row="bar", col="b") + SetBit(field=f, row="baz", col="b") + SetBit(field=other, row="foo", col="a") + `), nil, nil); err != nil { + t.Fatal(err) + } + + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(result, []interface{}{ + []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + }); diff != "" { + t.Fatal(diff) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -1213,6 +1284,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { diff --git a/field.go b/field.go index 2c7598037..d9c88c924 100644 --- a/field.go +++ b/field.go @@ -282,6 +282,7 @@ func (f *Field) loadMeta() error { f.options.Min = pb.Min f.options.Max = pb.Max f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) + f.options.Keys = pb.Keys return nil } @@ -317,6 +318,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = 0 f.options.Max = 0 f.options.TimeQuantum = "" + f.options.Keys = opt.Keys case FieldTypeInt: f.options.Type = opt.Type f.options.CacheType = CacheTypeNone @@ -324,6 +326,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Min = opt.Min f.options.Max = opt.Max f.options.TimeQuantum = "" + f.options.Keys = opt.Keys // Create new bsiGroup. bsig := &bsiGroup{ @@ -345,6 +348,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.CacheSize = 0 f.options.Min = 0 f.options.Max = 0 + f.options.Keys = opt.Keys // Set the time quantum. if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() @@ -378,6 +382,13 @@ func (f *Field) Close() error { return nil } +// Keys returns true if the field uses string keys. +func (f *Field) Keys() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Keys +} + // bsiGroup returns a bsiGroup by name. func (f *Field) bsiGroup(name string) *bsiGroup { f.mu.RLock() @@ -1038,6 +1049,7 @@ type FieldOptions struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Keys bool `json:"keys,omitempty"` } // Validate ensures that FieldOption values are valid. @@ -1075,6 +1087,7 @@ func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { Min: o.Min, Max: o.Max, TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, } } @@ -1089,6 +1102,7 @@ func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { Min: options.Min, Max: options.Max, TimeQuantum: TimeQuantum(options.TimeQuantum), + Keys: options.Keys, } } diff --git a/holder.go b/holder.go index 7cb285d78..316ca1244 100644 --- a/holder.go +++ b/holder.go @@ -111,7 +111,8 @@ func (h *Holder) Open() error { } for _, fi := range fis { - if !fi.IsDir() { + // Skip files or hidden directories. + if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { continue } @@ -338,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { return nil, errors.Wrap(err, "creating") } + index.keys = opt.Keys + if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") + } else if err := index.saveMeta(); err != nil { + return nil, errors.Wrap(err, "meta") } // Update options. - h.indexes[index.Name()] = index return index, nil diff --git a/http/handler.go b/http/handler.go index 41a5faf55..bc09def3b 100644 --- a/http/handler.go +++ b/http/handler.go @@ -203,6 +203,8 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET") + router.Use(handler.queryArgValidator) return router } @@ -1184,6 +1186,56 @@ func (h *Handler) GetAPI() *pilosa.API { type defaultClusterMessageResponse struct{} +// TranslateStoreBufferSize is the buffer size used for streaming data. +const TranslateStoreBufferSize = 65536 + +func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) + + rc, err := h.API.TranslateStore.Reader(r.Context(), offset) + if err == pilosa.ErrNotImplemented { + http.Error(w, err.Error(), http.StatusNotImplemented) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer rc.Close() + + // Ensure reader is closed when the client disconnects. + go func() { <-r.Context().Done(); rc.Close() }() + + // Flush header so client can continue. + w.WriteHeader(http.StatusOK) + if w, ok := w.(http.Flusher); ok { + w.Flush() + } + + // Copy from reader to client until store or client disconnect. + buf := make([]byte, TranslateStoreBufferSize) + for { + // Read from store. + n, err := rc.Read(buf) + if err == io.EOF { + return + } else if err != nil { + h.Logger.Printf("http: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + h.Logger.Printf("http: translate store response write error: %s", err) + return + } else if w, ok := w.(http.Flusher); ok { + w.Flush() + } + } +} + type queryValidationSpec struct { required []string args map[string]struct{} diff --git a/http/translator.go b/http/translator.go new file mode 100644 index 000000000..3ca9b840d --- /dev/null +++ b/http/translator.go @@ -0,0 +1,87 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/pilosa/pilosa" +) + +// Ensure implementation implements inteface. +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +// TranslateStore represents an implementation of TranslateStore that +// communicates over HTTP. This is used with the TranslateHandler. +type TranslateStore struct { + URL string +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore(rawurl string) *TranslateStore { + return &TranslateStore{URL: rawurl} +} + +// TranslateColumnsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateColumnToString is not currently implemented. +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// TranslateRowsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateRowToString is not currently implemented. +func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// Reader returns a reader that can stream data from a remote store. +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + // Generate remote URL. + u, err := url.Parse(s.URL) + if err != nil { + return nil, err + } + u.Path = "/translate/data" + u.RawQuery = (url.Values{ + "offset": {strconv.FormatInt(off, 10)}, + }).Encode() + + // Connect a stream to the remote server. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + // Connect a stream to the remote server. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err) + } + + // Handle error codes or return body as stream. + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusNotImplemented: + resp.Body.Close() + return nil, pilosa.ErrNotImplemented + default: + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body)) + } +} diff --git a/http/translator_test.go b/http/translator_test.go new file mode 100644 index 000000000..3378ddc58 --- /dev/null +++ b/http/translator_test.go @@ -0,0 +1,134 @@ +package http_test + +import ( + "context" + "io" + "io/ioutil" + "net/http/httptest" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/mock" + "github.com/pilosa/pilosa/test" +) + +func TestTranslateStore_Reader(t *testing.T) { + // Ensure client can connect and stream the translate store data. + t.Run("OK", func(t *testing.T) { + t.Run("ServerDisconnect", func(t *testing.T) { + var mrc mock.ReadCloser + var readN int + mrc.ReadFunc = func(p []byte) (int, error) { + readN++ + switch readN { + case 1: + copy(p, []byte("foo")) + return 3, nil + case 2: + copy(p, []byte("barbaz")) + return 6, nil + case 3: + return 0, io.EOF + default: + t.Fatal("unexpected read") + return 0, nil + } + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + // Setup handler on test server. + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + // Connect to server and stream all available data. + store := http.NewTranslateStore(s.URL) + rc, err := store.Reader(context.Background(), 100) + if err != nil { + t.Fatal(err) + } else if data, err := ioutil.ReadAll(rc); err != nil { + t.Fatal(err) + } else if string(data) != `foobarbaz` { + t.Fatalf("unexpected data: %q", data) + } else if err := rc.Close(); err != nil { + t.Fatal(err) + } + + if !closeInvoked { + t.Fatal("expected server close") + } + }) + + // Ensure server closes store reader if client disconnects. + t.Run("ClientDisconnect", func(t *testing.T) { + // Setup mock so that Read() hangs. + done := make(chan struct{}) + + var mrc mock.ReadCloser + mrc.ReadFunc = func(p []byte) (int, error) { + <-done + return 0, io.EOF + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + defer close(done) + + // Connect to server and begin streaming. + ctx, cancel := context.WithCancel(context.Background()) + store := http.NewTranslateStore(s.URL) + if _, err := store.Reader(ctx, 0); err != nil { + t.Fatal(err) + } + + // Cancel the context and check if server is closed. + cancel() + time.Sleep(100 * time.Millisecond) + if !closeInvoked { + t.Fatal("expected server-side close") + } + }) + }) + + // Ensure client is notified if the server doesn't support streaming replication. + t.Run("ErrNotImplemented", func(t *testing.T) { + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return nil, pilosa.ErrNotImplemented + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + _, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0) + if err != pilosa.ErrNotImplemented { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/index.go b/index.go index 46ca48e47..68c48c829 100644 --- a/index.go +++ b/index.go @@ -33,6 +33,7 @@ type Index struct { mu sync.RWMutex path string name string + keys bool // use string keys // Fields by name. fields map[string]*Field @@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// Keys returns true if the index uses string keys. +func (i *Index) Keys() bool { return i.keys } + // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } @@ -164,18 +168,17 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. + i.keys = pb.Keys return nil } -// NOTE: Until we introduce new attributes to store in the index .meta file, -// we don't need to actually write the file. The code related to index.options -// and the index meta file are left in place for future use. -/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{}) + buf, err := proto.Marshal(&internal.IndexMeta{ + Keys: i.keys, + }) if err != nil { return errors.Wrap(err, "marshalling") } @@ -187,7 +190,6 @@ func (i *Index) saveMeta() error { return nil } -*/ // Close closes the index and its fields. func (i *Index) Close() error { @@ -407,11 +409,15 @@ func encodeIndex(d *Index) *internal.Index { } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct{} +type IndexOptions struct { + Keys bool `json:"keys"` +} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{} + return &internal.IndexMeta{ + Keys: i.Keys, + } } // hasTime returns true if a contains a non-nil time. diff --git a/inmem/translator.go b/inmem/translator.go new file mode 100644 index 000000000..b620b2d03 --- /dev/null +++ b/inmem/translator.go @@ -0,0 +1,215 @@ +package inmem + +import ( + "context" + "io" + "sync" + + "github.com/pilosa/pilosa" +) + +// Ensure type implements interface. +var _ pilosa.TranslateStore = &TranslateStore{} + +// TranslateStore is an in-memory storage engine for translating string-to-uint64 values. +type TranslateStore struct { + mu sync.RWMutex + + cols map[string]*translateIndex + rows map[frameKey]*translateIndex +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore() *TranslateStore { + return &TranslateStore{ + cols: make(map[string]*translateIndex), + rows: make(map[frameKey]*translateIndex), + } +} + +// Reader returns an error because it is not supported by the inmem store. +func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + return nil, pilosa.ErrReplicationNotSupported +} + +// TranslateColumnsToUint64 converts value to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.cols[index] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.rows[key] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +type frameKey struct { + index string + frame string +} + +type translateIndex struct { + seq uint64 + lookup map[string]uint64 + reverse map[uint64]string +} + +func newTranslateIndex() *translateIndex { + return &translateIndex{ + lookup: make(map[string]uint64), + reverse: make(map[uint64]string), + } +} diff --git a/inmem/translator_test.go b/inmem/translator_test.go new file mode 100644 index 000000000..d4d232566 --- /dev/null +++ b/inmem/translator_test.go @@ -0,0 +1,132 @@ +package inmem_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/pilosa/pilosa/inmem" +) + +func TestTranslateStore_TranslateColumn(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func TestTranslateStore_TranslateRow(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/1000) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index 1e2b34f38..c3dadb455 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: private.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -61,6 +60,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -68,6 +68,13 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type FieldOptions struct { Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -75,6 +82,7 @@ type FieldOptions struct { Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *FieldOptions) Reset() { *m = FieldOptions{} } @@ -124,6 +132,13 @@ func (m *FieldOptions) GetTimeQuantum() string { return "" } +func (m *FieldOptions) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } @@ -966,6 +981,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Keys { + dAtA[i] = 0x18 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1017,6 +1042,16 @@ func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) } + if m.Keys { + dAtA[i] = 0x58 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -2105,24 +2140,6 @@ func (m *RecalculateCaches) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Private(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2135,6 +2152,9 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l + if m.Keys { + n += 2 + } return n } @@ -2162,6 +2182,9 @@ func (m *FieldOptions) Size() (n int) { if m.Max != 0 { n += 1 + sovPrivate(uint64(m.Max)) } + if m.Keys { + n += 2 + } return n } @@ -2675,6 +2698,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2869,6 +2912,26 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3485,51 +3548,14 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - var keykey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - keykey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey if m.Standard == nil { m.Standard = make(map[string]uint64) } - if iNdEx < postIndex { - var valuekey uint64 + var mapkey string + var mapvalue uint64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -3539,31 +3565,69 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - valuekey |= (uint64(b) & 0x7F) << shift + wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - var mapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } } + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy } - m.Standard[mapkey] = mapvalue - } else { - var mapvalue uint64 - m.Standard[mapkey] = mapvalue } + m.Standard[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -6617,69 +6681,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1011 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1c, 0x35, - 0x18, 0x67, 0x1e, 0xbb, 0xd9, 0xfd, 0xd2, 0x0d, 0x89, 0x0b, 0x61, 0x8a, 0x50, 0x58, 0xac, 0x4a, - 0x0d, 0x3d, 0x44, 0xa5, 0xbd, 0xf0, 0xaa, 0x14, 0x25, 0x1b, 0x60, 0x10, 0x09, 0xe0, 0x49, 0x7a, - 0xeb, 0xc1, 0xdd, 0xb5, 0xda, 0x51, 0x66, 0xc7, 0xc3, 0x8c, 0x27, 0xc9, 0xf6, 0xc0, 0x15, 0x2e, - 0xdc, 0x11, 0x67, 0xfe, 0x18, 0x8e, 0xfc, 0x09, 0x28, 0xfc, 0x23, 0xc8, 0x9f, 0x3d, 0x8f, 0x64, - 0x37, 0x4d, 0x15, 0x7a, 0xf3, 0xf7, 0x7e, 0xfd, 0x3e, 0xdb, 0x30, 0xc8, 0xf2, 0xf8, 0x84, 0x2b, - 0xb1, 0x95, 0xe5, 0x52, 0x49, 0xd2, 0x8b, 0x53, 0x25, 0xf2, 0x94, 0x27, 0x74, 0x19, 0xfa, 0x61, - 0x3a, 0x11, 0x67, 0xfb, 0x42, 0x71, 0xfa, 0xa7, 0x03, 0xb7, 0xbe, 0x8a, 0x45, 0x32, 0xf9, 0x3e, - 0x53, 0xb1, 0x4c, 0x0b, 0xf2, 0x01, 0xf4, 0x77, 0xf9, 0xf8, 0x85, 0x38, 0x9c, 0x65, 0x22, 0xf0, - 0x86, 0xce, 0x66, 0x9f, 0x35, 0x8c, 0x5a, 0x1a, 0xc5, 0x2f, 0x45, 0xe0, 0x0f, 0x9d, 0xcd, 0x01, - 0x6b, 0x18, 0x64, 0x08, 0xcb, 0x87, 0xf1, 0x54, 0xfc, 0x58, 0xf2, 0x54, 0x95, 0xd3, 0xa0, 0x83, - 0xd6, 0x6d, 0x16, 0x21, 0xe0, 0xa3, 0xe3, 0x1e, 0x8a, 0xf0, 0x4c, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, - 0xa0, 0x3f, 0x74, 0x36, 0x3d, 0xa6, 0x8f, 0xc8, 0xe1, 0x67, 0x01, 0x58, 0x0e, 0x3f, 0xa3, 0x14, - 0x56, 0xc2, 0x69, 0x26, 0x73, 0xc5, 0x44, 0x91, 0xc9, 0xb4, 0x40, 0xab, 0xbd, 0x3c, 0x0f, 0x1c, - 0x74, 0xa4, 0x8f, 0xf4, 0x67, 0x58, 0xdd, 0x49, 0xe4, 0xf8, 0x78, 0xc4, 0x15, 0x67, 0xe2, 0xa7, - 0x52, 0x14, 0x8a, 0xbc, 0x03, 0x1d, 0xac, 0xd5, 0xea, 0x19, 0x42, 0x73, 0xb1, 0xe6, 0xc0, 0x35, - 0x5c, 0x24, 0x34, 0x17, 0xed, 0xb1, 0x6a, 0x9f, 0x19, 0x42, 0x73, 0xa3, 0x24, 0x1e, 0x9b, 0x6a, - 0x7d, 0x66, 0x08, 0x5d, 0xc7, 0x93, 0x58, 0x9c, 0xda, 0x12, 0xf1, 0x4c, 0x43, 0x58, 0x6b, 0xc5, - 0xb7, 0x69, 0xae, 0x43, 0x97, 0xc9, 0xd3, 0x70, 0x54, 0x04, 0xce, 0xd0, 0xdb, 0xf4, 0x99, 0xa5, - 0xb0, 0x91, 0x32, 0x29, 0xa7, 0xa9, 0x16, 0xb9, 0x28, 0x6a, 0x18, 0xf4, 0x0e, 0x74, 0xb0, 0xab, - 0xba, 0xca, 0xc6, 0x56, 0x1f, 0xe9, 0x2f, 0x0e, 0xf4, 0xf7, 0xf9, 0x19, 0xa6, 0x51, 0x90, 0xc7, - 0xd0, 0x8b, 0x14, 0x4f, 0x27, 0x3c, 0x9f, 0xa0, 0xd2, 0xf2, 0xc3, 0x8f, 0xb6, 0xaa, 0x41, 0x6f, - 0xd5, 0x6a, 0x5b, 0x95, 0xce, 0x5e, 0xaa, 0xf2, 0x19, 0xab, 0x4d, 0xde, 0xff, 0x02, 0x06, 0x17, - 0x44, 0x3a, 0xde, 0xb1, 0x98, 0x55, 0x5d, 0x3d, 0x16, 0x33, 0x5d, 0xff, 0x09, 0x4f, 0x4a, 0x81, - 0xbd, 0xf2, 0x99, 0x21, 0x3e, 0x77, 0x3f, 0x75, 0xe8, 0x36, 0x90, 0xdd, 0x5c, 0x70, 0x25, 0x30, - 0xc8, 0xbe, 0x28, 0x0a, 0xfe, 0x5c, 0x5c, 0xdd, 0x71, 0xd3, 0x45, 0xb7, 0xd5, 0x45, 0x7a, 0x1f, - 0xc8, 0x48, 0x24, 0x42, 0x09, 0x8b, 0xc7, 0x57, 0x78, 0xa0, 0x51, 0x15, 0xed, 0x7a, 0x5d, 0x72, - 0x0f, 0x7c, 0x0d, 0x6e, 0x0c, 0xb6, 0xfc, 0xf0, 0x76, 0xd3, 0x91, 0x1a, 0xf7, 0x0c, 0x15, 0x68, - 0x52, 0x39, 0x45, 0x04, 0x5c, 0x5b, 0xc2, 0x02, 0xd0, 0xdc, 0xb7, 0xa1, 0x3c, 0x0c, 0xb5, 0xde, - 0x84, 0x6a, 0x2f, 0x95, 0x8d, 0xb6, 0x5d, 0x95, 0x7b, 0xd3, 0x68, 0xf4, 0xa9, 0xe5, 0x6a, 0xfc, - 0x1d, 0xf0, 0xa9, 0xb0, 0x36, 0x78, 0xae, 0x53, 0x71, 0xaf, 0x4f, 0x45, 0xbb, 0xd7, 0x98, 0x2d, - 0x02, 0x6f, 0xe8, 0x69, 0xf7, 0x48, 0xd0, 0x47, 0xd0, 0x8d, 0xc6, 0x2f, 0xc4, 0x94, 0x93, 0x8f, - 0x61, 0x09, 0xf3, 0x10, 0x85, 0x85, 0xd5, 0xdb, 0x97, 0x9a, 0xc8, 0x2a, 0x39, 0x1d, 0xd9, 0xfc, - 0x17, 0xe6, 0x74, 0x0f, 0xba, 0x18, 0xbd, 0x08, 0xfc, 0xcb, 0x6e, 0x90, 0xcf, 0xac, 0x98, 0xee, - 0x81, 0x77, 0xc4, 0x42, 0xbd, 0x2e, 0x98, 0x41, 0xe5, 0xc5, 0x52, 0xda, 0xf7, 0x37, 0xb2, 0x50, - 0xb6, 0x1b, 0x78, 0xd6, 0xbc, 0x1f, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x67, 0xfa, 0x14, 0xfc, - 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x7d, 0xb8, 0xe1, 0x88, 0x7c, 0x88, 0xee, 0x6d, - 0x6b, 0x06, 0x4d, 0x12, 0x47, 0x2c, 0x64, 0x18, 0xf8, 0x2e, 0x0c, 0xc2, 0x62, 0x57, 0xca, 0x7c, - 0x12, 0xa7, 0x5c, 0xc9, 0x1c, 0xbd, 0xf6, 0xd8, 0x45, 0x26, 0xdd, 0x86, 0x55, 0xed, 0x3e, 0x52, - 0x5c, 0xd5, 0x80, 0x5f, 0x87, 0xae, 0xe6, 0xd5, 0xe1, 0x2c, 0x85, 0x90, 0xd7, 0x7a, 0xd5, 0x04, - 0x91, 0xa0, 0xdf, 0x19, 0x0f, 0x7b, 0x27, 0x22, 0x55, 0x2d, 0x04, 0x20, 0x8d, 0x0e, 0x06, 0xcc, - 0x10, 0x84, 0x9a, 0x52, 0x6c, 0xce, 0x2b, 0x4d, 0xce, 0x9a, 0xcb, 0x50, 0x46, 0x7f, 0x73, 0x00, - 0xaa, 0x84, 0xca, 0xa2, 0x36, 0x71, 0xae, 0x36, 0x21, 0x9f, 0xb4, 0xae, 0x8f, 0xf9, 0x05, 0xa9, - 0x45, 0xac, 0x75, 0xc9, 0x6c, 0x56, 0xb0, 0xb0, 0x28, 0x5f, 0x6d, 0xf4, 0x0d, 0xdf, 0x8e, 0x89, - 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0xdb, 0x8c, 0xf4, 0x35, 0x67, 0x18, 0x75, 0x7f, - 0x1a, 0xc6, 0xe2, 0x16, 0x91, 0xbb, 0xd0, 0xd1, 0x99, 0x1a, 0x6c, 0xce, 0x97, 0x61, 0x84, 0xf4, - 0x09, 0xf4, 0x76, 0xa2, 0xf0, 0xeb, 0x5c, 0x96, 0xd9, 0x42, 0xe4, 0x55, 0x2f, 0x8d, 0x3b, 0xff, - 0xd2, 0x78, 0x73, 0x2f, 0x8d, 0xdf, 0xbc, 0x34, 0x11, 0xac, 0x99, 0x2b, 0x41, 0xaf, 0xc4, 0x4d, - 0x6e, 0x84, 0xea, 0x69, 0xf0, 0x5a, 0x4f, 0x43, 0x04, 0x6b, 0x66, 0xf3, 0xdf, 0xa4, 0xd3, 0x3f, - 0x5c, 0x58, 0x63, 0xa2, 0x88, 0x5f, 0x8a, 0x30, 0x2d, 0x54, 0x5e, 0x8e, 0xf5, 0x82, 0x6b, 0xfb, - 0x6f, 0xe5, 0x33, 0xdb, 0x6d, 0x8f, 0x19, 0xe2, 0x75, 0xc0, 0x44, 0x1e, 0xc0, 0xf2, 0xe5, 0x05, - 0x98, 0x57, 0x6d, 0xab, 0x90, 0x07, 0xb0, 0x14, 0xc9, 0x32, 0xd7, 0x48, 0x32, 0xeb, 0xdd, 0xba, - 0x74, 0x4c, 0x66, 0x46, 0xcc, 0x2a, 0xb5, 0x16, 0x94, 0x3a, 0xaf, 0x86, 0x12, 0x79, 0x7c, 0x09, - 0x4a, 0x41, 0x17, 0x0d, 0xde, 0x6b, 0x0c, 0x2e, 0x88, 0xd9, 0x45, 0x6d, 0xfa, 0xab, 0x03, 0xb7, - 0xda, 0x29, 0xbc, 0xd6, 0x6e, 0xd4, 0x13, 0x71, 0x17, 0x4e, 0xc4, 0x5b, 0x34, 0x11, 0xbf, 0x99, - 0x48, 0xf3, 0xca, 0x75, 0xda, 0xaf, 0xdc, 0x31, 0xdc, 0x99, 0x1b, 0xd3, 0xae, 0x9c, 0x66, 0x1a, - 0x0f, 0xff, 0x63, 0x5c, 0xfa, 0xd6, 0xc8, 0x73, 0x3b, 0xa8, 0x3e, 0x33, 0x04, 0xfd, 0x0c, 0xde, - 0x8d, 0x84, 0x6a, 0x0d, 0xa9, 0x42, 0xdb, 0x10, 0xbc, 0x03, 0x71, 0x7a, 0x45, 0xf9, 0x5a, 0x44, - 0xbf, 0x84, 0xe0, 0x28, 0x9b, 0x70, 0x25, 0x6e, 0x64, 0xbd, 0x03, 0xbd, 0x43, 0x99, 0xc9, 0x44, - 0x3e, 0x9f, 0x5d, 0xb3, 0xf5, 0x01, 0x2c, 0x99, 0x2b, 0xd2, 0x7c, 0x7c, 0xfa, 0xac, 0x22, 0xe9, - 0x6d, 0x0d, 0xe8, 0x31, 0x4f, 0xc6, 0x65, 0xa2, 0xd3, 0xd0, 0x3f, 0xa0, 0x62, 0x67, 0xf5, 0xaf, - 0xf3, 0x0d, 0xe7, 0xef, 0xf3, 0x0d, 0xe7, 0x9f, 0xf3, 0x0d, 0xe7, 0xf7, 0x7f, 0x37, 0xde, 0x7a, - 0xd6, 0xc5, 0x1f, 0xed, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x97, 0xf0, 0x12, 0xfd, 0xe2, - 0x0a, 0x00, 0x00, + // 1028 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, + 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, + 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, + 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, + 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, + 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, + 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, + 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, + 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, + 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, + 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, + 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, + 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, + 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, + 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, + 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, + 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, + 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, + 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, + 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, + 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, + 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, + 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, + 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, + 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, + 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, + 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, + 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, + 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, + 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, + 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, + 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, + 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, + 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, + 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, + 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, + 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, + 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, + 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, + 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, + 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, + 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, + 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, + 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, + 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, + 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, + 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, + 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, + 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, + 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, + 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, + 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, + 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, + 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, + 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, + 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, + 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, + 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, + 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, + 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, + 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, + 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, + 0x0a, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 9cab31828..23bb4886c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package internal; message IndexMeta { + bool Keys = 3; } message FieldOptions { @@ -12,6 +13,7 @@ message FieldOptions { int64 Min = 9; int64 Max = 10; string TimeQuantum = 5; + bool Keys = 11; } message ImportResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 76266d2db..3cb6fa270 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: public.proto -// DO NOT EDIT! /* Package internal is a generated protocol buffer package. @@ -28,6 +27,8 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import encoding_binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -799,7 +800,8 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + i += 8 } return i, nil } @@ -1235,24 +1237,6 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Public(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Public(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2333,15 +2317,8 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex diff --git a/mock/mock.go b/mock/mock.go new file mode 100644 index 000000000..46469c2f9 --- /dev/null +++ b/mock/mock.go @@ -0,0 +1,14 @@ +package mock + +type ReadCloser struct { + ReadFunc func(p []byte) (int, error) + CloseFunc func() error +} + +func (rc *ReadCloser) Read(p []byte) (int, error) { + return rc.ReadFunc(p) +} + +func (rc *ReadCloser) Close() error { + return rc.CloseFunc() +} diff --git a/mock/translator.go b/mock/translator.go new file mode 100644 index 000000000..3f815b89f --- /dev/null +++ b/mock/translator.go @@ -0,0 +1,38 @@ +package mock + +import ( + "context" + "io" + + "github.com/pilosa/pilosa" +) + +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +type TranslateStore struct { + TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) + TranslateColumnToStringFunc func(index string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) +} + +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return s.TranslateColumnsToUint64Func(index, values) +} + +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return s.TranslateColumnToStringFunc(index, values) +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, frame, values) +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, frame, value) +} + +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + return s.ReaderFunc(ctx, off) +} diff --git a/pilosa.go b/pilosa.go index c18fb5c0c..bff167a7d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -61,6 +61,8 @@ var ( ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") + + ErrNotImplemented = errors.New("not implemented") ) // ApiMethodNotAllowedError wraps an error value indicating that a particular @@ -83,6 +85,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..2d3f59e58 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int { return n } +// HasKeys returns true if any call in the query uses keys and requires translation to ids. +func (q *Query) HasKeys() bool { + for _, call := range q.Calls { + if call.Args["col"] != nil { + if _, ok := call.Args["col"].(string); ok { + return true + } + } + if call.Args["row"] != nil { + if _, ok := call.Args["row"].(string); ok { + return true + } + } + } + return false +} + // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// StringArg is for reading the value at key from call.Args as a string. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. An error is returned if the value is not a string. +func (c *Call) StringArg(key string) (string, bool, error) { + val, ok := c.Args[key] + if !ok { + return "", false, nil + } + switch tval := val.(type) { + case string: + return tval, true, nil + default: + return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) + } +} + // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/row.go b/row.go index b7643a975..a59bd73e0 100644 --- a/row.go +++ b/row.go @@ -27,6 +27,9 @@ import ( type Row struct { segments []RowSegment + // String keys translated to/from segment columns. + Keys []string + // Attributes associated with the row. Attrs map[string]interface{} } @@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) { return s.ClearBit(i) } +// Segments returns a list of all segments in the row. +func (r *Row) Segments() []RowSegment { + return r.segments +} + // segment returns a segment for a given slice. // Returns nil if segment does not exist. func (r *Row) segment(slice uint64) *RowSegment { @@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` Columns []uint64 `json:"columns"` + Keys []string `json:"keys,omitempty"` } o.Columns = r.Columns() + o.Keys = r.Keys o.Attrs = r.Attrs if o.Attrs == nil { diff --git a/server.go b/server.go index cbc79a86c..4a453cef5 100644 --- a/server.go +++ b/server.go @@ -52,10 +52,11 @@ type Server struct { closing chan struct{} // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - executor *Executor + Holder *Holder + Cluster *Cluster + TranslateFile *TranslateFile + diagnostics *DiagnosticsCollector + executor *Executor // External handler Handler @@ -75,6 +76,8 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int + primaryTranslateStore TranslateStore + defaultClient InternalClient dataDir string } @@ -169,6 +172,13 @@ func OptServerInternalClient(c InternalClient) ServerOption { } } +func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { + return func(s *Server) error { + s.primaryTranslateStore = store + return nil + } +} + func OptServerStatsClient(sc StatsClient) ServerOption { return func(s *Server) error { s.Holder.Stats = sc @@ -241,6 +251,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder + // Initialize translation database. + s.TranslateFile = NewTranslateFile() + s.TranslateFile.Path = filepath.Join(path, "keys") + s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore + if err := s.TranslateFile.Open(); err != nil { + return nil, err + } + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -259,8 +277,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Holder = s.Holder s.executor.Node = node s.executor.Cluster = s.Cluster + s.executor.TranslateStore = s.TranslateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest s.handler.GetAPI().Executor = s.executor + s.handler.GetAPI().TranslateStore = s.TranslateFile return s, nil } @@ -354,6 +374,9 @@ func (s *Server) Close() error { if s.Holder != nil { s.Holder.Close() } + if s.TranslateFile != nil { + s.TranslateFile.Close() + } return nil } diff --git a/server/config.go b/server/config.go index 6c7db569a..1b74b177b 100644 --- a/server/config.go +++ b/server/config.go @@ -78,6 +78,11 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` + // Translation config supports translation store replication. + Translation struct { + PrimaryURL string `toml:"primary-url"` + } + AntiEntropy struct { Interval toml.Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/server/server.go b/server/server.go index d034a4aa8..0c4184d60 100644 --- a/server/server.go +++ b/server/server.go @@ -217,6 +217,12 @@ func (m *Command) SetupServer() error { c := http.GetHTTPClient(TLSConfig) + // Setup connection to primary store if this is a replica. + var primaryTranslateStore pilosa.TranslateStore + if m.Config.Translation.PrimaryURL != "" { + primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) + } + m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), @@ -235,6 +241,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), + pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), ) return errors.Wrap(err, "new server") diff --git a/statik/statik.go b/statik/statik.go new file mode 100644 index 000000000..54ef98b8f --- /dev/null +++ b/statik/statik.go @@ -0,0 +1,10 @@ +package statik + +import ( + "github.com/rakyll/statik/fs" +) + +func init() { + data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00assets/chevron-down.png\x89PNG\x0d\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\xc8\x00\x00\x00\xc8\x08\x06\x00\x00\x00\xadX\xae\x9e\x00\x00\x0e\x0eIDATx\xda\xed\xddy\x90\x14\xd5\x1d\x07\xf0\xc7\xce\xd5=}\xce\xec\x1c;\xb33\xb3;3{\xc0\x9e\xec\x01\xcb.\xbb\xec1\xbb\xa8A\xa3$h\xc5#\x1e \xb95\xa5Dc*\x95C<\"\xa5\xd1T\x02\xc6Jb\x89\xe6\x1f\xe3\x91hb\"\xa8\x89g\x8c\xa6\"\xc6\x8ax!`\x8c ^ \x88r\xaf\xc9\xef\x07\xa31D\x84\x85\xdd\x9973\xdfOUWQ\xcbL\xf7{\xef\xf7\xeb\xee7\xef\xf5!\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\x8d\x93\x16\xc3\x8cu\xfby\xa1\x7f\xeb\xb48\xd0,p\x10\x8e\xfd\xf2\xc6\xc8\xe6RQ\xd1\\V\xbc\xd3\xa1X\x0b\xed\x9a\xcc\xb7yq(\xf6\x99n_\xba\x89\xfe\xcf\x83\x1c\x80\x03\xf0p\x8eP\xae\xcc\x7f?o\xca<\xd6\xe78\x978\xa7\x8a\xa5\x92\x15j\xb0a\x9eQ9m\xb9Z^\xb7\xdeW;\xf2o^\xd4@\xfdsf|\xc62oE\xdb1\xf4\x19\x1fr\x01\xf6\xe3\xe3\xdc\xa0\x1c\xb9\x96r\xe5\x85\x0f\xf2\xa6\xbc\xee\xa5\xbd\xb9D9\xc5\xb9U\xe8\x95\xacR}\xa9\xaf\x1bU=\xcf\xbc_\xc1\xfd\x17\xb3\xba\xef/\xde\xc0\xe4\x85\xf4\xd90r\x02\xb2\xc2\x9c\x13\x9c\x1b\x07\xca\x1b\xce)\xd5\x97\xbe\x90s\xac\x10+\xa8*v\xb2\xca\xa1\x96/;P\x05\xf7_\x9cZ\xf8\"::T\xa2\xcbU\xda]*\xce\x01\xce\x85C\xcd\x1b\xca\xb1\xa5\x8a/\x95\xa0\xef*\x85RIEx\xb4\xd9\xde\xd0\x94{\x0f\xb5\x92\xef/\xde\x8a\xe6[\x85jt\xd3:\xdc\xc8\x95\x92\xe3\xa6\xd8\xf7p\x0e\x8c9o(\xd7(\xe7F\ne'9\xdf\xa8\x9a\xf9\x8a]32:\xd6\x8a\xf2w\xcc\xea\xde\xf5\xb4\x8e3\x91/%\xe7,\x8e\xfd\xe1\xe6\x0d\xe7\x1c\xad\xe3<\x99+\x18sx\xfd\xd7\xd9\xe9\xc1\xcdv\xcd\xf0\x9e\xb1V\xf2\xbf\x95\x1d\xdeC\xeb\xd8\xe4\xd4\x02W\xd2:#\xc8\x9b\xa2\x17\xa1X_\xc51\x1f\x87\xbc\xd9\xcc9\xc8\xb9([%\x8f\xd3\xc2\xcd\xbf\xb1\x92\xb36\x1dn\x05\xf7_\xacd\xff\x9bz\xb8\xf5\x97\xb4\xee\xa3\x91CE\xebh\x8e1\xc7z\xfc\xf2f\xd6&\xceEZ\xf7\xb12T\xb0\x8c\x96\x0b\x8d\xea\x9e\xa7\xecd\xff\xd6\xf1\xaa\xe4\x07G\x85T\xff\xdbf\xf5\xcc'\xb9\xdb\x96\xdd\x16\x14\x07\x8e\xe5\"\x8e-\xc7x\xdc\xf3\x86r\x91s\x92\xb6qA\xbe\xf2f\x12-\x8dn\xbb\xfab\xbd\xb2}-\x9d\xde\xde\x1b\xefJ~\xe8\xd4\xf9\x1emc\x0do\x8b\xb6\xd9\x94\xdd6\x14&\x8e]s6o^\xccA\xde\xac\xcd\xe6Mc.\xf3\x86\xf7\xc8\x11\xad\xa2e\xb9\x11\x9f\xbea\xa2*\xf8\x7f\xe3\xde\xb4-\xda\xe6\x8d\xb4\xed\x13\x04.Q)D|y\xc8\\\x8a\xe1My\xc8\x9b\xe5\x9c\xb3\xb9:\x9b\xd4z\xc3M+\xa9\xdf\xf8N\xae*\xf9\xa1\xfe\xe5\xbbj\xa8\xe1a\xe12\xe6S9\xca\x91s\x05\xa3\x9ccF\xb1{\x84c\x98\xfb\xbc\xe9\x7f\xc7[\xd1\xb4\x82\xca\x91\xceEeO\xceu\x05\xf7_\x94@\xed:\x8f]\xcdCz\xad\xa2\x08/\\+\xb2\xb3F+\xc7\x8ac\x96\xef\xbc\xa1\xb2\x9c\x94\x8bJ\x9f\x91\xef\x8a\xf2\xa2G\xdb7\xba\xcd\x04\x9f:3\x02\xb3\xef2\xe2\x98\x0cs\x8c8V2\xe4\x0c\x95\xe7\xb4\\T\xfdR*[\x1dr\xfb\x88\xd5s[r\x9b\xca\x12_=\xd6\xf9\x00\xe7\x1c\xe7^\xc1\x1ca<\xbe\xf41z\xb4\xfdf:\xfdJ1\x14l\xa7\x07\xff\xa9'\xba~.\x9cJ\xaf\xc0\xec\xfb\xe1\xf5\x10\xa8\xed\xb8\x0d\xb9-%\xe9Fo\xe7\x1c\xe3\\\x93\xb1Ku(\xda\x8dD\xcfO\xadT\xff\x1aY\x8e6\xdeH\xcb\x1djp\xf2\xa7\x05n\xc6\x1a\x8b\x08\xb7\x19\xb7\x9d4\xbd\x82T\xff\x0b\x9c[\x9cc\x85\xde\xb8 \xad\xb2c\x91Y\xdd\xfb'Y\x1a\xd7\x8cM_\xa5\x84\x1b.\xe2\xee\x02r\xff\xa0&s[q\x9bI\x13?\xca%\xce)\xce\xadbid\x8f\xd3\xe3\x9fm\xa5\x06\x1e\xb3k2[\xe48=\x0fn\xf1X\x89\x1f\xbb\xb50\xff\xa8\xf3b?\xf8?^n\x1bj\xa3\xeb\xb8\xad\xe4\x18t\xc9l\xe1\x1c\xe2\\\x12Ez\xfd]\xbd\x91\xec\xbb\xc5J\x0fm\x90\xe5h\xa4\xf8k\xeeq\x9a\x91\xa3\xa8l~\xec\x13\x1f\xf0s\x9b(\xfe\xf4\xbd\x12\xcdmm\xe0\xdc)\x85\x81\x16\x9f\x1an9\x97\x8e\x06odg\xbf\xf3>\xebj\xc4\xa7\xaf+s\x1b\x17e\x87\x82K\xf9\x8eE\xae\xbb\x8f\xdb\x82\xdbD\x92\xab#\xf8\x92\x917\xd4p\xeb9\xa56T?\x85*\xbe\xd5JgFe\xd8I\xf6^\x0e=\xc9\xc9\x0f\x890Kx\x071\xa9\x0d~%\xc9\x1c\xd6{Vjh\x94s\x84\x7f\x07\x95b0\x1c\x1e#Y\xef\xb1\x93?SC\x0d\xb2\xfc.\xd9A\xdd\x8a\x87\xa9l\xc7\x8b\xd2\xba*\x98\xebz<\xd5\xfd\x11n\x03\x19b\xc19\xe11\xa2?\xe1\x1c\x11%|\x85\xb6C\xa8\xfe\xb8C\xf1\x9d\xea\xd4C\xaf\xc8ry\xb4\x11\xebz\x8e\xca\xb6\x84\x16\xb5\x04b\xc0u\\\xc2u\x96\xe5\xf6\x05\xce\x05\xca\x89S(7b\x02\xb7/\xec\xed\xf7Z\xc2\xe9\x99\xed\xd2\xc3\xbf7\xabfn\xcb\xff\xe9}h\xb7\x91\x98\xb1\x81\x82t\x13\x95\x8dG\xb9\x8a\xf1\xf2y\xaeS3\xd7\x91\xeb\xcau\xce\xfb\xf0-\xc5\x9er\xe0>\xca\x85\x91\xbd9\x81'\xd8\xfc\x0f~\x1eo\x1fu\xb9\xae\xf2\x86\x9b\xa4\x18\xe52b\xd3\xb6\xa8\x81\xfa;\xa9\\\xa7\x17\xd9P0\xd7\xe5t\xae\x1b\xd7Q\x8a \\\x8a9\xc7\x9es@\xe0V\x85\x8f=\xaa\xd5\xba\x8c\xd8\x02\xc5\x9fz^\x86\x1f\xeftT\xe3;\xd2VQ\xb9.+\x92\xa1`\xae\xc3e\\'\xae\x9b\x0c?\xc6)\xd6\xcfr\xcc9\xf6\x027\xbb\x1d\x12>\x82,\xd0\"\xad+(\x88\x9b\xa4\xf8\xd1X\x9e\xde\xa8\xf8R\x97;\xfd\xe9\x1eQ\x98\x93T\x8a\x12\xe9\xea\xe5:p]\xa4\x98\x11\xa7\xd8r\x8c\xa9l\xf3q\xd68\xbc\xb3IF\x0dMY\xaaWv\xbe,\xc5U\xa3\x95\x1d\xdb\xf4X\xe7\xedf\xa2\xfb\xf4l\x1f\xb9PX\xf4;\xe3\x0c39\xeb\x0e\xae\x83\x1cm\xd9\xf92\xc7\x96\xca6\x84\xb3\xc6\x91\xa9t*\xfe\xefP\x80\x9f\xb1\xd2C;$\x18\xe5\x1a\xb5\xd3C/\xe9\xd1\xf6/ 5\x1a\x97|\x94\xc5\xc1e\xe4\xb2r\x99%\x99\x11\xdf\xc1\xb1\xa4\x98~K\xe0\x8e\xcfq\x1d\xe9\x9aO\x0d\xfb\x90\x95\xec\x7fK\x9a\xfb\x10\xa2m\x17k\xc1\x86\xa9\x92\x0e\x07\xab\\6=\xd2\xb6X\x9a\xcbE(v\x1cC*\xdbY\x18\xa1\x9a\x18\xd3\x94\xf2\xba\xe5\xd4\xd0\x1b\xe5\xb9\xb2t\xe6\n\x8f?9[\xb2Q.\xaf\xc7_}\x14\x95m\xa5<;\xc7\xc0\x9b\x14\xbb\x1b8\x86H\xe3\x89\xee6\x08\xf1M_:\xf3\x8e];\xbc[\x8e\xcb\"Fv\n\xb7>O\x88j%\xcfG\xc6I{\xcb\xe0\xd6O\xe42I\xd16\x1c#\x8a\x15\x95\xed\x1b\x02\x93~9\xdcI\x1c\x8e\xe3\x8d\xf8\xf4?\xcbr\xdd\x10-\xbb\xd4P\xe3\xd5\"\x87O\xef\xfb\x08I5\xd8x\x0d\x97E\x96\xeb\xdb(F\x8fS\xac\x8e\xc3\xce\x91\x87aK\xbe_A-\xaf[&K7\xc2\xae\xc9\xbcmT\xf5\xf2\xb5\\#yh\x8f\xd9\xbcm.\x83,\xedA\xb1Y\xea\xd6\xc3\xfcN\x17\x05\xe9\x9a\xaf\xf9\x125V\xe9\xb1\x13_\xb5\xd2\x83\xafK\xb2\x93l\xb7R\x03k\xca\xbc\xe5_\x16\xb9\xb92\xd8\xe4m\xf16y\xdb\x92\\\xf4\xf9:\xc5\xe4\\\xe1\x8d\xf3(\x15\xee\xff\x97@P 5\x9f\xaa'\xbaWH\xf3\xe3\xbd\xaa\xf7E-\xdcr\x0d\x95\xadc\x02\xeb\xdd\xc9\xdb\xe0m\xc9Ro\xa3\xaa\xfbn\x8e\x05\xc7\x04i)\x17C\x89\xb4\xf6j\x91\xf6k$\x1a\xd6|\xd5H\xcc\xf8-\x95m\xde\x04\xd4w\x9e\x11\x9f\xf1;\xde\x864O4\x8c\xb4_\xcd1\xe0X \x1d\xa5\x9d/\xf1\xa4\x8dD\xcf\x12:\xcd\xaf\x91\xa4\xcb\xb5K\x8b\xb4>\xe62bgS\xf9\xe2\xe3P\xc78\xadk!\xaf\x93\x1f\xce,I\x97\x8a\x9f0\xb2\x84\xda>\x85\xf9\x8d\x02\xe9r\xe9\x89\x19\xe7\xd0\xd1u\xa5<\x93\x8a\xed\xcf{\x83\x0d|\xc1c\xf7a&\x11\x7f\xa7\x9b\xd6q9\xad\xeb\x05\x89\xce\x92+\xf5\xaa\xee\xaf\x08\x89\x9e\x83\x0b\x878\xca\xa5T4\x0eZ\xa9\xfe\xdb\xcdD\xcf\xdbv:3*\xc1\xe3j6\xa9\xe5\xb5\xb7 \xa7\x87\xdf\xfd>\x96\x89E/\x7f\x87\xbf\xcb\xeb\xc8\xffC\xf82\xa3Fl\xda[Vj\xe06\xa5\xa2i\x00\xa3T\x85-\xa2\xf8jn\xd2+;\x9e\xa5\xe4\x92b\xf2\xccm'V\xb9\xed\xd8\xdcl\x97k\xd2A\xce\x1aq\xfe,}\xe7II\x9eI\xb5\x93\xdb\xd2\xa5\x85\xaf\xa7\xb2\x85\x91^\xc5\x81\xdf\x11\xb1P\xf1\xa7\x1f5\xaaz\xa4\x18\n\xf5V4\xff\xc3\xa9\x07/\xa1rM\x11\x1f=\x89\xc6\x7f\x9bB\x9f\xb9\x94?+\xc7\x08U\xcf6j\xc3?R\xb9\x16d\xdb\x14\x8a\x08_R]/\x1c\xca\x83vr`\xab\x0c].\xea\xfe\xed\x98\xe4R\xff N\x97Z\xd1z\x85\x11\xefZ]*;\x07\xd5\xf5i5\xd2\xfa=\x81\xdba\xe1\x10\xc5\x14\x7f\xf2l\xbd\xb2\x83\xdf\xe2\xba\xa7Xw\x0c\xae\x9b\x1e\xebxH\xf1\xa7\xf8\xb9T1\x84\x1d\xc6j\x8eY5\xf3\x1e+5\xf0j\xd1u\xa9\xa8N\xfcP\x07\xaa\xe31\x083\x1c\x89\x16o\xb4\xe3\x87\xd4G_[<;\xc7\xac\x17\xb5\xca\x0e\x9e\xf8kFxa<\x98.#r\xb6\x9d\x1a\\O\xdd\x92\x1d\x05\xdc\xa5\xda\xc1up\x19Q\xeeR\x99\x08+\x8c\xb7\x99fr\xd6\xe3\xbe\x9a\xcc\xb6\x82\xdbA\xa8\xcc\\v\xb1\xef~\x14\x80 \x93\xd0\"m\xdf/\xb8\xc9\xbfH\x1b\xbfZ \x8e\xf0\xc1D+\x13\x81z\xc3\xe1\x0d|\xb2Pv\x0e*\xeb\x1c.\xb3\xc0C\xa2!\x87\xbcZ\xb0\xe1h\xfa\xf1~\x9f\xbc\x93\x7f\xfd\xf7j\xa1&\xd9\x1e\x8b\n%D\xf1VL\x9b\xa6\xc7g\\\xa2E;\xb6\xcb\xb2cx\xc3M[\xf5x\xf7bodz\xa7(\xccw\x99@\x11q(\xbe\x86\x84\xc7W\xfb\x05\xb7\x9dXMg\x94]y\xbc\\d\x97\xdb\x8e\xff\x8d\x9fz\xa2\xfa\x9bd\x7f5\x03\x94\x18\xbe\x99\xe8\x145P\xf3\x0b=\xda\x96\xf3\xf7\xfe\xe9\x95m\x9bi\xdb7S\x19>C\x8b\x86p\x80\x8c\xf8E\xa4\xddn+~\xb9\x1a\x9c\x92\xb3\xd9w\xda\xd6\x06\xda\xe6\xa5\xb4\xed.\x81W\x99A\x01\xf0 Q\xb6X\xaf\xec|\xd2\xac\xee\x9b\xb09\x13\xab\xba\xef]\xda\xc6*\xda\xd6wi\x9b6\x9a\x1d\n\xedl\xf2Yo\xb0\xf1\xd7f\xa2{\xdc\xdf\x8aE\xeb\xdc\xe4\x0d5\xf2+\xaaO\xcbn\x0b\xa0\xe0\xf0\xbcC\xa3K\x0b\xde`V\xf5\xbcj\xa73{\xc6\xe1 #{x]\xb4N~.U\x83\xc0\xdc\x06\x14\x01\xfe]p\x99Y5s\xed\x91\xbc\x88\x94\xbe\xbb\x9d\xd7A\xeb\xe2\xdba\xf1Z\x01(*|o\xf7\xb1j\xb0\x9e_r\xb3k\xec\x17\x1afv\xa9\xc1\xc9\x0f\xd2:\xe6\xa0)\xa1X9\x84\xd0\x83en\xed\xc6\xb1\xee \xf4\x9d\xeb\xf9\xbb\x02s\x1bP\x02g\x92r\xa7\xea_d$\xba\x0f\xfa\xe2\x1b\xfa\xcc\x1a\x97\xea?\x8f\xbf#\xf0\x84\x11(!\x01\x8f\xafj\x8e7\xd8p\xd7\x01/\x19 5\xdeI\x9f\xf9Dv\xe7\x00(9\x9aK\xabl\xd5BM\x8b= 0) {\n output_string += `
\n
\n Just getting started? Try this:
\n :create index test
\n :use test
\n :create frame foo
\n SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit\n `\n }\n }\n }\n }\n\n\n var markup =`\n
\n
\n
\n
\n
Input
\n       \n Source: ${res.indexname}\n
\n
\n ${res.input}\n
\n
\n
\n
\n
output
\n       \n ${res.querytime_ms} ms\n
\n
\n ${output_string}\n
\n
Expand
\n \n
\n
\n
\n
\n \n Export raw .csv\n
\n
\n
\n `\n node.innerHTML = markup;\n this.output.insertBefore(node, this.output.firstChild);\n\n // Expand when overflow\n var element = this.output.firstChild.getElementsByClassName(result_class)[0];\n var expand = this.output.firstChild.getElementsByClassName(\"expand\")[0];\n if (element.clientHeight < element.scrollHeight) {\n expand.style.display = 'block';\n } else {\n expand.style.display = 'none';\n }\n expand.onclick = function () {\n element.style.height = element.scrollHeight + \"px\";\n expand.style.display = 'none';\n return false;\n };\n }\n\n populate_index_dropdown() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/schema')\n var select = document.getElementById('index-dropdown')\n\n xhr.onload = function() {\n var schema = JSON.parse(xhr.responseText)\n for(var i=0; i 0) {\n select.value = 1;\n }\n }\n xhr.send(null)\n }\n\n}\n\nfunction populate_version() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/version')\n var node = document.getElementById('server-version')\n\n xhr.onload = function() {\n var version = JSON.parse(xhr.responseText)['version']\n var version_major_minor = /(v\\d+\\.\\d+)/.exec(version)[0]\n var doc_link = document.getElementById('nav-documentation')\n doc_link.onclick = function() {\n window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')\n }\n node.innerHTML = version\n }\n xhr.send(null)\n}\n\nfunction handle_nav_click(e) {\n // e.id = \"nav-xxx\"\n name = e.id.substring(4)\n set_active_pane_by_name(name)\n window.location.hash = name\n}\n\nfunction set_active_pane_by_name(name) {\n // toggle the nav buttons\n document.getElementsByClassName(\"nav-active\")[0].classList.remove(\"nav-active\")\n document.getElementById(\"nav-\" + name).classList.add(\"nav-active\")\n\n // toggle the main interface content divs\n document.getElementsByClassName(\"interface-active\")[0].classList.remove(\"interface-active\")\n document.getElementById('interface-' + name).classList.add(\"interface-active\")\n\n // hack hack\n switch(name) {\n case \"cluster\":\n update_cluster_status()\n break\n case \"documentation\":\n open_external_docs()\n break\n }\n}\n\n\nfunction update_cluster_status() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/status')\n status_node = document.getElementById('status')\n xhr.onload = function() {\n var status = JSON.parse(xhr.responseText)\n render_status(status)\n }\n xhr.send(null)\n}\n\nfunction render_status(status) {\n // render node table\n var nodes_div = document.getElementById(\"status-nodes\")\n while (nodes_div.firstChild) {\n nodes_div.removeChild(nodes_div.firstChild);\n }\n\n var nodes = status[\"status\"][\"Nodes\"]\n table = document.createElement(\"table\")\n tbody = document.createElement(\"tbody\")\n table.appendChild(tbody)\n var caption = document.createElement(\"caption\")\n caption.innerHTML = \"(\" + nodes.length + \")\"\n table.appendChild(caption)\n\n var header = document.createElement('tr')\n markup = `Host\n State`\n header.innerHTML = markup\n tbody.appendChild(header)\n for(var n=0; n${nodes[n][\"Host\"]}\n ${nodes[n][\"State\"]}`\n row.innerHTML = markup\n tbody.appendChild(row)\n }\n nodes_div.appendChild(table)\n\n // render index tables\n var indexes_div = document.getElementById(\"status-indexes\")\n while (indexes_div.firstChild) {\n indexes_div.removeChild(indexes_div.firstChild);\n }\n\n var indexes = nodes[0][\"Indexes\"] // TODO currently comes from only node 0\n for(var n=0; nName\n Cache Type\n Cache Size`\n header.innerHTML = markup\n tbody.appendChild(header)\n\n var frames = indexes[n][\"Frames\"]\n if(frames) {\n for(var m=0; m${frames[m][\"Name\"]}\n ${frames[m][\"Meta\"][\"CacheType\"]}\n ${frames[m][\"Meta\"][\"CacheSize\"]}`\n tbody.appendChild(row)\n }\n }\n indexes_div.appendChild(table)\n }\n\n // render slice tables\n // TODO enable when Slices element is present in status response\n /*\n var slices_div = document.getElementById(\"status-slices\")\n data = \"\"\n for(var n=0; n\"\n }\n }\n slices_div.innerHTML = data\n */\n\n}\n\nfunction open_external_docs() {\n window.open(\"https://www.pilosa.com/docs\");\n}\n\nfunction check_anchor_uri() {\n var pane_names = {\"console\": 0, \"cluster\": 0, \"documentation\": 0}\n var anchor = window.location.hash.substr(1);\n if(anchor in pane_names) {\n set_active_pane_by_name(anchor)\n }\n}\n\nDate.prototype.today = function () {\n return this.getFullYear() +\"/\"+ (((this.getMonth()+1) < 10)?\"0\":\"\") + (this.getMonth()+1) +\"/\"+ ((this.getDate() < 10)?\"0\":\"\") + this.getDate();\n}\n\nDate.prototype.timeNow = function () {\n return ((this.getHours() < 10)?\"0\":\"\") + this.getHours() +\":\"+ ((this.getMinutes() < 10)?\"0\":\"\") + this.getMinutes() +\":\"+ ((this.getSeconds() < 10)?\"0\":\"\") + this.getSeconds();\n}\n\npopulate_version()\n\n\nclass Autocompleter {\n constructor(input, output) {\n this.input = input\n this.output = output\n this.keyword_map = this.static_keywords\n this.init_dynamic_keywords()\n }\n\n get static_keywords() {\n return {\n // keyword: length of substring that comes after cursor\n \"SetBit()\": 1,\n \"ClearBit()\": 1,\n \"SetRowAttrs()\": 1,\n \"SetColumnAttrs()\": 1,\n \"Bitmap()\": 1,\n \"Union()\": 1,\n \"Intersect()\": 1,\n \"Difference()\": 1,\n \"Count()\": 1,\n \"Range()\": 1,\n \"TopN()\": 1,\n \"frame=\": 0,\n }\n }\n\n complete() {\n var completer = this\n // extract word fragment ending at cursor. a word fragment:\n // - starts with last nonalpha character before cursor (or beginning of string)\n // - ends at cursor\n var word_start = completer.input.selectionEnd-1\n while(word_start>0) {\n var c = completer.input.value.charCodeAt(word_start)\n if(!((c>64 && c<91) || (c>96 && c<123))) {\n word_start++\n break\n }\n word_start--\n }\n var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)\n\n // check for keyword match and insert if exactly one match\n var matches = []\n for(var keyword in this.keyword_map) {\n if(keyword.startsWith(input_word)){\n matches.push(keyword)\n }\n }\n if(matches.length > 1) {\n // completer.output.innerHTML = whatever\n }\n\n if(matches.length == 1) {\n // completer.output.innerHTML = \"\"\n var cursor_pos = completer.input.selectionEnd\n var completion = matches[0].substring(input_word.length)\n var before = completer.input.value.substring(0, cursor_pos)\n var after = completer.input.value.substring(cursor_pos)\n completer.input.value = before + completion + after\n var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]\n completer.input.setSelectionRange(new_pos, new_pos)\n }\n }\n\n init_dynamic_keywords() {\n // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list\n }\n\n add_keyword() {\n // call when index or frame created in webui\n }\n\n remove_keyword() {\n // call when index or frame deleted in webui\n // issue: if e.g. multiple indexes have same frame, removing one removes all.\n // solution: maintain count. requires more elaborate representation of keywords.\n }\n}\n\nvar input = document.getElementById('query')\nvar output = document.getElementById('outputs')\nvar button = document.getElementById('query-btn')\nvar autocomplete_output = document.getElementById('autocomplete-container')\n\nautocompleter = new Autocompleter(input, autocomplete_output)\nrepl = new REPL(input, output, button, autocompleter)\nrepl.populate_index_dropdown()\nrepl.bind_events()\n\ninput.focus()\n\ncheck_anchor_uri()\n\nfunction isJSON(str) {\n try {\n JSON.parse(str)\n } catch (e) {\n return false\n }\n return true\n}\n\nfunction parse_query(query, indexname) {\n var keys = query.replace(/\\s+/g, \" \").split(\" \");\n var command = keys[0];\n var command_type = keys[1];\n var command_name = keys[2];\n var option_str = keys.slice(3, keys.length)\n var options = parse_options(option_str);\n if (command !== \":use\") {\n if (!command_name){\n return {}\n }\n }\n\n var parsed_query = {};\n parsed_query[\"command\"] = command.substr(1, command.length);\n parsed_query[\"command_name\"] = command_name;\n switch (command) {\n case \":create\":\n parsed_query[\"request\"] = \"POST\";\n if(Object.keys(options).length === 0) {\n parsed_query[\"data\"] = \"\";\n } else {\n var opts = {\"options\":{}};\n for (var o in options) {\n opts.options[o] = options[o]\n }\n parsed_query[\"data\"] = JSON.stringify(opts);\n }\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n break\n }\n break;\n case \":delete\":\n parsed_query[\"request\"] = \"DELETE\";\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n }\n break;\n case \":use\":\n parsed_query[\"command_name\"] = keys[1];\n break;\n default:\n return {}\n }\n return parsed_query;\n}\n\nfunction parse_options(option_str) {\n var int_keys = [\"cacheSize\"];\n var bool_keys = [\"inverseEnabled\"];\n var options = {};\n for (var i = 0; i < option_str.length; i++) {\n var parts = option_str[i].split('=');\n if (int_keys.indexOf(parts[0]) !== -1 ){\n options[parts[0]] = Number(parts[1])\n } else if (bool_keys.indexOf(parts[0]) !== -1){\n options[parts[0]] = (parts[1] == \"true\")\n } else {\n options[parts[0]] = parts[1]\n }\n }\n return options;\n}PK\x07\x08\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-cluster-active.svgnav_cluster_1\nPK\x07\x08\xc1J\xead \x02\x00\x00 \x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-cluster.svgnav_cluster_1PK\x07\x08\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-console-active.svgnav_consolePK\x07\x08\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-console.svgnav_console\nPK\x07\x08\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00assets/nav-documentation-active.svgdocumentation\nPK\x07\x08\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00assets/nav-documentation.svgdocumentationPK\x07\x08\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00assets/nav_item1.svgnav_item1PK\x07\x08+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xa0~\xe6J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00assets/style.css*{\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n}\nbody{\n font-family: sans-serif;\n background-color: #fbfcfd;\n margin: 0;\n color: #102445;\n}\nh2{\n margin-bottom: 30px;\n}\n\nh5{\n text-transform: uppercase;\n letter-spacing: 2px;\n line-height: 1.21;\n margin: 0;\n}\na{\n line-height: 1.38;\n letter-spacing: 0.2px;\n text-decoration: none;\n color: #102445;\n}\n\n\na:hover{\n color: #1db598;\n}\n\ntextarea{\n width: 100%;\n margin-bottom: 10px;\n border-radius: 2px;\n background-color: #fbfcfd;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n outline: none;\n padding: 30px;\n}\n\n\nselect{\n /*-webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n background: url(\"img/chevron-down.png\") no-repeat calc(100% - 10px) !important;*/\n border-radius: 3px;\n background-color: #fbfcfd;\n width: 187px;\n height: 50px;\n border: solid 1.5px #e4eff4;\n font-size: 18px;\n font-weight: bold;\n line-height: 1.39;\n letter-spacing: 0.2px;\n color: #102445;\n padding: 10.5px;\n\n}\n\nbutton{\n width: 165px;\n height: 50px;\n border-radius: 3px;\n background-color: #1db598;\n outline: none;\n border: none;\n font-size: 16px;\n color: white;\n}\n\nem{\n font-style: normal;\n opacity: 0.5;\n font-size: 14px;\n font-weight: 500;\n letter-spacing: 0.2px;\n color: #102445;\n}\n\n.header{\n height: 92px;\n display: flex;\n align-items: center;\n justify-content: space-between;\n width: 90%;\n margin: auto;\n}\n\n.container{\n display: flex;\n height:100%;\n min-height: 100vh;\n}\n.nav{\n color: white;\n display: flex;\n flex-direction: column;\n width: 150px;\n background: #3c5f8d;\n}\n\n.nav-item{\n height:150px;\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n border-bottom: 3px solid #2a4871;\n cursor: pointer;\n}\n\n.nav-active{\n background: #f2f7f9;\n font-weight: bold;\n color: #1db598;\n}\n\n.nav-item > .nav-image {\n display: flex;\n}\n\n.nav-item > .nav-image-active {\n display: none;\n}\n\n.nav-active > .nav-image {\n display: none;\n}\n\n.nav-active > .nav-image-active {\n display: flex;\n}\n\n\n.interface{\n display: none;\n flex: 1;\n flex-direction: column;\n align-items: center;\n background: #f2f7f9;\n}\n\n.interface-active{\n display: flex;\n}\n\n.query{\n margin-bottom: 30px;\n}\n.query,\n.output-container,\n.status-container{\n width: 75%;\n}\n\n.output{\n margin-bottom: 30px;\n}\n\n.input-controls{\n display: flex;\n justify-content: flex-end;\n}\n\n.tabs{\n display: flex;\n background: #eaf2f6;\n}\n.active-tab{\n background: white;\n font-weight: bold;\n color: #1db598;\n\n}\n\n.tab{\n height:60px;\n width: 100px;\n border-top-right-radius: 5px;\n display: flex;\n align-items: center;\n justify-content: center;\n visibility: visible;\n cursor: pointer;\n\n}\n\n.pane{\n background: white;\n padding: 30px;\n display: none;\n}\n\n.active{\n display: block;\n}\n\n.result-io-header{\n display: flex;\n align-items: center;\n margin-bottom: 15px;\n}\n\n.result-input,\n.result-output,\n.result-error{\n height: 60px;\n border-radius: 2px;\n background-color: #fafafa;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n color: #102445;\n padding: 15px;\n margin-bottom: 15px;\n word-break: break-all;\n overflow-wrap: break-word;\n overflow:hidden;\n}\n\n\n.result-output{\n background-color: #edf9f7;\n border-left: solid 4px #1db598;\n}\n\n.result-error{\n background-color: #fbf1f0;\n border-left: solid 4px #fa3035;\n color: #fa3035;\n}\n\n.raw{\n height: 253px;\n display: flex;\n align-items: center;\n justify-content: center;\n}\n\n\n.result-table > table {\n border-left: solid 4px #1db598;\n}\n\ntable{\n border: solid 0.5px #e0e0e0;\n width: 100%;\n margin-bottom: 30px;\n /*color:#3c5f8d;*/\n}\ncaption{\n text-align:left;\n font-size: 16px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n text-align: left;\n}\nth{\n font-size: 14px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n color: #102445;\n text-transform: uppercase;\n text-align: left;\n padding: 21px 30px;\n background-color: white;\n}\ntr{\n border: solid 0.5px #e0e0e0;\n background-color: white;\n}\ntr:nth-child(even) {\n background-color: #f2f7f9;\n}\ntd{\n padding: 21px 30px;\n}\n\n.expand {\n text-align: center;\n}\n\n.query h2 {\n display: inline-block;\n}\n\n.query-tooltip {\n position: relative;\n display: inline;\n color: #000;\n margin-left: 5px;\n}\n\n.query-tooltip:hover {\n color: #000;\n}\n\n.query-tooltip-content {\n background-color: rgb(250, 250, 250);\n border: solid 1.5px #e4eff4;\n color: #102445;\n border-radius: 2px;\n padding: 15px;\n margin-bottom: 15px;\n\n position: absolute;\n left: 80px;\n top: -30px;\n z-index: 1;\n}\n\n.query-tooltip-container {\n position: relative;\n visibility: hidden;\n}\n\n.query-tooltip:hover+.query-tooltip-container{\n visibility: visible;\n}\n\n.code{\n font-family: monospace;\n}\n\nPK\x07\x08\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00cz\xbfJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00index.html\n\n\n \n \n \n \n Pilosa WebUI\n \n\n\n
\n \"\"\n
\n
\n
\n
\n
\n \"\"\n \"\"\n Console\n
\n
\n \"\"\n \"\"\n Cluster Admin\n
\n
\n \"\"\n \"\"\n Documentation\n
\n
\n
\n\n
\n

Query

\n ?\n
\n
\n
PQL
\n
\n SetBit(frame=foo, rowID=0, columnID=0)
\n ClearBit(frame=foo, rowID=0, columnID=0)
\n SetRowAttrs(frame=foo, rowID=0, color=\"blue\")
\n SetColumnAttrs(frame=foo, columnID=0, shape=\"circle\")
\n Bitmap(frame=foo, rowID=0)
\n Range(frame=foo, rowID=0, start=\"2010-01\", end=\"2017-03\")
\n Count(<BITMAP_CALL>)
\n TopN([BITMAP_CALL], frame=foo, n=20)
\n Union([BITMAP_CALL, ...])
\n Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
\n Difference(<BITMAP_CALL>, <BITMAP_CALL>)\n
\n
\n
Special commands
\n
\n :create index test [columnLabel=column]
\n :use test
\n :create frame foo [rowLabel=row]
\n :delete index test
\n :delete frame foo\n
\n
\n <tab>: autocomplete
\n <up>/<down>: history
\n
\n
\n \n
\n
\n \n    \n \n
\n
\n
\n\n
\n

Output

\n
\n \n
\n
\n\n
\n\n
\n
\n

Nodes

\n
\n
\n
\n
\n

Indexes

\n
\n
\n
\n
\n \n
\n\n
\n\n
\n docs!\n
\n\n
\n \n\n\nPK\x07\x08\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3JJ\x1c\xff\xa8G\x0e\x00\x00G\x0e\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00assets/chevron-down.pngPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x96\x84jK\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x8c\x0e\x00\x00assets/main.jsPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc1J\xead \x02\x00\x00 \x02\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x92W\x00\x00assets/nav-cluster-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xe6Y\x00\x00assets/nav-cluster.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81/\\\x00\x00assets/nav-console-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x1a^\x00\x00assets/nav-console.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xfc_\x00\x00assets/nav-documentation-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x819b\x00\x00assets/nav-documentation.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81kd\x00\x00assets/nav_item1.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xa0~\xe6J\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81Of\x00\x00assets/style.cssPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00cz\xbfJ\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xcay\x00\x00index.htmlPK\x05\x06\x00\x00\x00\x00\x0b\x00\x0b\x00\xf2\x02\x00\x00\xf1\x89\x00\x00\x00\x00" + fs.Register(data) +} diff --git a/test/executor.go b/test/executor.go index c04f91eca..c4af65b01 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/inmem" "github.com/pilosa/pilosa/pql" ) @@ -42,6 +43,7 @@ func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster + e.TranslateStore = inmem.NewTranslateStore() e.Node = cluster.Nodes[0] return e } diff --git a/translate.go b/translate.go new file mode 100644 index 000000000..398055fdb --- /dev/null +++ b/translate.go @@ -0,0 +1,1006 @@ +package pilosa + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/cespare/xxhash" +) + +const ( + LogEntryTypeInsertColumn = 1 + LogEntryTypeInsertRow = 2 +) + +const ( + DefaultMapSize = 10 * (1 << 30) // 10GB + + DefaultReplicationRetryInterval = 1 * time.Second +) + +const ( + ReplicationBufferSize = 65536 +) + +var ( + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") +) + +// TranslateStore is the storage for translation string-to-uint64 values. +type TranslateStore interface { + TranslateColumnsToUint64(index string, values []string) ([]uint64, error) + TranslateColumnToString(index string, values uint64) (string, error) + + TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) + TranslateRowToString(index, frame string, values uint64) (string, error) + + // Returns a reader from the given offset of the raw data file. + // The returned reader must be closed by the caller when done. + Reader(ctx context.Context, off int64) (io.ReadCloser, error) +} + +// Ensure type implements interface. +var _ TranslateStore = &TranslateFile{} + +// TranslateFile is an on-disk storage engine for translating string-to-uint64 values. +type TranslateFile struct { + mu sync.RWMutex + data []byte + file *os.File + w *bufio.Writer + n int64 + writeNotify chan struct{} + + once sync.Once + wg sync.WaitGroup + closing chan struct{} + + cols map[string]*index + rows map[frameKey]*index + + Path string + MapSize int + + // If non-nil, data is streamed from a primary and this is a read-only store. + PrimaryTranslateStore TranslateStore + + // Delay after attempting to connect to a primary that the store will retry. + ReplicationRetryInterval time.Duration +} + +// NewTranslateFile returns a new instance of TranslateFile. +func NewTranslateFile() *TranslateFile { + return &TranslateFile{ + writeNotify: make(chan struct{}), + closing: make(chan struct{}), + cols: make(map[string]*index), + rows: make(map[frameKey]*index), + + MapSize: DefaultMapSize, + + ReplicationRetryInterval: DefaultReplicationRetryInterval, + } +} + +func (s *TranslateFile) Open() (err error) { + // Open writer & buffered writer. + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + return err + } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { + return err + } + s.w = bufio.NewWriter(s.file) + + // Memory map data file. + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + return err + } + + // Replay the log. + if err := s.replayEntries(); err != nil { + return err + } + + // Stream from primary, if available. + if s.PrimaryTranslateStore != nil { + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorReplication() }() + } + + return nil +} + +func (s *TranslateFile) Close() (err error) { + s.once.Do(func() { + close(s.closing) + + if s.file != nil { + if e := s.file.Close(); e != nil && err == nil { + err = e + } + } + if s.data != nil { + if e := syscall.Munmap(s.data); e != nil && err == nil { + err = e + } + } + }) + s.wg.Wait() + return err +} + +// Closing returns a channel that is closed when the store is closed. +func (s *TranslateFile) Closing() <-chan struct{} { + return s.closing +} + +// Size returns the number of bytes in use in the data file. +func (s *TranslateFile) Size() int64 { + s.mu.RLock() + n := s.n + s.mu.RUnlock() + return n +} + +// IsReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) IsReadOnly() bool { + return s.PrimaryTranslateStore != nil +} + +// WriteNotify returns a channel that is closed when a new entry is written. +func (s *TranslateFile) WriteNotify() <-chan struct{} { + s.mu.RLock() + ch := s.writeNotify + s.mu.RUnlock() + return ch +} + +func (s *TranslateFile) appendEntry(entry *LogEntry) error { + offset := s.n + + // Append entry to the end of the WAL. + n, err := entry.WriteTo(s.w) + if err != nil { + return err + } else if err := s.w.Flush(); err != nil { + return err + } + + // Move position forward. + s.n += n + + // Apply the entry to the current state. + if err := s.applyEntry(entry, offset); err != nil { + return err + } else if err := s.file.Sync(); err != nil { + return err + } + + // Notify others of write update. + close(s.writeNotify) + s.writeNotify = make(chan struct{}) + + return nil +} + +func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { + // Move offset to the start of the id/key pairs. + offset += entry.HeaderSize() + + var idx *index + switch entry.Type { + case LogEntryTypeInsertColumn: + idx = s.col(string(entry.Index)) + + case LogEntryTypeInsertRow: + idx = s.row(string(entry.Index), string(entry.Frame)) + + default: + return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) + } + + // Insert id/key pairs into index. + for i, id := range entry.IDs { + key := entry.Keys[i] + + // Determine key offset based on ID size. + sz := int64(UvarintSize(id)) + idx.insert(id, offset+sz) + + // Move sequence forward. + if id > idx.seq { + idx.seq = id + } + + // Move offset forward. + offset += sz + int64(UvarintSize(uint64(len(key)))) + int64(len(key)) + } + + return nil +} + +func (s *TranslateFile) replayEntries() error { + // Build a reader from the memory-map data. + fi, err := os.Stat(s.Path) + if err != nil { + return err + } + r := bytes.NewReader(s.data[:fi.Size()]) + + // Iterate over each entry and reapply. + for { + offset := s.n + + var entry LogEntry + if n, err := entry.ReadFrom(r); err == io.EOF { + return nil + } else if err != nil { + return err + } else { + s.n += n + } + + if err := s.applyEntry(&entry, offset); err != nil { + return err + } + } +} + +// monitorReplication is executed in a separate goroutine and continually streams +// from the primary store until this store is closed. +func (s *TranslateFile) monitorReplication() { + // Create context that will cancel on close. + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-s.closing; cancel() }() + + // Keep attempting to replicate until the store closes. + for { + if err := s.replicate(ctx); err != nil { + log.Printf("pilosa: replication error: %s", err) + } + + select { + case <-s.closing: + return + case <-time.After(s.ReplicationRetryInterval): + log.Printf("pilosa: reconnecting to primary replica") + } + } +} + +func (s *TranslateFile) replicate(ctx context.Context) error { + off := s.Size() + + // Connect to remote primary. + log.Printf("pilosa: replicating from offset %d", off) + rc, err := s.PrimaryTranslateStore.Reader(ctx, off) + if err != nil { + return err + } + defer rc.Close() + + // Wrap in bufferred I/O so it implements io.ByteReader. + bufr := bufio.NewReader(rc) + + // Continually read new entries from primary and append to local store. + for { + // Read next available entry. + var entry LogEntry + if _, err := entry.ReadFrom(bufr); err == io.EOF { + return nil + } else if err != nil { + return err + } + + // Write to local store. + if err := s.appendEntry(&entry); err != nil { + return err + } + } +} + +func (s *TranslateFile) col(index string) *index { + idx := s.cols[index] + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + return idx +} + +func (s *TranslateFile) row(index, frame string) *index { + idx := s.rows[frameKey{index, frame}] + if idx == nil { + idx = newIndex(s.data) + s.rows[frameKey{index, frame}] = idx + } + return idx +} + +// TranslateColumnsToUint64 converts values to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertColumn, + Index: []byte(index), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + + ret[i] = v + + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.keyByID(value); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.rows[key] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertRow, + Index: []byte(index), + Frame: []byte(frame), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + ret[i] = v + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.keyByID(id); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +// Reader returns a reader that streams the underlying data file. +func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + rc := NewTranslateFileReader(ctx, s, offset) + if err := rc.Open(); err != nil { + return nil, err + } + return rc, nil +} + +type LogEntry struct { + Type uint8 + Index []byte + Frame []byte + + IDs []uint64 + Keys [][]byte + + // Length of the entry, in bytes. + // This is only populated after ReadFrom() or WriteTo(). + Length uint64 +} + +// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) HeaderSize() int64 { + sz := UvarintSize(e.Length) + // total entry length + 1 + // type + UvarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data + UvarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + UvarintSize(uint64(len(e.IDs))) // ID/Key pair count + return int64(sz) +} + +// ReadFrom deserializes a LogEntry from r. r must be a ByteReader. +func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { + br := r.(io.ByteReader) + + // Read the entry length. + if e.Length, err = binary.ReadUvarint(br); err != nil { + return int64(UvarintSize(e.Length)), err + } + + // Slurp entire entry and replace reader. + buf := make([]byte, e.Length) + n, err := io.ReadFull(r, buf) + n64 := int64(n + UvarintSize(e.Length)) + if err != nil { + return n64, err + } + bufr := bytes.NewReader(buf) + br, r = bufr, bufr + + // Read the entry type. + if err := binary.Read(r, binary.BigEndian, &e.Type); err != nil { + return n64, err + } + + // Read index name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Index = nil + } else { + e.Index = make([]byte, sz) + if _, err := io.ReadFull(r, e.Index); err != nil { + return n64, err + } + } + + // Read frame name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Frame = nil + } else { + e.Frame = make([]byte, sz) + if _, err := io.ReadFull(r, e.Frame); err != nil { + return n64, err + } + } + + // Read key count. + if n, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if n == 0 { + e.IDs, e.Keys = nil, nil + } else { + e.IDs, e.Keys = make([]uint64, n), make([][]byte, n) + } + + // Read each id/key pairs. + for i := range e.Keys { + // Read identifier. + if e.IDs[i], err = binary.ReadUvarint(br); err != nil { + return n64, err + } + + // Read key. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz > 0 { + e.Keys[i] = make([]byte, sz) + if _, err := io.ReadFull(r, e.Keys[i]); err != nil { + return n64, err + } + } + } + return n64, nil +} + +// WriteTo serializes a LogEntry to w. +func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { + var buf bytes.Buffer + b := make([]byte, binary.MaxVarintLen64) + + // Write the entry type. + if err := binary.Write(&buf, binary.BigEndian, e.Type); err != nil { + return 0, err + } + + // Write the index name. + sz := binary.PutUvarint(b, uint64(len(e.Index))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Index); err != nil { + return 0, err + } + + // Write frame name. + sz = binary.PutUvarint(b, uint64(len(e.Frame))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Frame); err != nil { + return 0, err + } + + // Write key count. + sz = binary.PutUvarint(b, uint64(len(e.IDs))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write each id/key pairs. + for i := range e.Keys { + // Write identifier. + sz = binary.PutUvarint(b, e.IDs[i]) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write key. + sz = binary.PutUvarint(b, uint64(len(e.Keys[i]))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Keys[i]); err != nil { + return 0, err + } + } + + // Write buffer size. + e.Length = uint64(buf.Len()) + sz = binary.PutUvarint(b, e.Length) + if n, err := w.Write(b[:sz]); err != nil { + return int64(n), err + } + + // Write buffer. + n, err := buf.WriteTo(w) + return int64(sz) + n, err +} + +// ValidLogEntriesLen returns the maximum length of p that contains valid entries. +func ValidLogEntriesLen(p []byte) (n int) { + r := bytes.NewReader(p) + for { + if sz, err := binary.ReadUvarint(r); err != nil { + return n + } else if off, err := r.Seek(int64(sz), io.SeekCurrent); err != nil { + return n + } else if off > int64(len(p)) { + return n + } else { + n = int(off) + } + } +} + +type frameKey struct { + index string + frame string +} + +const defaultLoadFactor = 90 + +// index represents a two-way index between IDs and keys. +type index struct { + seq uint64 // autoincrement sequence + data []byte // memory-mapped file containing key data + + // RHH hashmap for id-to-offset mapping. + // This is required so we don't need to store key data on the heap. + // https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf + elems []elem // id/offset key pairs + n uint64 // number of inuse elements + mask uint64 // mask applied for modulus + threshold uint64 // threshold when capacity doubles + loadFactor int // factor used to calculate threshold + + // Builtin hashmap for offset-to-id mapping. + offsetsByID map[uint64]int64 +} + +func newIndex(data []byte) *index { + idx := &index{ + data: data, + offsetsByID: make(map[uint64]int64), + + loadFactor: defaultLoadFactor, + } + idx.alloc(pow2(uint64(256))) + return idx +} + +// keyByID returns the key for a given ID, if it exists. +func (idx *index) keyByID(id uint64) ([]byte, bool) { + offset, ok := idx.offsetsByID[id] + if !ok { + return nil, false + } + return idx.lookupKey(offset), true +} + +// idByKey returns the ID for a given key, if it exists. +func (idx *index) idByKey(key []byte) (uint64, bool) { + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + if e := &idx.elems[pos]; e.hash == 0 { + return 0, false + } else if dist > idx.dist(e.hash, pos) { + return 0, false + } else if e.hash == hash && bytes.Equal(idx.lookupKey(e.offset), key) { + return e.id, true + } + + pos = (pos + 1) & idx.mask + dist++ + } +} + +// insert adds the id/offset pair to the index. +// This function will resize the map if it crosses the threshold. +func (idx *index) insert(id uint64, offset int64) { + idx.n++ + + // Add to reverse lookup. + idx.offsetsByID[id] = offset + + // Grow the map if we've run out of slots. + if idx.n > idx.threshold { + elems, capacity := idx.elems, uint64(len(idx.elems)) + idx.alloc(uint64(len(idx.elems) * 2)) + + for i := uint64(0); i < capacity; i++ { + e := &elems[i] + if e.hash == 0 { + continue + } + idx.insertIDbyOffset(e.offset, e.id) + } + } + + // If the key was overwritten then decrement the size. + if overwritten := idx.insertIDbyOffset(offset, id); overwritten { + idx.n-- + } +} + +// insertIDbyOffset writes to the RHH id-by-offset map. +func (idx *index) insertIDbyOffset(offset int64, id uint64) (overwritten bool) { + key := idx.lookupKey(offset) + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + e := &idx.elems[pos] + + // Exit if a matching or empty slot exists. + if e.hash == 0 { + e.hash, e.offset, e.id = hash, offset, id + return false + } else if bytes.Equal(idx.lookupKey(e.offset), key) { + e.hash, e.offset, e.id = hash, offset, id + return true + } + + // Swap if current element has a lower probe distance. + d := idx.dist(e.hash, pos) + if d < dist { + hash, e.hash = e.hash, hash + offset, e.offset = e.offset, offset + id, e.id = e.id, id + dist = d + } + + // Move position forward. + pos = (pos + 1) & idx.mask + dist++ + } +} + +// lookupKey returns the key at the given offset in the memory-mapped file. +func (idx *index) lookupKey(offset int64) []byte { + data := idx.data[offset:] + n, sz := binary.Uvarint(data) + if sz == 0 { + return nil + } + return data[sz : sz+int(n)] +} + +func (idx *index) alloc(capacity uint64) { + idx.elems = make([]elem, capacity) + idx.threshold = (capacity * uint64(idx.loadFactor)) / 100 + idx.mask = uint64(capacity - 1) +} + +func (idx *index) dist(hash, i uint64) uint64 { + return (i + uint64(len(idx.elems)) - (hash & idx.mask)) & idx.mask +} + +type elem struct { + offset int64 + id uint64 + hash uint64 +} + +func (e *elem) reset() { + e.offset = 0 + e.id = 0 + e.hash = 0 +} + +func hashKey(key []byte) uint64 { + h := xxhash.Sum64(key) + if h == 0 { + h = 1 + } + return h +} + +func pow2(v uint64) uint64 { + for i := uint64(2); i < 1<<62; i *= 2 { + if i >= v { + return i + } + } + panic("unreachable") +} + +// TranslateFileReader implements a reader that continuously streams data from a store. +type TranslateFileReader struct { + ctx context.Context + store *TranslateFile + file *os.File + offset int64 + notify <-chan struct{} + + once sync.Once + closing chan struct{} +} + +// NewTranslateFileReader returns a new instance of TranslateFileReader. +func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { + return &TranslateFileReader{ + ctx: ctx, + store: store, + offset: offset, + notify: store.WriteNotify(), + closing: make(chan struct{}), + } +} + +// Open initializes the reader. +func (r *TranslateFileReader) Open() (err error) { + if r.file, err = os.Open(r.store.Path); err != nil { + return err + } + return nil +} + +// Close closes the underlying file reader. +func (r *TranslateFileReader) Close() error { + r.once.Do(func() { close(r.closing) }) + + if r.file != nil { + return r.file.Close() + } + return nil +} + +// Read reads the next section of the available data to p. This should always +// read from the start of an entry and read n bytes to the end of another entry. +func (r *TranslateFileReader) Read(p []byte) (n int, err error) { + for { + // Obtain notification channel before we check for new data. + notify := r.store.WriteNotify() + + // Exit if we can read one or more valid entries or we receive an error. + if n, err = r.read(p); n > 0 || err != nil { + return n, err + } + + // Wait for new data or close. + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.closing: + return 0, ErrTranslateStoreReaderClosed + case <-r.store.Closing(): + return 0, ErrTranslateStoreClosed + case <-notify: + continue + } + } +} + +// read writes the bytes for zero or more valid entries to p. +func (r *TranslateFileReader) read(p []byte) (n int, err error) { + sz := r.store.Size() + + // Exit if there is no new data. + if sz < r.offset { + return 0, fmt.Errorf("pilosa: translate store reader past file size: sz=%d off=%d", sz, r.offset) + } else if sz == r.offset { + return 0, nil + } + + // Shorten buffer to maximum read size. + if max := sz - r.offset; int64(len(p)) > max { + p = p[:max] + } + + // Read data from file at offset. + // Limit the number of bytes read to only whole entries. + n, err = r.file.ReadAt(p, r.offset) + n = ValidLogEntriesLen(p[:n]) + r.offset += int64(n) + return n, err +} + +// Copied & modified from encoding/binary. +func UvarintSize(x uint64) (i int) { + for x >= 0x80 { + x >>= 7 + i++ + } + return i + 1 +} + +func hexdump(b []byte) { os.Stderr.Write([]byte(hex.Dump(b))) } diff --git a/translate_test.go b/translate_test.go new file mode 100644 index 000000000..c73853d69 --- /dev/null +++ b/translate_test.go @@ -0,0 +1,565 @@ +package pilosa_test + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pilosa/pilosa" +) + +func TestTranslateFile_TranslateColumn(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateColumn_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateColumnsToUint64("IDX0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_TranslateRow(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateRow_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_Reader(t *testing.T) { + t.Run("NoOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + rc, err := s.Reader(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b. + var entry pilosa.LogEntry + if n, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if n != 14 { + t.Fatalf("unexpected n: %d", n) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{1}, + Keys: [][]byte{[]byte("foo")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Read second entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + + // Write new entry. + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil { + t.Fatal(err) + } + + // Read new entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{2}, + Keys: [][]byte{[]byte("xyz")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Close reader and ensure it returns EOF. + if err := rc.Close(); err != nil { + t.Fatal(err) + } else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("WithOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Start offset after the first entry. + rc, err := s.Reader(context.Background(), 14) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // This should be the second entry. + var entry pilosa.LogEntry + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { + // Create a primary store that accepts writes. + primary := MustOpenTranslateFile() + defer primary.MustClose() + + // Create a replica that accepts writes from primary. + replica := NewTranslateFile() + replica.PrimaryTranslateStore = primary + if err := replica.Open(); err != nil { + t.Fatal(err) + } + defer replica.MustClose() + + // Write to the primary. + if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until writes appear. + if err := retryFor(2*time.Second, func() error { + // Verify that replica have received writes. + if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil { + return err + } else if value != "foo" { + return fmt.Errorf("unexpected column 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + return err + } else if value != "bar" { + return fmt.Errorf("unexpected row 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected row 2 value: %s", value) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect primary store & write more values. + if err := primary.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected column 2 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect replica store & write more values. + if err := replica.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil { + return err + } else if value != "foobar" { + return fmt.Errorf("unexpected column 3 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/batchSize) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} + +type TranslateFile struct { + *pilosa.TranslateFile +} + +func NewTranslateFile() *TranslateFile { + f, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } + f.Close() + + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s.Path = f.Name() + return s +} + +func MustOpenTranslateFile() *TranslateFile { + s := NewTranslateFile() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +func (s *TranslateFile) Close() error { + defer os.Remove(s.Path) + return s.TranslateFile.Close() +} + +func (s *TranslateFile) MustClose() { + if err := s.Close(); err != nil { + panic(err) + } +} + +// Reopen closes the store and opens a new instance of it for the same path. +func (s *TranslateFile) Reopen() error { + prev := s.TranslateFile + if err := s.TranslateFile.Close(); err != nil { + return err + } + + s.TranslateFile = pilosa.NewTranslateFile() + s.Path = prev.Path + s.PrimaryTranslateStore = prev.PrimaryTranslateStore + if err := s.Open(); err != nil { + return err + } + return nil +} + +// retryFor executes fn every 100ms until d time passes or until fn return nil. +func retryFor(d time.Duration, fn func() error) (err error) { + timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond) + defer timer.Stop() + defer ticker.Stop() + + for { + if err = fn(); err == nil { + return nil + } + + select { + case <-timer.C: + return err + case <-ticker.C: + } + } +}