From f3f11f4a448452275b851b37952e096162a0591e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Tue, 18 Feb 2020 12:54:24 +0100 Subject: [PATCH] Let translate keys as empty strings (#120) --- boltdb/translate.go | 61 +++++++++++++++++++++++++++++++--------- boltdb/translate_test.go | 11 ++++++++ cache.go | 2 +- docs/data-model.md | 12 ++++++++ server/handler_test.go | 2 +- server/server_test.go | 2 +- translate.go | 8 ++++++ 7 files changed, 81 insertions(+), 17 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index 31d7364f4..a607d494f 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -11,9 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package boltdb import ( + "bytes" "context" "os" "path/filepath" @@ -45,6 +47,14 @@ func OpenTranslateStore(path, index, field string, partitionID, partitionN int) var _ pilosa.TranslateStore = &TranslateStore{} // TranslateStore is an on-disk storage engine for translating string-to-uint64 values. +// An empty string will be converted into the sentinel byte slice: +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore struct { mu sync.RWMutex db *bolt.DB @@ -149,7 +159,7 @@ func (s *TranslateStore) Size() int64 { func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { // Find id by key under read lock. if err := s.db.View(func(tx *bolt.Tx) error { - id = findIDByKey(tx.Bucket([]byte("keys")), key) + id, _ = findIDByKey(tx.Bucket([]byte("keys")), key) return nil }); err != nil { return 0, err @@ -165,14 +175,16 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) { var written bool if err := s.db.Update(func(tx *bolt.Tx) (err error) { bkt := tx.Bucket([]byte("keys")) - if id = findIDByKey(bkt, key); id != 0 { + + var boltKey []byte + if id, boltKey = findIDByKey(bkt, key); id != 0 { return nil } id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) - if err := bkt.Put([]byte(key), u64tob(id)); err != nil { + if err := bkt.Put(boltKey, u64tob(id)); err != nil { return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil { + } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil { return err } written = true @@ -203,7 +215,7 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) { if err := s.db.View(func(tx *bolt.Tx) error { bkt := tx.Bucket([]byte("keys")) for i, key := range keys { - if id := findIDByKey(bkt, key); id != 0 { + if id, _ := findIDByKey(bkt, key); id != 0 { ids[i] = id found++ } @@ -228,14 +240,15 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) { continue } - if ids[i] = findIDByKey(bkt, key); ids[i] != 0 { + var boltKey []byte + if ids[i], boltKey = findIDByKey(bkt, key); ids[i] != 0 { continue } ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) - if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil { + if err := bkt.Put(boltKey, u64tob(ids[i])); err != nil { return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil { + } else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), boltKey); err != nil { return err } written = true @@ -297,7 +310,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error { return nil } -// Reader returns a reader that streams the underlying data file. +// EntryReader returns a reader that streams the underlying data file. func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) { ctx, cancel := context.WithCancel(ctx) return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil @@ -404,13 +417,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { } } -func findIDByKey(bkt *bolt.Bucket, key string) uint64 { - if value := bkt.Get([]byte(key)); value != nil { - return btou64(value) +// emptyKey is a sentinel byte slice which stands for "" as a key. +var emptyKey = []byte{ + 0x00, 0x00, 0x00, + 0x4d, 0x54, 0x4d, 0x54, // MTMT + 0x00, + 0xc2, 0xa0, // NO-BREAK SPACE + 0x00, +} + +func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) { + var boltKey []byte + if key == "" { + boltKey = emptyKey + } else { + boltKey = []byte(key) } - return 0 + + if value := bkt.Get(boltKey); value != nil { + return btou64(value), boltKey + } + return 0, boltKey } func findKeyByID(bkt *bolt.Bucket, id uint64) string { - return string(bkt.Get(u64tob(id))) + boltKey := bkt.Get(u64tob(id)) + if bytes.Equal(boltKey, emptyKey) { + return "" + } + return string(boltKey) } diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 219885324..d3fb1b6e8 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -97,6 +97,10 @@ func TestTranslateStore_TranslateID(t *testing.T) { if err != nil { t.Fatal(err) } + id3, err := s.TranslateKey("") + if err != nil { + t.Fatal(err) + } // Ensure IDs can be translated back to keys. if key, err := s.TranslateID(id1); err != nil { @@ -110,6 +114,13 @@ func TestTranslateStore_TranslateID(t *testing.T) { } else if got, want := key, "bar"; got != want { t.Fatalf("TranslateID()=%s, want %s", got, want) } + + if key, err := s.TranslateID(id3); err != nil { + t.Fatal(err) + } else if got, want := key, ""; got != want { + t.Fatalf("TranslateID()=%s, want %s", got, want) + } + } func TestTranslateStore_TranslateIDs(t *testing.T) { diff --git a/cache.go b/cache.go index 6ff6f954a..9c0a7dc79 100644 --- a/cache.go +++ b/cache.go @@ -319,7 +319,7 @@ func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } // Pair holds an id/count pair. type Pair struct { ID uint64 `json:"id"` - Key string `json:"key,omitempty"` + Key string `json:"key"` Count uint64 `json:"count"` } diff --git a/docs/data-model.md b/docs/data-model.md index 7de27b575..85104bad3 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th #### Set Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records. +Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice: +```go +[]byte{ + 0x00, 0x00, 0x00, + 0x4d, 0x54, 0x4d, 0x54, // MTMT + 0x00, + 0xc2, 0xa0, // NO-BREAK SPACE + 0x00, +} +``` +(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte). +In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`). ``` request curl localhost:10101/index/repository/field/info \ diff --git a/server/handler_test.go b/server/handler_test.go index 50aef2764..a24373ba1 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -492,7 +492,7 @@ func TestHandler_Endpoints(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[[{"id":30,"key":"","count":3},{"id":31,"key":"","count":1}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) } }) diff --git a/server/server_test.go b/server/server_test.go index fe8997c78..b3c82a142 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -371,7 +371,7 @@ func TestMain_RecalculateHashes(t *testing.T) { t.Fatalf("recalculating caches: %v", err) } - target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}` + target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}` // Run a TopN query on all nodes. The result should be the same as the target. for _, m := range cluster { diff --git a/translate.go b/translate.go index faf456054..b5f469f2f 100644 --- a/translate.go +++ b/translate.go @@ -39,6 +39,14 @@ var ( ) // TranslateStore is the storage for translation string-to-uint64 values. +// For BoltDB implementation an empty string will be converted into the sentinel byte slice: +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore interface { io.Closer