scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 04:47:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Let translate keys as empty strings (#120)

Browse source
This commit is contained in:
Kuba Podgórski 2020-02-18 12:54:24 +01:00 • committed by GitHub
parent 57eb741c24
commit f3f11f4a44
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 81 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
boltdb/translate.go
View file

@ -11,9 +11,11 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"context"
"os"
"path/filepath"
@ -45,6 +47,14 @@ func OpenTranslateStore(path, index, field string, partitionID, partitionN int)
var _ pilosa.TranslateStore = &TranslateStore{}
// TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
// An empty string will be converted into the sentinel byte slice:
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
type TranslateStore struct {
mu sync.RWMutex
db *bolt.DB
@ -149,7 +159,7 @@ func (s *TranslateStore) Size() int64 {
func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
// Find id by key under read lock.
if err := s.db.View(func(tx *bolt.Tx) error {
id = findIDByKey(tx.Bucket([]byte("keys")), key)
id, _ = findIDByKey(tx.Bucket([]byte("keys")), key)
return nil
}); err != nil {
return 0, err
@ -165,14 +175,16 @@ func (s *TranslateStore) TranslateKey(key string) (id uint64, _ error) {
var written bool
if err := s.db.Update(func(tx *bolt.Tx) (err error) {
bkt := tx.Bucket([]byte("keys"))
if id = findIDByKey(bkt, key); id != 0 {
var boltKey []byte
if id, boltKey = findIDByKey(bkt, key); id != 0 {
return nil
}
id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
if err := bkt.Put([]byte(key), u64tob(id)); err != nil {
if err := bkt.Put(boltKey, u64tob(id)); err != nil {
return err
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), []byte(key)); err != nil {
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil {
return err
}
written = true
@ -203,7 +215,7 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
if err := s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket([]byte("keys"))
for i, key := range keys {
if id := findIDByKey(bkt, key); id != 0 {
if id, _ := findIDByKey(bkt, key); id != 0 {
ids[i] = id
found++
}
@ -228,14 +240,15 @@ func (s *TranslateStore) TranslateKeys(keys []string) (ids []uint64, _ error) {
continue
}
if ids[i] = findIDByKey(bkt, key); ids[i] != 0 {
var boltKey []byte
if ids[i], boltKey = findIDByKey(bkt, key); ids[i] != 0 {
continue
}
ids[i] = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
if err := bkt.Put([]byte(key), u64tob(ids[i])); err != nil {
if err := bkt.Put(boltKey, u64tob(ids[i])); err != nil {
return err
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), []byte(key)); err != nil {
} else if err := tx.Bucket([]byte("ids")).Put(u64tob(ids[i]), boltKey); err != nil {
return err
}
written = true
@ -297,7 +310,7 @@ func (s *TranslateStore) ForceSet(id uint64, key string) error {
return nil
}
// Reader returns a reader that streams the underlying data file.
// EntryReader returns a reader that streams the underlying data file.
func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
ctx, cancel := context.WithCancel(ctx)
return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
@ -404,13 +417,33 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
}
}
func findIDByKey(bkt *bolt.Bucket, key string) uint64 {
if value := bkt.Get([]byte(key)); value != nil {
return btou64(value)
// emptyKey is a sentinel byte slice which stands for "" as a key.
var emptyKey = []byte{
0x00, 0x00, 0x00,
0x4d, 0x54, 0x4d, 0x54, // MTMT
0x00,
0xc2, 0xa0, // NO-BREAK SPACE
0x00,
}
func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
var boltKey []byte
if key == "" {
boltKey = emptyKey
} else {
boltKey = []byte(key)
}
return 0
if value := bkt.Get(boltKey); value != nil {
return btou64(value), boltKey
}
return 0, boltKey
}
func findKeyByID(bkt *bolt.Bucket, id uint64) string {
return string(bkt.Get(u64tob(id)))
boltKey := bkt.Get(u64tob(id))
if bytes.Equal(boltKey, emptyKey) {
return ""
}
return string(boltKey)
}

11
boltdb/translate_test.go
View file

@ -97,6 +97,10 @@ func TestTranslateStore_TranslateID(t *testing.T) {
if err != nil {
t.Fatal(err)
}
id3, err := s.TranslateKey("")
if err != nil {
t.Fatal(err)
}
// Ensure IDs can be translated back to keys.
if key, err := s.TranslateID(id1); err != nil {
@ -110,6 +114,13 @@ func TestTranslateStore_TranslateID(t *testing.T) {
} else if got, want := key, "bar"; got != want {
t.Fatalf("TranslateID()=%s, want %s", got, want)
}
if key, err := s.TranslateID(id3); err != nil {
t.Fatal(err)
} else if got, want := key, ""; got != want {
t.Fatalf("TranslateID()=%s, want %s", got, want)
}
}
func TestTranslateStore_TranslateIDs(t *testing.T) {

2
cache.go
View file

@ -319,7 +319,7 @@ func (p bitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
// Pair holds an id/count pair.
type Pair struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Key string `json:"key"`
Count uint64 `json:"count"`
}

12
docs/data-model.md
View file

@ -121,6 +121,18 @@ Upon creation, fields are configured to be of a certain type. Pilosa supports th
#### Set
Set is the default field type in Pilosa. Set fields represent a standard, binary matrix of rows and columns where each row key represents a possible field value. The following example creates a `set` field called "info" with a ranked cache containing up to 100,000 records.
Row and/or column key can be a string literal (e.g. "value"). This mapping is also stored in a separate BoltDB data structure. Becauase BoltDB does not allow to have empty strings as keys, in pilosa we translate an empty string key into sentinel byte slice:
```go
[]byte{
0x00, 0x00, 0x00,
0x4d, 0x54, 0x4d, 0x54, // MTMT
0x00,
0xc2, 0xa0, // NO-BREAK SPACE
0x00,
}
```
(where the first three bytes are _zero_ bytes, next four bytes stands for `MTMT` literal and the rest four bytes represent NBSP prefixed and suffixed with _zero_ byte).
In reverse translation, if we get from BoltDB the sentinel key, pilosa will rewrite it into an empty string (`""`).
``` request
curl localhost:10101/index/repository/field/info \

2
server/handler_test.go
View file

@ -492,7 +492,7 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[[{"id":30,"key":"","count":3},{"id":31,"key":"","count":1}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})

2
server/server_test.go
View file

@ -371,7 +371,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
t.Fatalf("recalculating caches: %v", err)
}
target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}`
target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}`
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {

8
translate.go
View file

@ -39,6 +39,14 @@ var (
)
// TranslateStore is the storage for translation string-to-uint64 values.
// For BoltDB implementation an empty string will be converted into the sentinel byte slice:
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
type TranslateStore interface {
io.Closer