diff --git a/boltdb/translate.go b/boltdb/translate.go index d6c80a115..d37dae0d9 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -180,33 +180,6 @@ func (s *TranslateStore) TranslateKey(key string, writable bool) (uint64, error) if len(ids) == 0 { return 0, ErrTranslateKeyNotFound } - - // Find or create id under write lock. - var written bool - if err := s.db.Update(func(tx *bolt.Tx) (err error) { - bkt := tx.Bucket([]byte("keys")) - - var boltKey []byte - var id uint64 - if id, boltKey = findIDByKey(bkt, key); id != 0 { - return nil - } - - id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) - if err := bkt.Put(boltKey, u64tob(id)); err != nil { - return err - } else if err := tx.Bucket([]byte("ids")).Put(u64tob(id), boltKey); err != nil { - return err - } - written = true - return nil - }); err != nil { - return 0, err - } - if len(ids) == 0 { - // this should not happen - return 0, ErrTranslateKeyNotFound - } return ids[0], nil } @@ -248,9 +221,7 @@ func (s *TranslateStore) translateKeys(keys []string, writable bool) ([]uint64, } return nil, nil } - if !writable { - return nil, pilosa.ErrTranslatingKeyNotFound - } + // Find or create ids under write lock if any keys were not found. var written bool if err := s.db.Update(func(tx *bolt.Tx) (err error) { @@ -303,9 +274,11 @@ func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) { } defer func() { _ = tx.Rollback() }() + bucket := tx.Bucket(bucketIDs) + keys := make([]string, len(ids)) for i, id := range ids { - keys[i] = findKeyByID(tx.Bucket(bucketIDs), id) + keys[i] = findKeyByID(bucket, id) } return keys, nil } diff --git a/go.mod b/go.mod index 9d9ecd366..5bc2ba126 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,11 @@ require ( github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/zeebo/blake3 v0.0.4 go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.28.0 modernc.org/mathutil v1.0.0 diff --git a/go.sum b/go.sum index 9da5feace..9448f25e2 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,11 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI= +github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34= +github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -210,6 +215,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= diff --git a/translator_test.go b/translator_test.go index 8d71cc729..1229585c6 100644 --- a/translator_test.go +++ b/translator_test.go @@ -289,138 +289,12 @@ func TestTranslation_Reset(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := node0.API.TranslateKeys(ctx, bytes.NewReader(reqBody), true); err != nil { + if _, err := node0.API.TranslateKeys(ctx, bytes.NewReader(reqBody)); err != nil { t.Fatal(err) } }) } -func TestTranslation_KeyNotFound(t *testing.T) { - c := test.MustRunCluster(t, 4, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerIsCoordinator(true), - pilosa.OptServerNodeID("node0"), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerIsCoordinator(false), - pilosa.OptServerNodeID("node1"), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerIsCoordinator(false), - pilosa.OptServerNodeID("node2"), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerIsCoordinator(false), - pilosa.OptServerNodeID("node3"), - pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - ) - defer c.Close() - - node0 := c.GetNode(0) - node1 := c.GetNode(1) - node2 := c.GetNode(2) - node3 := c.GetNode(3) - - ctx := context.Background() - idx, fld := "i", "f" - // Create an index with keys. - if _, err := node0.API.CreateIndex(ctx, idx, pilosa.IndexOptions{Keys: true}); err != nil { - t.Fatal(err) - } - // Create an index with keys. - if _, err := node0.API.CreateField(ctx, idx, fld, pilosa.OptFieldKeys()); err != nil { - t.Fatal(err) - } - - // write a new key and get id - req, err := node0.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ - Index: idx, - Field: fld, - Keys: []string{"k1"}, - NotWritable: false, - }) - if err != nil { - t.Fatal(err) - } - - if buf, err := node0.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil { - t.Fatal(err) - } else { - var resp pilosa.TranslateKeysResponse - if err = node0.API.Serializer.Unmarshal(buf, &resp); err != nil { - t.Fatal(err) - } - id1 := resp.IDs[0] - - // read non-existing key - req, err = node3.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ - Index: idx, - Field: fld, - Keys: []string{"k2"}, - NotWritable: true, - }) - if err != nil { - t.Fatal(err) - } - if buf, err = node3.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil { - t.Fatal(err) - } - if err = node3.API.Serializer.Unmarshal(buf, &resp); err != nil { - t.Fatal(err) - } else if resp.IDs != nil { - t.Fatalf("TranslateKeys(%+v): expected: nil, got: %d", string(req), resp) - } - - req, err = node1.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ - Index: idx, - Keys: []string{"k2"}, - NotWritable: true, - }) - if err != nil { - t.Fatal(err) - } - if buf, err = node1.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil { - t.Fatal(err) - } - if err = node1.API.Serializer.Unmarshal(buf, &resp); err != nil { - t.Fatal(err) - } else if resp.IDs != nil { - t.Fatalf("TranslateKeys(%+v): expected: nil, got: %d", req, resp) - } - - req, err = node2.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ - Index: idx, - Field: fld, - Keys: []string{"k2", "k1"}, - NotWritable: false, - }) - if err != nil { - t.Fatal(err) - } - if buf, err = node2.API.TranslateKeys(ctx, bytes.NewReader(req)); err != nil { - t.Fatal(err) - } - if err = node2.API.Serializer.Unmarshal(buf, &resp); err != nil { - t.Fatal(err) - } - if resp.IDs[0] != id1+1 || resp.IDs[1] != id1 { - t.Fatalf("TranslateKeys(%+v): expected: %d,%d, got: %d,%d", req, id1+1, id1, resp.IDs[0], resp.IDs[1]) - } - } -} - func TestInMemTranslateStore_ReadKey(t *testing.T) { s := pilosa.NewInMemTranslateStore("IDX", "FLD", 0, pilosa.DefaultPartitionN)