diff --git a/bluegreentx.go b/bluegreentx.go index bfb75277f..32f51da4d 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -18,9 +18,7 @@ import ( "bytes" "fmt" "io" - "math" "reflect" - "sort" "sync" "github.com/pilosa/pilosa/v2/roaring" @@ -909,53 +907,6 @@ func (c *blueGreenTx) Sn() int64 { return bsn } -func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - // doesn't change state, so we don't really need see() call here. And we don't have a single shard for it. - //c.checker.see(index, field, view, shard) // don't have shard. - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, math.MaxUint64) - AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack()) - panic(r) - } - }() - slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath) - slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - - // sort order may be different, and that's ok. - cpa := append([]uint64{}, slcA...) - cpb := append([]uint64{}, slcB...) - sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] }) - sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] }) - - if !reflect.DeepEqual(cpa, cpb) { - // report the first difference - ma := make(map[uint64]bool) - for _, ka := range slcA { - ma[ka] = true - } - for _, kb := range slcB { - if !ma[kb] { - //vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb) - c.Dump(c.short, math.MaxUint64) - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)) - } - delete(ma, kb) - } - if len(ma) != 0 { - for firstDifference := range ma { - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb)) - } - } - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb)) - } - } - return slcB, errB -} - // MultiReaderB is returned by RoaringBitmapReader. It verifies // that identical byte streams are read from its two members. type MultiReaderB struct { diff --git a/bolt.go b/bolt.go index 4d7830a44..3c186f3cf 100644 --- a/bolt.go +++ b/bolt.go @@ -728,33 +728,6 @@ func (tx *BoltTx) Contains(index, field, view string, shard uint64, key uint64) return exists, err } -func (tx *BoltTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - - prefix := txkey.AllShardPrefix(index, field, view) - - bi := NewBoltIterator(tx, prefix) - defer bi.Close() - - lastShard := uint64(0) - firstDone := false - for bi.Next() { - shard := txkey.ShardFromKey(bi.lastKey) - if firstDone { - if shard != lastShard { - sliceOfShards = append(sliceOfShards, shard) - } - lastShard = shard - } else { - // first time - lastShard = shard - firstDone = true - sliceOfShards = append(sliceOfShards, shard) - } - - } - return -} - // key is the container key for the first roaring Container // roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a diff --git a/bolt_test.go b/bolt_test.go index 54fefc6f8..9a6961d29 100644 --- a/bolt_test.go +++ b/bolt_test.go @@ -1239,29 +1239,6 @@ func TestBolt_DeleteIndex_over100k(t *testing.T) { } } -func TestBolt_SliceOfShards(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - shards := []uint64{0, 1, 2, 3, 1000001, 2000001} - putme := uint64(179) - for _, shard := range shards { - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - } - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - slc, err := tx.SliceOfShards(index, field, view, "") - panicOn(err) - for i := range shards { - if shards[i] != slc[i] { - panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i])) - } - } -} - func TestBolt_HasData(t *testing.T) { db, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") diff --git a/catcher.go b/catcher.go index be1095783..7ff8b6fd8 100644 --- a/catcher.go +++ b/catcher.go @@ -294,15 +294,6 @@ func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, func (c *catcherTx) Type() string { return c.b.Type() } -func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack()) - panic(r) - } - }() - return c.b.SliceOfShards(index, field, view, optionalViewPath) -} func (c *catcherTx) Group() *TxGroup { return c.b.Group() diff --git a/rbf.go b/rbf.go index 7191639f4..ddddd6316 100644 --- a/rbf.go +++ b/rbf.go @@ -29,7 +29,7 @@ import ( "github.com/pilosa/pilosa/v2/rbf" rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/txkey" + txkey "github.com/pilosa/pilosa/v2/short_txkey" "github.com/pkg/errors" ) @@ -399,26 +399,6 @@ func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fr return ioutil.NopCloser(&buf), sz, err } -func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - - prefix := string(txkey.AllShardPrefix(index, field, view)) - - names, err := tx.tx.BitmapNames() - if err != nil { - return nil, err - } - - // Iterate over shard names and collect shards from matching field/view prefix. - for _, name := range names { - if !strings.HasPrefix(name, prefix) { - continue - } - shard := txkey.ShardFromPrefix([]byte(name)) - sliceOfShards = append(sliceOfShards, shard) - } - return sliceOfShards, nil -} - func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { b, err := tx.RoaringBitmap(index, field, view, shard) panicOn(err) diff --git a/rbf/ingest_test.go b/rbf/ingest_test.go index 89456b678..3ba99e84d 100644 --- a/rbf/ingest_test.go +++ b/rbf/ingest_test.go @@ -27,7 +27,7 @@ import ( "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/txkey" + txkey "github.com/pilosa/pilosa/v2/short_txkey" ) func rbfName(index, field, view string, shard uint64) string { diff --git a/rbf/tx.go b/rbf/tx.go index f28760a97..b4f1c7306 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -25,7 +25,7 @@ import ( "github.com/benbjohnson/immutable" "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/txkey" + txkey "github.com/pilosa/pilosa/v2/short_txkey" ) var _ = txkey.ToString diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 77f69b40d..23ee02488 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -22,7 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/v2/rbf" - "github.com/pilosa/pilosa/v2/txkey" + txkey "github.com/pilosa/pilosa/v2/short_txkey" ) func TestTx_CommitRollback(t *testing.T) { diff --git a/rbf/util.go b/rbf/util.go index 83e4f2ec7..dc829e3e1 100644 --- a/rbf/util.go +++ b/rbf/util.go @@ -18,7 +18,7 @@ import ( "io" "strings" - "github.com/pilosa/pilosa/v2/txkey" + txkey "github.com/pilosa/pilosa/v2/short_txkey" ) func (tx *Tx) dumpAllPages(showLeaves bool) error { diff --git a/short_txkey/txkey.go b/short_txkey/txkey.go new file mode 100644 index 000000000..2c5af85ef --- /dev/null +++ b/short_txkey/txkey.go @@ -0,0 +1,192 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package txkey consolidates in one place the use of keys to index into our +// various storage/txn back-ends. The short_txkey version omits the +// index and shard, since these are implicitly part of our database-per-shard +// in an index scheme. In other words, every database is only in exactly +// one shard of one index already. There is no need to repeat the index +// and shard in these keys. +package short_txkey + +import ( + "encoding/binary" + "fmt" +) + +// Key produces the bytes that we use as a key to query the storage/tx engine. +// The roaringContainerKey argument to Key() is a container key into a roaring Container. +// The return value from Key() is constructed as follows: +// +// ~field;view'. Keys always end with '#'. +// Keys always contain exactly one each of ';' and '<', in that order. +// The field is between the '~' and the ';'. It must be at least 1 byte long. +// The view is between the ';' and the '<'. It must be at least 1 byte long. +// The ckey is the 8 bytes between the '<' and the '#'. +// The Prefix of a key ends at, and includes, the '<'. It is at least 13 bytes long. +// The index, field, and view are not allowed to contain these reserved bytes: +// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} +// +// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable +// base-64 encoding. +// +// The shortest possible key is 14 bytes. It would be laid out like this: +// ~f;v<12345678# +// 12345678901234 +// +// keys starting with '~' are regular value keys. +// keys starting with '>' are symlink keys. +// +// NB must be kept in sync with Prefix() and KeyExtractContainerKey(). +// +func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) { + + prefix := Prefix(index, field, view, shard) + + var ckey [9]byte + binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey) + ckey[8] = byte('#') + return append(prefix, ckey[:]...) +} + +// KeyAndPrefix returns the equivalent of Key() and Prefix() calls. +func KeyAndPrefix(index, field, view string, shard, roaringContainerKey uint64) (key, prefix []byte) { + prefix = Prefix(index, field, view, shard) + + var ckey [9]byte + binary.BigEndian.PutUint64(ckey[:8], roaringContainerKey) + ckey[8] = byte('#') + key = append(prefix, ckey[:]...) + return +} + +var _ = KeyAndPrefix // keep linter happy + +func MustValidateKey(bkey []byte) { + n := len(bkey) + if n < 14 { + panic(fmt.Sprintf("bkey too short, must have at least 14 bytes: '%v'", string(bkey))) + } + typ := bkey[0] + if typ != '~' && typ != '>' { + panic(fmt.Sprintf("bkey did not start with '~' for value nor '>' for symlink: '%v'", string(bkey))) + } + if bkey[n-10] != '<' { + panic(fmt.Sprintf("bkey did not have '<' at 9 bytes from the end: '%v'", string(bkey))) + } + if bkey[n-1] != '#' { + panic(fmt.Sprintf("bkey did not end in '#': '%v'", string(bkey))) + } +} + +// KeyExtractContainerKey extracts the containerKey from bkey. +// key example: field;view