// Copyright 2020 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package txpath consolidates in one place the use of keys to index into our // various storage/txn back-ends. Databases badgerDB and rbfDB both use it, // so that debug Dumps are comparable. package txpath import ( "bytes" "fmt" "strconv" ) // Key produces the bytes that we use as a key to query the storage/tx engine. // The roaringContainerKey argument is a container key into a roaring Container. // Output examples: // // "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@00000000000000000000" // smallest container-key // "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615" // largest container-key (math.MaxUint64) // // NB must be kept in sync with Prefix() and KeyExtractContainerKey(). // func Key(index, field, view string, shard uint64, roaringContainerKey uint64) []byte { // The %020d which adds zero padding up to 20 runes is required to // allow the textual sort to accurately // reflect a numeric sort order. This is because, as a string, // math.MaxUint64 is 20 bytes long. // Example of such a Key with a container-key that is math.MaxUint64: // ...........................................12345678901234567890 // idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 prefix := Prefix(index, field, view, shard) ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey)) bkey := append(prefix, ckey...) MustValidateKey(bkey) return bkey } var ckeyPartExpected = []byte(";ckey@") // MustValidatekey will panic on a bad Key with an informative message. func MustValidateKey(bkey []byte) { n := len(bkey) if n < 56 { panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey))) } beforeCkey := bkey[n-26 : n-20] if !bytes.Equal(beforeCkey, ckeyPartExpected) { panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey))) } } func ShardFromKey(bkey []byte) (shard uint64) { MustValidateKey(bkey) n := len(bkey) // idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1 by := bkey[:n-27] beg := bytes.LastIndex(by, []byte("'")) if beg == -1 { panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey))) } parseMe := string(by[beg+1:]) shard, err := strconv.ParseUint(parseMe, 10, 64) if err != nil { panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err)) } return shard } func ShardFromPrefix(prefix []byte) (shard uint64) { n := len(prefix) // idx:'i';fld:'f';vw:'standard';shd:'1';ckey@ -> idx:'i';fld:'f';vw:'standard';shd:'1 by := prefix[:n-7] beg := bytes.LastIndex(by, []byte("'")) if beg == -1 { panic(fmt.Sprintf("bad prefix='%v' did not have single quote to being shard decoding", string(prefix))) } parseMe := string(by[beg+1:]) shard, err := strconv.ParseUint(parseMe, 10, 64) if err != nil { panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err)) } return shard } // KeyAndPrefix returns the equivalent of Key() and Prefix() calls. func KeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) { prefix = Prefix(index, field, view, shard) ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey)) bkey := append(prefix, ckey...) MustValidateKey(bkey) return bkey, prefix } var _ = KeyAndPrefix // keep linter happy // KeyExtractContainerKey extracts the containerKey from bkey. func KeyExtractContainerKey(bkey []byte) (containerKey uint64) { MustValidateKey(bkey) // The zero padding means that the container-key is always the last 20 bytes of the bkey. // // Be sure to catch the problematic case of a user passing in only a prefix. A prefix // ends in 'key@' rather than a full key that has 'key@00000000000000000001' (for example) // at the end. The ParseUint call below will fail in that case. n := len(bkey) if n < 20 { panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', too short!", string(bkey))) } last := bkey[n-20:] // Key() and Prefix() always return more than 20 rune []byte. var err error containerKey, err = strconv.ParseUint(string(last), 10, 64) // has to be the container key if err != nil { panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', could not convert last 20 bytes ('%v') to a unit64: '%v'", string(bkey), string(last), err)) } return } func AllShardPrefix(index, field, view string) []byte { return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view)) } // Prefix returns everything from Key up to and // including the '@' fune in a Key. The prefix excludes the roaring container key itself. // NB must be kept in sync with Key() and KeyExtractContainerKey(). func Prefix(index, field, view string, shard uint64) []byte { return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard)) } // IndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to // remove all storage associated with one index. // // The full name of the index must be provided, no partial index names will work. // // The provided key is terminated by `';` and so DeleteIndex("i") will not delete the index "i2". // func IndexOnlyPrefix(indexName string) []byte { return []byte(fmt.Sprintf("idx:'%v';", indexName)) } // same for deleting a whole field. func FieldPrefix(index, field string) []byte { return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field)) }