From 56803e6632c18e2685cbca377209e1017e2faa5f Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Thu, 3 Sep 2020 12:00:19 -0500 Subject: [PATCH] pilosa/cmd/translatorchk checksums and summarizes key counts from column key translators. opens ~/.pilosa/index/_keys boltdbs and hashes the StringKey->ID mappings. --- api.go | 1 + boltdb/translate.go | 41 +++++++++++++++++ boltdb/translate_test.go | 51 +++++++++++++++++++++ cmd/translatorchk/translatorchk.go | 73 ++++++++++++++++++++++++++++++ index.go | 54 ++++++++++++++++++++++ mock/translator.go | 4 ++ translate.go | 21 +++++++++ 7 files changed, 245 insertions(+) create mode 100644 cmd/translatorchk/translatorchk.go diff --git a/api.go b/api.go index 773c73ae4..5edab36ee 100644 --- a/api.go +++ b/api.go @@ -1870,6 +1870,7 @@ func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error) } // TranslateIndexDB is an internal function to load the index keys database +// rd is a boltdb file. func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitionID int, rd io.Reader) error { idx := api.holder.Index(indexName) store := idx.TranslateStore(partitionID) diff --git a/boltdb/translate.go b/boltdb/translate.go index 4977c7670..0886821be 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -26,6 +26,7 @@ import ( "github.com/boltdb/bolt" "github.com/pilosa/pilosa/v2" "github.com/pkg/errors" + "github.com/zeebo/blake3" ) var ( @@ -480,3 +481,43 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string { } return string(boltKey) } + +func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { + sum = &pilosa.TranslatorSummary{} + hasher := blake3.New() + + err = s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketKeys) + if bkt == nil { + panic("bucketKeys not found") + } + + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + input := append(k, v...) + _, _ = hasher.Write(input) + sum.KeyCount++ + } + + bkt = tx.Bucket(bucketIDs) + if bkt == nil { + panic("bucketIDs not found") + } + + cur = bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + input := append(k, v...) + _, _ = hasher.Write(input) + sum.IDCount++ + } + + return nil + }) + if err != nil { + return nil, err + } + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + sum.Checksum = buf[:] + return sum, nil +} diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 73b4e5345..c6e1fd5b9 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -482,3 +482,54 @@ func MustCloseTranslateStore(s *boltdb.TranslateStore) { panic(err) } } + +func TestCryptoHashPerKey(t *testing.T) { + s := MustOpenNewTranslateStore() + defer MustCloseTranslateStore(s) + + // hash one translation + + expect := map[int][]byte{ + 1: []byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12}, + 2: []byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0}, + 3: []byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9}, + } + for n := 1; n < 4; n++ { + var batch0 []string + for i := 0; i < n; i++ { + batch0 = append(batch0, fmt.Sprintf("key%d", i)) + } + + // Populate the store with the keys in batch0. + batch0IDs, err := s.TranslateKeys(batch0, true) + _ = batch0IDs + if err != nil { + t.Fatal(err) + } + + // done with setup + + sum, err := s.ComputeTranslatorSummary() + if err != nil { + panic(err) + } + nkey := sum.KeyCount + nid := sum.IDCount + observedChecksum := sum.Checksum + if nkey != n { + panic("wrong key count") + } + if nkey != nid { + panic("key count should match id count") + } + + // shardwidth 22 has different hashes, of course. + if pilosa.ShardWidth == 20 { + expectedChecksum := expect[n] + if !bytes.Equal(observedChecksum, expectedChecksum) { + panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum)) + } + } + } + +} diff --git a/cmd/translatorchk/translatorchk.go b/cmd/translatorchk/translatorchk.go new file mode 100644 index 000000000..1fc036ff5 --- /dev/null +++ b/cmd/translatorchk/translatorchk.go @@ -0,0 +1,73 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "log" + "os" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/zeebo/blake3" +) + +// translatorchk : read boltdb files and print checksums and counts on the keys. + +func main() { + + var dir string + home := os.Getenv("HOME") + flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read") + flag.Parse() + + fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir) + + holder := pilosa.NewHolder(256) + holder.Path = dir + holder.OpenTranslateStore = boltdb.OpenTranslateStore + + err := holder.Open() + + if err != nil { + log.Fatal(err) + } + + fmt.Printf("\ncalculating checksums on data from dir '%v'...\n", dir) + + final := pilosa.NewAllTranslatorSummary() + const verbose = true + for _, idx := range holder.Indexes() { + asum, err := idx.ComputeTranslatorSummary(verbose) + if err != nil { + log.Fatal(err) + } + final.Merge(asum) + } + final.Sort() + + hasher := blake3.New() + fmt.Printf("\nsummary of %v:\n", dir) + for _, sum := range final.Sums { + //fmt.Printf("index: %v partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount) + _, _ = hasher.Write(sum.Checksum) + } + + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + + fmt.Printf("all-checksum = blake3-%x\n", buf) +} diff --git a/index.go b/index.go index 13eab7f85..5b3fc3c9c 100644 --- a/index.go +++ b/index.go @@ -768,3 +768,57 @@ func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []u } return } + +type AllTranslatorSummary struct { + Sums []*TranslatorSummary +} + +func NewAllTranslatorSummary() *AllTranslatorSummary { + return &AllTranslatorSummary{} +} +func (ats *AllTranslatorSummary) Merge(b *AllTranslatorSummary) { + ats.Sums = append(ats.Sums, b.Sums...) +} + +func (ats *AllTranslatorSummary) Sort() { + // return sorted by index then PartitionID + sort.Slice(ats.Sums, func(i, j int) bool { + a := ats.Sums[i] + b := ats.Sums[j] + if a.Index < b.Index { + return true + } + if a.Index > b.Index { + return false + } + // INVAR: a.Index == b.Index + return a.PartitionID < b.PartitionID + }) +} + +// sums is only guaranteed to be sorted by (index, PartitionID) if err returns nil +func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) { + i.mu.RLock() // avoid race with Index.Close() doing i.translateStores = make(map[int]TranslateStore) + defer i.mu.RUnlock() + + ats = &AllTranslatorSummary{} + + for partitionID, store := range i.translateStores { + sum, err := store.ComputeTranslatorSummary() + if err != nil { + return ats, err + } + if sum == nil { + // probably one of the Noop stores + continue + } + sum.PartitionID = partitionID + sum.Index = i.Name() + if verbose { + fmt.Printf("index: %v paritionID: %03v blake3-%x keyN: %10v idN: %10v\n", i.name, partitionID, sum.Checksum, sum.KeyCount, sum.IDCount) + } + ats.Sums = append(ats.Sums, sum) + } + + return +} diff --git a/mock/translator.go b/mock/translator.go index 8a28b504c..e577baf12 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -37,6 +37,10 @@ type TranslateStore struct { EntryReaderFunc func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) } +func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { + return +} + func (s *TranslateStore) Close() error { return s.CloseFunc() } diff --git a/translate.go b/translate.go index 4efa26129..b2864c248 100644 --- a/translate.go +++ b/translate.go @@ -89,6 +89,24 @@ type TranslateStore interface { // It should read from the reader and replace the data store with // the read payload. ReadFrom(io.Reader) (int64, error) + + ComputeTranslatorSummary() (sum *TranslatorSummary, err error) +} + +// TranslatorSummary is returned, for example from the boltdb string key translators, +// by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. +type TranslatorSummary struct { + PartitionID int + Index string + + // Checksum has a blake3 crypto hash of all the keys->ID and all the ID->key mappings + Checksum []byte + + // KeyCount has the number of Key->ID mappings + KeyCount int + + // IDCount has the number of ID->Key mappings + IDCount int } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. @@ -289,6 +307,9 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } +func (s *InMemTranslateStore) ComputeTranslatorSummary() (sum *TranslatorSummary, err error) { + panic("TODO") +} func (s *InMemTranslateStore) Close() error { return nil }