pilosa/cmd/translatorchk checksums and summarizes key counts from column key translators.

opens ~/.pilosa/index/_keys boltdbs and hashes the StringKey->ID mappings.

api.go

@@ -1870,6 +1870,7 @@ func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error)
 }
 
 // TranslateIndexDB is an internal function to load the index keys database
+// rd is a boltdb file.
 func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitionID int, rd io.Reader) error {
 	idx := api.holder.Index(indexName)
 	store := idx.TranslateStore(partitionID)

boltdb/translate.go

@@ -26,6 +26,7 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pkg/errors"
+	"github.com/zeebo/blake3"
 )
 
 var (
@@ -480,3 +481,43 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string {
 	}
 	return string(boltKey)
 }
+
+func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) {
+	sum = &pilosa.TranslatorSummary{}
+	hasher := blake3.New()
+
+	err = s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(bucketKeys)
+		if bkt == nil {
+			panic("bucketKeys not found")
+		}
+
+		cur := bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			input := append(k, v...)
+			_, _ = hasher.Write(input)
+			sum.KeyCount++
+		}
+
+		bkt = tx.Bucket(bucketIDs)
+		if bkt == nil {
+			panic("bucketIDs not found")
+		}
+
+		cur = bkt.Cursor()
+		for k, v := cur.First(); k != nil; k, v = cur.Next() {
+			input := append(k, v...)
+			_, _ = hasher.Write(input)
+			sum.IDCount++
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+	sum.Checksum = buf[:]
+	return sum, nil
+}

boltdb/translate_test.go

@@ -482,3 +482,54 @@ func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 		panic(err)
 	}
 }
+
+func TestCryptoHashPerKey(t *testing.T) {
+	s := MustOpenNewTranslateStore()
+	defer MustCloseTranslateStore(s)
+
+	// hash one translation
+
+	expect := map[int][]byte{
+		1: []byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12},
+		2: []byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0},
+		3: []byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9},
+	}
+	for n := 1; n < 4; n++ {
+		var batch0 []string
+		for i := 0; i < n; i++ {
+			batch0 = append(batch0, fmt.Sprintf("key%d", i))
+		}
+
+		// Populate the store with the keys in batch0.
+		batch0IDs, err := s.TranslateKeys(batch0, true)
+		_ = batch0IDs
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// done with setup
+
+		sum, err := s.ComputeTranslatorSummary()
+		if err != nil {
+			panic(err)
+		}
+		nkey := sum.KeyCount
+		nid := sum.IDCount
+		observedChecksum := sum.Checksum
+		if nkey != n {
+			panic("wrong key count")
+		}
+		if nkey != nid {
+			panic("key count should match id count")
+		}
+
+		// shardwidth 22 has different hashes, of course.
+		if pilosa.ShardWidth == 20 {
+			expectedChecksum := expect[n]
+			if !bytes.Equal(observedChecksum, expectedChecksum) {
+				panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum))
+			}
+		}
+	}
+
+}

cmd/translatorchk/translatorchk.go

@@ -0,0 +1,73 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/boltdb"
+	"github.com/zeebo/blake3"
+)
+
+// translatorchk : read boltdb files and print checksums and counts on the keys.
+
+func main() {
+
+	var dir string
+	home := os.Getenv("HOME")
+	flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
+	flag.Parse()
+
+	fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
+
+	holder := pilosa.NewHolder(256)
+	holder.Path = dir
+	holder.OpenTranslateStore = boltdb.OpenTranslateStore
+
+	err := holder.Open()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("\ncalculating checksums on data from dir '%v'...\n", dir)
+
+	final := pilosa.NewAllTranslatorSummary()
+	const verbose = true
+	for _, idx := range holder.Indexes() {
+		asum, err := idx.ComputeTranslatorSummary(verbose)
+		if err != nil {
+			log.Fatal(err)
+		}
+		final.Merge(asum)
+	}
+	final.Sort()
+
+	hasher := blake3.New()
+	fmt.Printf("\nsummary of %v:\n", dir)
+	for _, sum := range final.Sums {
+		//fmt.Printf("index: %v  partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		_, _ = hasher.Write(sum.Checksum)
+	}
+
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	fmt.Printf("all-checksum = blake3-%x\n", buf)
+}

index.go

@@ -768,3 +768,57 @@ func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []u
 	}
 	return
 }
+
+type AllTranslatorSummary struct {
+	Sums []*TranslatorSummary
+}
+
+func NewAllTranslatorSummary() *AllTranslatorSummary {
+	return &AllTranslatorSummary{}
+}
+func (ats *AllTranslatorSummary) Merge(b *AllTranslatorSummary) {
+	ats.Sums = append(ats.Sums, b.Sums...)
+}
+
+func (ats *AllTranslatorSummary) Sort() {
+	// return sorted by index then PartitionID
+	sort.Slice(ats.Sums, func(i, j int) bool {
+		a := ats.Sums[i]
+		b := ats.Sums[j]
+		if a.Index < b.Index {
+			return true
+		}
+		if a.Index > b.Index {
+			return false
+		}
+		// INVAR: a.Index == b.Index
+		return a.PartitionID < b.PartitionID
+	})
+}
+
+// sums is only guaranteed to be sorted by (index, PartitionID) if err returns nil
+func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) {
+	i.mu.RLock() // avoid race with Index.Close() doing i.translateStores = make(map[int]TranslateStore)
+	defer i.mu.RUnlock()
+
+	ats = &AllTranslatorSummary{}
+
+	for partitionID, store := range i.translateStores {
+		sum, err := store.ComputeTranslatorSummary()
+		if err != nil {
+			return ats, err
+		}
+		if sum == nil {
+			// probably one of the Noop stores
+			continue
+		}
+		sum.PartitionID = partitionID
+		sum.Index = i.Name()
+		if verbose {
+			fmt.Printf("index: %v paritionID: %03v blake3-%x keyN: %10v idN: %10v\n", i.name, partitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
+		}
+		ats.Sums = append(ats.Sums, sum)
+	}
+
+	return
+}

mock/translator.go

@@ -37,6 +37,10 @@ type TranslateStore struct {
 	EntryReaderFunc   func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error)
 }
 
+func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) {
+	return
+}
+
 func (s *TranslateStore) Close() error {
 	return s.CloseFunc()
 }

translate.go

@@ -89,6 +89,24 @@ type TranslateStore interface {
 	// It should read from the reader and replace the data store with
 	// the read payload.
 	ReadFrom(io.Reader) (int64, error)
+
+	ComputeTranslatorSummary() (sum *TranslatorSummary, err error)
+}
+
+// TranslatorSummary is returned, for example from the boltdb string key translators,
+// by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method.
+type TranslatorSummary struct {
+	PartitionID int
+	Index       string
+
+	// Checksum has a blake3 crypto hash of all the keys->ID and all the ID->key mappings
+	Checksum []byte
+
+	// KeyCount has the number of Key->ID mappings
+	KeyCount int
+
+	// IDCount has the number of ID->Key mappings
+	IDCount int
 }
 
 // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore.
@@ -289,6 +307,9 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition
 	return NewInMemTranslateStore(index, field, partitionID, partitionN), nil
 }
 
+func (s *InMemTranslateStore) ComputeTranslatorSummary() (sum *TranslatorSummary, err error) {
+	panic("TODO")
+}
 func (s *InMemTranslateStore) Close() error {
 	return nil
 }