mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-07 11:27:50 +00:00
Merge pull request #788 from molecula/fldschk
pilosa-chk: show col, row translation, -v for fragment details
This commit is contained in:
jaten-molecula
GitHub
commit
921f15991b
7 changed files with 139 additions and 36 deletions
|
|
@ -518,6 +518,6 @@ func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSumma
|
|||
}
|
||||
var buf [16]byte
|
||||
_, _ = hasher.Digest().Read(buf[0:])
|
||||
sum.Checksum = buf[:]
|
||||
sum.Checksum = string(buf[:])
|
||||
return sum, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -489,10 +489,10 @@ func TestCryptoHashPerKey(t *testing.T) {
|
|||
|
||||
// hash one translation
|
||||
|
||||
expect := map[int][]byte{
|
||||
1: []byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12},
|
||||
2: []byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0},
|
||||
3: []byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9},
|
||||
expect := map[int]string{
|
||||
1: string([]byte{0x76, 0x48, 0x8b, 0x70, 0xe8, 0x54, 0x35, 0xc6, 0x8e, 0xa6, 0x4, 0x6c, 0xfa, 0xd2, 0x1a, 0x12}),
|
||||
2: string([]byte{0x81, 0x46, 0x84, 0x37, 0x26, 0x96, 0x41, 0xf3, 0x54, 0x4e, 0x98, 0xbc, 0x48, 0xab, 0x1b, 0xf0}),
|
||||
3: string([]byte{0x7f, 0xe9, 0xf, 0x6d, 0x7b, 0x14, 0x1, 0x44, 0xb2, 0x4e, 0xd0, 0x86, 0x2f, 0x62, 0x8c, 0xa9}),
|
||||
}
|
||||
for n := 1; n < 4; n++ {
|
||||
var batch0 []string
|
||||
|
|
@ -526,7 +526,7 @@ func TestCryptoHashPerKey(t *testing.T) {
|
|||
// shardwidth 22 has different hashes, of course.
|
||||
if pilosa.ShardWidth == 20 {
|
||||
expectedChecksum := expect[n]
|
||||
if !bytes.Equal(observedChecksum, expectedChecksum) {
|
||||
if observedChecksum != expectedChecksum {
|
||||
panic(fmt.Sprintf("got wrong checksum obs '%#v' vs expected '%#v'", observedChecksum, expectedChecksum))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,33 @@ import (
|
|||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// translatorchk : read boltdb files and print checksums and counts on the keys.
|
||||
// pilosa-chk : read boltdb files and print checksums and counts on the keys. With
|
||||
// -v and -ops and -bits you can display every last bit if you want.
|
||||
//
|
||||
// pilosa-chk is deliberately NOT a part of pilosa so that it can run without
|
||||
// forcing a customer to upgrade or downgrade their installed version.
|
||||
|
||||
func main() {
|
||||
|
||||
var dir string
|
||||
var showOpsLog bool
|
||||
var showBits bool
|
||||
var showFrags bool
|
||||
home := os.Getenv("HOME")
|
||||
flag.StringVar(&dir, "dir", fmt.Sprintf("%v/.pilosa", home), "pilosa data dir to read")
|
||||
flag.BoolVar(&showFrags, "v", false, "show the checksum hash for each fragment in each index. Warning: long output")
|
||||
flag.BoolVar(&showOpsLog, "ops", false, "show the ops log for each fragment. Warning: very long output. Implies -v")
|
||||
flag.BoolVar(&showBits, "bits", false, "show the hot bits for each fragment. Warning: very, very long output. Implies -v")
|
||||
flag.Parse()
|
||||
|
||||
if showBits {
|
||||
showFrags = true
|
||||
}
|
||||
if showOpsLog {
|
||||
showFrags = true
|
||||
}
|
||||
fmt.Printf("opening dir '%v'... this may take a few seconds...\n", dir)
|
||||
fmt.Printf(" the blake-3 hash includes the value of each mapping and the field or partitionID.\n")
|
||||
|
||||
holder := pilosa.NewHolder(256)
|
||||
holder.Path = dir
|
||||
|
|
@ -46,7 +63,8 @@ func main() {
|
|||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("\ncalculating checksums on data from dir '%v'...\n", dir)
|
||||
fmt.Printf("\ncalculating hashes of row and column key translation maps on data from dir '%v'...\n", dir)
|
||||
var indexes []*pilosa.Index
|
||||
|
||||
final := pilosa.NewAllTranslatorSummary()
|
||||
const verbose = true
|
||||
|
|
@ -55,7 +73,8 @@ func main() {
|
|||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
final.Merge(asum)
|
||||
final.Append(asum)
|
||||
indexes = append(indexes, idx)
|
||||
}
|
||||
final.Sort()
|
||||
|
||||
|
|
@ -63,11 +82,20 @@ func main() {
|
|||
fmt.Printf("\nsummary of %v:\n", dir)
|
||||
for _, sum := range final.Sums {
|
||||
//fmt.Printf("index: %v partitionID: %v blake3-%x keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
|
||||
_, _ = hasher.Write(sum.Checksum)
|
||||
_, _ = hasher.Write([]byte(sum.Checksum))
|
||||
}
|
||||
|
||||
var buf [16]byte
|
||||
_, _ = hasher.Digest().Read(buf[0:])
|
||||
|
||||
fmt.Printf("all-checksum = blake3-%x\n", buf)
|
||||
|
||||
if showFrags {
|
||||
for _, idx := range indexes {
|
||||
fmt.Printf("==============================\n")
|
||||
fmt.Printf("index: %v\n", idx.Name())
|
||||
fmt.Printf("==============================\n")
|
||||
idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog)
|
||||
}
|
||||
}
|
||||
}
|
||||
61
index.go
61
index.go
|
|
@ -17,6 +17,7 @@ package pilosa
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -776,12 +777,12 @@ type AllTranslatorSummary struct {
|
|||
func NewAllTranslatorSummary() *AllTranslatorSummary {
|
||||
return &AllTranslatorSummary{}
|
||||
}
|
||||
func (ats *AllTranslatorSummary) Merge(b *AllTranslatorSummary) {
|
||||
func (ats *AllTranslatorSummary) Append(b *AllTranslatorSummary) {
|
||||
ats.Sums = append(ats.Sums, b.Sums...)
|
||||
}
|
||||
|
||||
func (ats *AllTranslatorSummary) Sort() {
|
||||
// return sorted by index then PartitionID
|
||||
// return sorted by index then PartitionID then Field
|
||||
sort.Slice(ats.Sums, func(i, j int) bool {
|
||||
a := ats.Sums[i]
|
||||
b := ats.Sums[j]
|
||||
|
|
@ -792,17 +793,41 @@ func (ats *AllTranslatorSummary) Sort() {
|
|||
return false
|
||||
}
|
||||
// INVAR: a.Index == b.Index
|
||||
return a.PartitionID < b.PartitionID
|
||||
if a.PartitionID < b.PartitionID {
|
||||
return true
|
||||
}
|
||||
if a.PartitionID > b.PartitionID {
|
||||
return false
|
||||
}
|
||||
return a.Field < b.Field
|
||||
})
|
||||
}
|
||||
|
||||
// sums is only guaranteed to be sorted by (index, PartitionID) if err returns nil
|
||||
// sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil
|
||||
func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) {
|
||||
i.mu.RLock() // avoid race with Index.Close() doing i.translateStores = make(map[int]TranslateStore)
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
ats = &AllTranslatorSummary{}
|
||||
|
||||
fmt.Printf("\nindex: %v\n=================\n", i.name)
|
||||
for _, fld := range i.fields {
|
||||
sum, err := fld.translateStore.ComputeTranslatorSummary()
|
||||
if err != nil {
|
||||
return ats, err
|
||||
}
|
||||
sum.Field = fld.name
|
||||
sum.Index = i.Name()
|
||||
sum.Checksum = blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name())))
|
||||
|
||||
if verbose {
|
||||
fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name)
|
||||
}
|
||||
ats.Sums = append(ats.Sums, sum)
|
||||
}
|
||||
|
||||
fmt.Printf("====================\n")
|
||||
|
||||
for partitionID, store := range i.translateStores {
|
||||
sum, err := store.ComputeTranslatorSummary()
|
||||
if err != nil {
|
||||
|
|
@ -814,11 +839,33 @@ func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummar
|
|||
}
|
||||
sum.PartitionID = partitionID
|
||||
sum.Index = i.Name()
|
||||
|
||||
sum.Checksum = blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name())))
|
||||
if verbose {
|
||||
fmt.Printf("index: %v paritionID: %03v blake3-%x keyN: %10v idN: %10v\n", i.name, partitionID, sum.Checksum, sum.KeyCount, sum.IDCount)
|
||||
fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID)
|
||||
}
|
||||
ats.Sums = append(ats.Sums, sum)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) {
|
||||
paths, err := listFilesUnderDir(idx.path, false, "", true)
|
||||
panicOn(err)
|
||||
index := idx.name
|
||||
n := 0
|
||||
for _, relpath := range paths {
|
||||
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
|
||||
if err != nil {
|
||||
continue // ignore .meta paths
|
||||
}
|
||||
abspath := idx.path + sep + relpath
|
||||
|
||||
checksum, hotbits := RoaringFragmentChecksum(abspath, index, field, view, shard)
|
||||
fmt.Fprintf(w, "frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v\n", checksum, field, view, shard, hotbits)
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
fmt.Fprintf(w, "empty index '%v'", idx.path)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
rrtx.go
2
rrtx.go
|
|
@ -39,7 +39,7 @@ func (tx *RoaringTx) Type() string {
|
|||
}
|
||||
|
||||
func (tx *RoaringTx) Dump() {
|
||||
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys())
|
||||
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(false, false))
|
||||
}
|
||||
|
||||
func (tx *RoaringTx) UseRowCache() bool {
|
||||
|
|
|
|||
|
|
@ -96,11 +96,16 @@ type TranslateStore interface {
|
|||
// TranslatorSummary is returned, for example from the boltdb string key translators,
|
||||
// by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method.
|
||||
type TranslatorSummary struct {
|
||||
Index string
|
||||
|
||||
// ParitionID is filled for column keys
|
||||
PartitionID int
|
||||
Index string
|
||||
|
||||
// Field is filled for row keys
|
||||
Field string
|
||||
|
||||
// Checksum has a blake3 crypto hash of all the keys->ID and all the ID->key mappings
|
||||
Checksum []byte
|
||||
Checksum string
|
||||
|
||||
// KeyCount has the number of Key->ID mappings
|
||||
KeyCount int
|
||||
|
|
|
|||
55
txfactory.go
55
txfactory.go
|
|
@ -16,6 +16,7 @@ package pilosa
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
|
@ -26,6 +27,7 @@ import (
|
|||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
"github.com/pilosa/pilosa/v2/txkey"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// public strings that pilosa/server/config.go can reference
|
||||
|
|
@ -394,7 +396,9 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
|
|||
return
|
||||
}
|
||||
|
||||
func (idx *Index) StringifiedRoaringKeys() (r string) {
|
||||
// hashOnly means only show the value hash, not the content bits.
|
||||
// showOps means display the ops log.
|
||||
func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) {
|
||||
paths, err := listFilesUnderDir(idx.path, false, "", true)
|
||||
panicOn(err)
|
||||
index := idx.name
|
||||
|
|
@ -407,8 +411,8 @@ func (idx *Index) StringifiedRoaringKeys() (r string) {
|
|||
continue // ignore .meta paths
|
||||
}
|
||||
abspath := idx.path + sep + relpath
|
||||
const showOps = false
|
||||
s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps)
|
||||
|
||||
s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout)
|
||||
panicOn(err)
|
||||
//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
|
||||
if s == "" {
|
||||
|
|
@ -426,7 +430,20 @@ func (idx *Index) StringifiedRoaringKeys() (r string) {
|
|||
return "roaring-" + r
|
||||
}
|
||||
|
||||
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps bool) (r string, err error) {
|
||||
func RoaringFragmentChecksum(path string, index, field, view string, shard uint64) (r string, hotbits int) {
|
||||
hasher := blake3.New()
|
||||
showOps := false
|
||||
hashOnly := true
|
||||
_, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher)
|
||||
panicOn(err)
|
||||
fmt.Fprintf(hasher, "%v/%v/%v/%v", index, field, view, shard)
|
||||
var buf [16]byte
|
||||
_, _ = hasher.Digest().Read(buf[0:])
|
||||
return fmt.Sprintf("%x", buf), hotbits
|
||||
|
||||
}
|
||||
|
||||
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps, hashOnly bool, w io.Writer) (r string, hotbits int, err error) {
|
||||
|
||||
var info roaring.BitmapInfo
|
||||
_ = info
|
||||
|
|
@ -473,10 +490,10 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
|
|||
to: info.To,
|
||||
}
|
||||
if info.ContainerCount > 0 {
|
||||
printContainers(info, pC)
|
||||
printContainers(w, info, pC)
|
||||
}
|
||||
if info.Ops > 0 {
|
||||
printOps(info)
|
||||
printOps(w, info)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -491,13 +508,19 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
|
|||
cts := roaring.NewSliceContainers()
|
||||
cts.Put(ckey, ct)
|
||||
rbm := &roaring.Bitmap{Containers: cts}
|
||||
srbm := bitmapAsString(rbm)
|
||||
panicOn(err)
|
||||
var srbm string
|
||||
if !hashOnly {
|
||||
srbm = bitmapAsString(rbm)
|
||||
}
|
||||
|
||||
bkey := string(txkey.Key(index, field, view, shard, ckey))
|
||||
|
||||
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
|
||||
r += " ......." + srbm + "\n"
|
||||
n := ct.N()
|
||||
hotbits += int(n)
|
||||
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, n)
|
||||
if !hashOnly {
|
||||
r += " ......." + srbm + "\n"
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
|
|
@ -579,9 +602,9 @@ type pointerContext struct {
|
|||
from, to uintptr
|
||||
}
|
||||
|
||||
func printOps(info roaring.BitmapInfo) {
|
||||
fmt.Fprintln(os.Stdout, " Ops:")
|
||||
tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
|
||||
func printOps(w io.Writer, info roaring.BitmapInfo) {
|
||||
fmt.Fprintln(w, " Ops:")
|
||||
tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0)
|
||||
fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
|
||||
printed := 0
|
||||
for _, op := range info.OpDetails {
|
||||
|
|
@ -606,9 +629,9 @@ func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
|
|||
}
|
||||
|
||||
// stolen from ctl/inspect.go
|
||||
func printContainers(info roaring.BitmapInfo, pC pointerContext) {
|
||||
fmt.Fprintln(os.Stdout, " Containers:")
|
||||
tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
|
||||
func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) {
|
||||
fmt.Fprintln(w, " Containers:")
|
||||
tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0)
|
||||
fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
|
||||
fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
|
||||
c1s := info.Containers
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue