diff --git a/api.go b/api.go index 8b6f3249d..23c36cb0b 100644 --- a/api.go +++ b/api.go @@ -1515,6 +1515,10 @@ func (api *API) Info() serverInfo { } } +func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) { + return api.holder.Inspect(ctx, req) +} + // GetTranslateEntryReader provides an entry reader for key translation logs starting at offset. func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader") diff --git a/cmd/inspect.go b/cmd/inspect.go index 47a40cae8..86452905d 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -48,5 +48,9 @@ Inspects a data file and provides stats. flags := inspectCmd.Flags() flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers") flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)") + flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes") + flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views") + flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields") + flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards") return inspectCmd } diff --git a/ctl/inspect.go b/ctl/inspect.go index 11471caa9..eb24a0597 100644 --- a/ctl/inspect.go +++ b/ctl/inspect.go @@ -16,15 +16,23 @@ package ctl import ( "context" + "encoding/binary" "fmt" + "hash/fnv" "io" + "io/ioutil" "os" + "path/filepath" + "sort" + "strconv" + "strings" "syscall" "text/tabwriter" "time" - "unsafe" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) @@ -37,6 +45,8 @@ type InspectCommand struct { Quiet bool // list only this many objects Max int + // Filters: + InspectOpts pilosa.InspectRequest // Standard input/output *pilosa.CmdIO @@ -68,12 +78,12 @@ func (p *pointerContext) pretty(c roaring.ContainerInfo) string { } func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) { - fmt.Fprintln(cmd.Stdout, "== Ops ==") + fmt.Fprintln(cmd.Stdout, " Ops:") tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, "%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") + fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") printed := 0 for _, op := range info.OpDetails { - fmt.Fprintf(tw, "%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) + fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) printed++ if cmd.Max != 0 && printed >= cmd.Max { break @@ -83,10 +93,10 @@ func (cmd *InspectCommand) PrintOps(info roaring.BitmapInfo) { } func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerContext) { - fmt.Fprintln(cmd.Stdout, "== Containers ==") + fmt.Fprintln(cmd.Stdout, " Containers:") tw := tabwriter.NewWriter(cmd.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, "\tRoaring\t\t\t\tOps\t\t\t\t\n") - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET") + fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") + fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") c1s := info.Containers c2s := info.OpContainers l1 := len(c1s) @@ -115,10 +125,14 @@ func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerCo var key uint64 c1fmt := "-\t\t\t" c2fmt := "-\t\t\t" + // If c2 exists, we'll always prefer its flags, + // if it doesn't, this gets overwritten. + flags := c2.Flags if !c2e || (c1e && c1.Key < c2.Key) { c1fmt = pC.pretty(c1) key = c1.Key c1used = true + flags = c1.Flags } else if !c1e || (c2e && c2.Key < c1.Key) { c2fmt = pC.pretty(c2) key = c2.Key @@ -147,7 +161,7 @@ func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerCo c2e = false } } - fmt.Fprintf(tw, "%d\t%s\t%s\t\n", key, c1fmt, c2fmt) + fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) printed++ if cmd.Max > 0 && printed >= cmd.Max { break @@ -157,7 +171,7 @@ func (cmd *InspectCommand) PrintContainers(info roaring.BitmapInfo, pC pointerCo } // Run executes the inspect command. -func (cmd *InspectCommand) Run(_ context.Context) error { +func (cmd *InspectCommand) Run(ctx context.Context) error { // Open file handle. f, err := os.Open(cmd.Path) if err != nil { @@ -169,7 +183,173 @@ func (cmd *InspectCommand) Run(_ context.Context) error { if err != nil { return errors.Wrap(err, "statting file") } + if fi.IsDir() { + total := 0 + infos, err := f.Readdir(0) + if err != nil { + return err + } + if len(infos) == 0 { + return errors.New("directory contains no files") + } + names := make([]string, len(infos)) + nameToInfo := make(map[string]os.FileInfo, len(infos)) + // find numeric-only names; we'll operate on + // either those, or the whole holder if we find + // a .topology file. + n := 0 + for _, fi := range infos { + name := fi.Name() + if name == ".topology" { + return cmd.InspectHolder(ctx, cmd.Path) + } + if _, err := strconv.Atoi(name); err == nil { + names[n] = name + nameToInfo[name] = fi + n++ + } + } + if n == 0 { + return fmt.Errorf("directory contains no fragments (looking for numeric names)") + } + names = names[:n] + fmt.Fprintf(cmd.Stdout, "%s contains %d fragments:\n", cmd.Path, n) + for _, name := range names { + f2, err := os.Open(filepath.Join(cmd.Path, name)) + if err != nil { + return fmt.Errorf("opening %q: %v", name, err) + } + fmt.Fprintf(cmd.Stdout, "%s/%s:\n", cmd.Path, name) + err = cmd.InspectFile(f2, nameToInfo[name]) + total++ + f2.Close() + if err != nil { + return fmt.Errorf("inspecting %q: %v", name, err) + } + } + return nil + } + return cmd.InspectFile(f, fi) +} + +// loadTopology is copied almost exactly from pilosa/cluster.go. +func loadTopology(path string) (topology internal.Topology, myID string, err error) { + buf, err := ioutil.ReadFile(filepath.Join(path, ".topology")) + if os.IsNotExist(err) { + return topology, myID, err + } else if err != nil { + return topology, myID, errors.Wrap(err, "reading file") + } + if err := proto.Unmarshal(buf, &topology); err != nil { + return topology, myID, errors.Wrap(err, "unmarshalling") + } + sort.Slice(topology.NodeIDs, + func(i, j int) bool { + return topology.NodeIDs[i] < topology.NodeIDs[j] + }) + buf, err = ioutil.ReadFile(filepath.Join(path, ".id")) + if os.IsNotExist(err) { + return topology, myID, err + } else if err != nil { + return topology, myID, nil + } + myID = strings.TrimSpace(string(buf)) + return topology, myID, nil +} + +var partitions = make(map[string]map[uint64]int) + +func findPartition(index string, shard uint64, partitionN int) (partition int) { + var shardMap map[uint64]int + var ok bool + if shardMap, ok = partitions[index]; !ok { + shardMap = make(map[uint64]int) + partitions[index] = shardMap + } + if partition, ok = shardMap[shard]; !ok { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], shard) + + // Hash the bytes and mod by partition count. + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write(buf[:]) + partition = int(h.Sum64() % uint64(partitionN)) + shardMap[shard] = partition + } + return partition +} + +func findPartitionPath(path string, partitionN int) (int, error) { + parts := strings.Split(path, "/") + shard, err := strconv.ParseUint(parts[len(parts)-1], 10, 64) + if err != nil { + return 0, err + } + return findPartition(parts[0], shard, partitionN), nil +} + +func (cmd *InspectCommand) InspectHolder(ctx context.Context, path string) error { + holder := pilosa.NewHolder(pilosa.DefaultPartitionN) + holder.Path = path + holder.Opts.Inspect = true + holder.Opts.ReadOnly = true + err := holder.Open() + if err != nil { + return fmt.Errorf("%s: holder open: %v", path, err) + } + holderInfo, err := holder.Inspect(ctx, &cmd.InspectOpts) + if err != nil { + return fmt.Errorf("%s: inspect: %v", path, err) + } + myPartition := 0 + topology, myID, err := loadTopology(path) + if err == nil { + fmt.Fprintf(cmd.Stdout, "Cluster ID: %q\n", topology.ClusterID) + if len(topology.NodeIDs) > 1 { + fmt.Fprintf(cmd.Stdout, "Cluster of %d nodes, this node %q\n", len(topology.NodeIDs), myID) + } else { + fmt.Fprintf(cmd.Stdout, "Cluster has only one node: %q\n", myID) + } + found := false + for i := range topology.NodeIDs { + if topology.NodeIDs[i] == myID { + found = true + myPartition = i + break + } + } + if !found { + fmt.Fprintf(cmd.Stdout, "Warning: node ID %q not found in topology (%q)\n", myID, topology.NodeIDs) + } + } else { + fmt.Fprintf(cmd.Stdout, "warning: reading topology failed: %v\n", err) + } + for _, name := range holderInfo.FragmentNames { + partition, err := findPartitionPath(name, len(topology.NodeIDs)) + if err != nil { + fmt.Fprintf(cmd.Stdout, "%s: [can't find partition: %v]\n", name, err) + } else { + if partition == myPartition { + fmt.Fprintf(cmd.Stdout, "%s:\n", name) + } else { + fmt.Fprintf(cmd.Stdout, "%s: [primary node %q]\n", name, topology.NodeIDs[partition]) + } + } + details := holderInfo.FragmentInfo[name] + cmd.DisplayInfo(details.BitmapInfo) + if details.BlockChecksums != nil { + fmt.Fprintf(cmd.Stdout, " Checksums [%d total]:\n", len(details.BlockChecksums)) + for _, block := range details.BlockChecksums { + fmt.Fprintf(cmd.Stdout, " %8d: %x\n", block.ID, block.Checksum) + } + } + } + return nil +} + +func (cmd *InspectCommand) InspectFile(f *os.File, fi os.FileInfo) error { // Memory map the file. data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { @@ -184,22 +364,27 @@ func (cmd *InspectCommand) Run(_ context.Context) error { // Attach the mmap file to the bitmap. t := time.Now() fmt.Fprintf(cmd.Stderr, "inspecting bitmap...") - info, err := roaring.InspectBinary(data) + var info roaring.BitmapInfo + _, _, err = roaring.InspectBinary(data, true, &info) + fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) + cmd.DisplayInfo(info) if err != nil { return errors.Wrap(err, "inspecting") } - fmt.Fprintf(cmd.Stderr, " (%s)\n", time.Since(t)) + return nil +} +func (cmd *InspectCommand) DisplayInfo(info roaring.BitmapInfo) { pC := pointerContext{ - from: uintptr(unsafe.Pointer(&data[0])), + from: info.From, + to: info.To, } - pC.to = pC.from + uintptr(len(data)) // Print top-level info. - fmt.Fprintf(cmd.Stdout, "== Bitmap Info ==\n") - fmt.Fprintf(cmd.Stdout, "Bits: %d\n", info.BitCount) - fmt.Fprintf(cmd.Stdout, "Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers)) - fmt.Fprintf(cmd.Stdout, "Operations: %d (%d bits)\n", info.Ops, info.OpN) + fmt.Fprintf(cmd.Stdout, " Bitmap Info:\n") + fmt.Fprintf(cmd.Stdout, " Bits: %d\n", info.BitCount) + fmt.Fprintf(cmd.Stdout, " Containers: %d (%d roaring)\n", info.ContainerCount, len(info.Containers)) + fmt.Fprintf(cmd.Stdout, " Operations: %d (%d bits)\n", info.Ops, info.OpN) fmt.Fprintln(cmd.Stdout, "") // Print info for each container. @@ -211,6 +396,4 @@ func (cmd *InspectCommand) Run(_ context.Context) error { cmd.PrintOps(info) } } - - return nil } diff --git a/fragment.go b/fragment.go index 9aeceb79a..61ac78313 100644 --- a/fragment.go +++ b/fragment.go @@ -124,6 +124,7 @@ type fragment struct { snapshotCond sync.Cond snapshotErr error // error yielded by the last snapshot operation snapshotStamp time.Time // timestamp of last snapshot + open bool // is this fragment actually open? // Cache for row counts. CacheType string // passed in by field @@ -156,6 +157,8 @@ type fragment struct { mutexVector vector stats stats.StatsClient + + bitmapInfo *roaring.BitmapInfo } // newFragment returns a new instance of Fragment. @@ -182,6 +185,23 @@ func newFragment(holder *Holder, path, index, field, view string, shard uint64, // cachePath returns the path to the fragment's cache data. func (f *fragment) cachePath() string { return f.path + cacheExt } +type FragmentInfo struct { + BitmapInfo roaring.BitmapInfo + BlockChecksums []FragmentBlock `json:"BlockChecksums,omitempty"` +} + +func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) { + if f.bitmapInfo == nil { + fi.BitmapInfo = f.storage.Info(params.Containers) + } else { + fi.BitmapInfo = *f.bitmapInfo + } + if params.Checksum { + fi.BlockChecksums = f.Blocks() + } + return fi +} + // Open opens the underlying storage. func (f *fragment) Open() error { f.mu.Lock() @@ -214,6 +234,7 @@ func (f *fragment) Open() error { f.close() return err } + f.open = true f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) return nil @@ -223,7 +244,7 @@ func (f *fragment) Open() error { // get no data. It tries to write the current storage to the provided file, // which is assumed to be the file they didn't get any data from. func (f *fragment) emptyStorage(file *os.File) (bool, error) { - if f.holder.ReadOnly { + if f.holder.Opts.ReadOnly { return false, errors.New("can't flush/create storage for read-only holder") } // No data. We'll mark this for no mapping, clear any existing @@ -279,7 +300,7 @@ func (f *fragment) importStorage(data []byte, file *os.File, newGen generation, } f.holder.Logger.Printf("warning: unmarshal storage, file=%s, err=%v", file.Name(), err) trunc, ok := cause.(roaring.FileShouldBeTruncatedError) - if ok && !f.holder.ReadOnly { + if ok && !f.holder.Opts.ReadOnly { // if the holder is ReadOnly, we silently ignore the "advisory" // error. This may be a bad idea. @@ -342,6 +363,12 @@ func (f *fragment) applyStorage(data []byte, file *os.File, newGen generation, m return mapped, err } +func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, mapped bool) (didMap bool, err error) { + f.bitmapInfo = &roaring.BitmapInfo{} + f.storage, didMap, err = roaring.InspectBinary(data, mapped, f.bitmapInfo) + return didMap, err +} + // openStorage opens the storage bitmap. // // This has been massively reworked recently, and now hands a lot of @@ -360,10 +387,17 @@ func (f *fragment) openStorage(unmarshalData bool) error { } f.rowCache = &simpleCache{make(map[uint64]*Row)} var storageOp func([]byte, *os.File, generation, bool) (bool, error) - if unmarshalData { - storageOp = f.importStorage + if f.holder.Opts.Inspect { + // note that this will unmarshal even if we already have + // storage; when Inspect is on for a holder, we actually want + // to be able to report this. + storageOp = f.inspectStorage } else { - storageOp = f.applyStorage + if unmarshalData { + storageOp = f.importStorage + } else { + storageOp = f.applyStorage + } } var err error f.gen, err = newGeneration(f.gen, f.path, unmarshalData, storageOp, f.holder.Logger) @@ -436,6 +470,9 @@ func (f *fragment) Close() error { for f.snapshotPending { f.snapshotCond.Wait() } + // Note: snapshots won't progress on a closed fragment, so we + // wait until after a possible pending snapshot to close. + f.open = false return f.close() } @@ -2283,6 +2320,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg // snapshot does the actual snapshot operation. it does not check or care // about f.snapshotPending. func (f *fragment) snapshot() (err error) { + if !f.open { + return errors.New("snapshot request on closed fragment") + } wouldPanic := debug.SetPanicOnFault(true) defer func() { debug.SetPanicOnFault(wouldPanic) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 2d8c27b73..fa1ffbeec 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2666,6 +2666,12 @@ func mustOpenBSIFragment(index, field, view string, shard uint64) *fragment { return mustOpenFragmentFlags(index, field, view, shard, "", 1) } +var testHolder = NewHolder(DefaultPartitionN) + +func init() { + testHolder.SnapshotQueue = newSnapshotQueue(1, 1, nil) +} + // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) *fragment { file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") @@ -2678,7 +2684,8 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st cacheType = DefaultCacheType } - f := newFragment(NewHolder(DefaultPartitionN), file.Name(), index, field, view, shard, flags) + f := newFragment(testHolder, file.Name(), index, field, view, shard, flags) + f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ store: make(map[uint64]map[string]interface{}), diff --git a/handler.go b/handler.go index 17156245b..bb5aa245d 100644 --- a/handler.go +++ b/handler.go @@ -287,3 +287,31 @@ type TranslateIDsRequest struct { type TranslateIDsResponse struct { Keys []string } + +// InspectRequestParams represents the parts of an InspectRequest that +// aren't generic holder filtering attributes. +type InspectRequestParams struct { + Containers bool // include container details + Checksum bool // perform checksums +} + +// InspectRequest represents a request for a possibly-partial +// holder inspection, using a provided holder filter and inspect-specific +// parameters. +type InspectRequest struct { + HolderFilterParams + InspectRequestParams +} + +// InspectResponse contains the structured results for an InspectRequest. +// It may some day be expanded to include metadata about views or indexes. +type InspectResponse struct { + Fragments []struct { + Index string + Field string + View string + Shard int64 + Path string + Info *FragmentInfo + } +} diff --git a/holder.go b/holder.go index 90df16860..f8ac267f9 100644 --- a/holder.go +++ b/holder.go @@ -21,7 +21,9 @@ import ( "os" "path" "path/filepath" + "regexp" "sort" + "strconv" "strings" "sync" "syscall" @@ -102,10 +104,17 @@ type Holder struct { // have opened. opening bool + Opts HolderOpts +} + +type HolderOpts struct { // ReadOnly indicates that this holder's contents should not produce // disk writes under any circumstances. It must be set before Open // is called, and changing it is not supported. ReadOnly bool + // If Inspect is set, we'll try to obtain additional information + // about fragments when opening them. + Inspect bool } func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { @@ -178,6 +187,280 @@ func NewHolder(partitionN int) *Holder { } } +type HolderInfo struct { + FragmentInfo map[string]FragmentInfo + FragmentNames []string +} + +type regexpList []*regexp.Regexp + +func newRegexpList(regexes string) (results regexpList, err error) { + if regexes == "" { + return nil, nil + } + for _, sub := range strings.Split(regexes, ",") { + re, err := regexp.Compile(sub) + if err != nil { + return nil, err + } + results = append(results, re) + } + return results, nil +} + +func (rl regexpList) Match(haystack string) bool { + if rl == nil { + return true + } + for _, re := range rl { + if re.MatchString(haystack) { + return true + } + } + return false +} + +// shardRange represents a series of shards +type shardRange struct { + min, max uint64 +} + +type shardRangeList []shardRange + +func newShardRangeList(shards string) (results shardRangeList, err error) { + if shards == "" { + return nil, nil + } + for _, sub := range strings.Split(shards, ",") { + var sr shardRange + minMax := strings.Split(sub, "-") + if len(minMax) > 2 { + return nil, fmt.Errorf("invalid range %q", sub) + } + sr.min, err = strconv.ParseUint(minMax[0], 10, 64) + if err != nil { + return nil, err + } + sr.max = sr.min + if len(minMax) == 2 { + sr.max, err = strconv.ParseUint(minMax[0], 10, 64) + if err != nil { + return nil, err + } + } + if sr.max < sr.min { + return nil, fmt.Errorf("invalid range %q: max < min", sub) + } + results = append(results, sr) + } + return results, nil +} + +func (sl shardRangeList) Match(shard uint64) bool { + if sl == nil { + return true + } + for _, sr := range sl { + if shard >= sr.min && shard <= sr.max { + return true + } + } + return false +} + +// HolderFilter represents something that potentially filters out +// parts of a holder, indicating whether or not to process them, +// or recurse into them. It is permissible to recurse a thing +// without processing it, or process it without recursing it. +// For instance, something looking to accumulate statistics +// about views might return (true, false) from CheckView, +// while a fragment scanning operation would return (false, true) +// from everything above CheckFrag. +type HolderFilter interface { + CheckIndex(iname string) (process bool, recurse bool) + CheckField(iname, fname string) (process bool, recurse bool) + CheckView(iname, fname, vname string) (process bool, recurse bool) + CheckFragment(iname, fname, vname string, shard uint64) (process bool) +} + +// HolderFilterAll is a placeholder type which always returns true for the +// check functions. You can embed it to make a HolderOperator which processes +// everything. +type HolderFilterAll struct{} + +func (HolderFilterAll) CheckIndex(string) (bool, bool) { + return true, true +} + +func (HolderFilterAll) CheckField(string, string) (bool, bool) { + return true, true +} + +func (HolderFilterAll) CheckView(string, string, string) (bool, bool) { + return true, true +} + +func (HolderFilterAll) CheckFragment(string, string, string, uint64) bool { + return true +} + +// HolderProcessNone is a placeholder type which does nothing for the +// process functions. You can embed it to make a HolderOperator which +// does nothing, or embed it and provide your own ProcessFragment to +// do just that. +type HolderProcessNone struct{} + +func (HolderProcessNone) ProcessIndex(*Index) error { + return nil +} + +func (HolderProcessNone) ProcessField(*Field) error { + return nil +} + +func (HolderProcessNone) ProcessView(*view) error { + return nil +} + +func (HolderProcessNone) ProcessFragment(*fragment) error { + return nil +} + +// HolderProcess represents something that has operations which can be +// performed on indexes, fields, views, and/or fragments. +type HolderProcess interface { + ProcessIndex(*Index) error + ProcessField(*Field) error + ProcessView(*view) error + ProcessFragment(*fragment) error +} + +// HolderOperator is both a filter and a process. This is the general +// form of "I want to do something to some part of a holder." +type HolderOperator interface { + HolderFilter + HolderProcess +} + +var _ HolderOperator = (*holderInspector)(nil) + +type HolderFilterParams struct { + Indexes string + Fields string + Views string + Shards string +} + +type holderFilterFull struct { + HolderFilterParams + indexRegexps regexpList + fieldRegexps regexpList + viewRegexps regexpList + shardRanges shardRangeList +} + +type inspectRequestFull struct { + HolderFilter + params InspectRequestParams +} + +func (i *holderFilterFull) CheckIndex(iname string) (process, recurse bool) { + return true, i.indexRegexps.Match(iname) +} + +func (i *holderFilterFull) CheckField(iname, fname string) (process, recurse bool) { + return true, i.fieldRegexps.Match(fname) +} + +func (i *holderFilterFull) CheckView(iname, fname, vname string) (process, recurse bool) { + return true, i.viewRegexps.Match(vname) +} + +func (i *holderFilterFull) CheckFragment(iname, fname, vname string, shard uint64) (process bool) { + return i.shardRanges.Match(shard) +} + +func NewHolderFilter(params HolderFilterParams) (result HolderFilter, err error) { + filter := &holderFilterFull{ + HolderFilterParams: params, + } + filter.indexRegexps, err = newRegexpList(params.Indexes) + if err != nil { + return nil, err + } + filter.fieldRegexps, err = newRegexpList(params.Fields) + if err != nil { + return nil, err + } + filter.viewRegexps, err = newRegexpList(params.Views) + if err != nil { + return nil, err + } + filter.shardRanges, err = newShardRangeList(params.Shards) + if err != nil { + return nil, err + } + return filter, nil +} + +func expandInspectRequest(req *InspectRequest) (*inspectRequestFull, error) { + filter, err := NewHolderFilter(req.HolderFilterParams) + if err != nil { + return nil, err + } + irf := &inspectRequestFull{ + HolderFilter: filter, + params: req.InspectRequestParams, + } + return irf, nil +} + +type holderInspector struct { + *inspectRequestFull + pathParts [3]string + path string + hi *HolderInfo +} + +func (h *holderInspector) ProcessIndex(i *Index) error { + h.pathParts[0] = i.name + return nil +} + +func (h *holderInspector) ProcessField(f *Field) error { + h.pathParts[1] = f.name + return nil +} + +func (h *holderInspector) ProcessView(v *view) error { + h.pathParts[2] = v.name + h.path = strings.Join(h.pathParts[:], "/") + return nil +} + +func (h *holderInspector) ProcessFragment(f *fragment) error { + path := h.path + "/" + strconv.FormatUint(f.shard, 10) + h.hi.FragmentInfo[path] = f.inspect(h.inspectRequestFull.params) + h.hi.FragmentNames = append(h.hi.FragmentNames, path) + return nil +} + +func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) { + fullReq, err := expandInspectRequest(req) + if err != nil { + return nil, err + } + inspector := &holderInspector{ + inspectRequestFull: fullReq, + hi: &HolderInfo{ + FragmentInfo: make(map[string]FragmentInfo), + }, + } + err = h.Process(ctx, inspector) + sort.Strings(inspector.hi.FragmentNames) + return inspector.hi, err +} + // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.opening = true @@ -1369,3 +1652,133 @@ func uint64InSlice(i uint64, s []uint64) bool { } return false } + +// Process loops through a holder based on the Check functions in op, calling +// the Process functions in op when indicated. +func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { + var indexNames, fieldNames, viewNames []string + var fragNums []uint64 + + h.mu.Lock() + indexNames = indexNames[:0] + for indexName := range h.indexes { + indexNames = append(indexNames, indexName) + } + h.mu.Unlock() + for _, indexName := range indexNames { + if err = ctx.Err(); err != nil { + return err + } + process, recurse := op.CheckIndex(indexName) + if !process && !recurse { + continue + } + h.mu.Lock() + index := h.indexes[indexName] + h.mu.Unlock() + if index == nil { + continue + } + if err = ctx.Err(); err != nil { + return err + } + if process { + err = op.ProcessIndex(index) + if err != nil { + return err + } + } + if !recurse { + continue + } + fieldNames = fieldNames[:0] + index.mu.Lock() + for fieldName := range index.fields { + fieldNames = append(fieldNames, fieldName) + } + index.mu.Unlock() + for _, fieldName := range fieldNames { + if err = ctx.Err(); err != nil { + return err + } + process, recurse := op.CheckField(indexName, fieldName) + if !process && !recurse { + continue + } + index.mu.Lock() + field := index.fields[fieldName] + index.mu.Unlock() + if field == nil { + continue + } + if err = ctx.Err(); err != nil { + return err + } + if process { + err = op.ProcessField(field) + if err != nil { + return err + } + } + if !recurse { + continue + } + viewNames = viewNames[:0] + field.mu.Lock() + for viewName := range field.viewMap { + viewNames = append(viewNames, viewName) + } + field.mu.Unlock() + for _, viewName := range viewNames { + if err = ctx.Err(); err != nil { + return err + } + process, recurse := op.CheckView(indexName, fieldName, viewName) + if !process && !recurse { + continue + } + field.mu.Lock() + view := field.viewMap[viewName] + field.mu.Unlock() + if view == nil { + continue + } + if err = ctx.Err(); err != nil { + return err + } + if process { + err = op.ProcessView(view) + if err != nil { + return err + } + } + if !recurse { + continue + } + fragNums := fragNums[:0] + view.mu.Lock() + for fragNum := range view.fragments { + fragNums = append(fragNums, fragNum) + } + view.mu.Unlock() + for _, fragNum := range fragNums { + if err = ctx.Err(); err != nil { + return err + } + process := op.CheckFragment(indexName, fieldName, viewName, fragNum) + if !process { + continue + } + view.mu.Lock() + frag := view.fragments[fragNum] + view.mu.Unlock() + err = op.ProcessFragment(frag) + if err != nil { + return err + } + } + } + } + } + return nil +} diff --git a/holder_internal_test.go b/holder_internal_test.go index ccc52c5b0..5ae365362 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 Pilosa Corp. +// Copyright 2020 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,313 +15,163 @@ package pilosa import ( + "context" "io/ioutil" "os" - "path/filepath" - "reflect" - "strings" "testing" - "time" - - "github.com/pilosa/pilosa/v2/roaring" ) -type tHolder struct { - *Holder +type testHolderOperator struct { + indexSeen, indexProcessed int + fieldSeen, fieldProcessed int + viewSeen, viewProcessed int + fragmentSeen, fragmentProcessed int + waitHere chan struct{} } -// Close closes the holder and removes all underlying data. -func (h *tHolder) Close() error { - defer os.RemoveAll(h.Path) - return h.Holder.Close() +func (t *testHolderOperator) CheckIndex(string) (bool, bool) { + t.indexSeen++ + return true, true } -// Reopen instantiates and opens a new holder. -// Note that the holder must be Closed first. -func (h *tHolder) Reopen() error { - path, logger := h.Path, h.Holder.Logger - h.Holder = NewHolder(DefaultPartitionN) - h.Holder.Path = path - h.Holder.Logger = logger - return h.Holder.Open() +func (t *testHolderOperator) CheckField(string, string) (bool, bool) { + t.fieldSeen++ + return true, true } -func newHolder() *tHolder { - path, err := ioutil.TempDir(*TempDir, "pilosa-") +func (t *testHolderOperator) CheckView(string, string, string) (bool, bool) { + t.viewSeen++ + return true, true +} + +func (t *testHolderOperator) CheckFragment(string, string, string, uint64) bool { + t.fragmentSeen++ + return true +} + +func (t *testHolderOperator) ProcessIndex(*Index) error { + t.indexProcessed++ + return nil +} + +func (t *testHolderOperator) ProcessField(*Field) error { + t.fieldProcessed++ + return nil +} + +func (t *testHolderOperator) ProcessView(*view) error { + t.viewProcessed++ + return nil +} + +func (t *testHolderOperator) ProcessFragment(*fragment) error { + if t.waitHere != nil { + <-t.waitHere + } + t.fragmentProcessed++ + return nil +} + +func makeHolder() (*Holder, string, error) { + path, err := ioutil.TempDir("", "pilosa-") if err != nil { - panic(err) + return nil, "", err } - - h := &tHolder{Holder: NewHolder(DefaultPartitionN)} - h.Path = path - return h -} - -// MustCreateFieldIfNotExists returns a given field. Panic on error. -func (h *tHolder) MustCreateFieldIfNotExists(index, field string) *Field { - f, err := h.MustCreateIndexIfNotExists(index, IndexOptions{}).CreateFieldIfNotExists(field, OptFieldTypeDefault()) - if err != nil { - panic(err) - } - return f -} - -// MustCreateIndexIfNotExists returns a given index. Panic on error. -func (h *tHolder) MustCreateIndexIfNotExists(index string, opt IndexOptions) *Index { - idx, err := h.Holder.CreateIndexIfNotExists(index, opt) - if err != nil { - panic(err) - } - return idx -} - -// SetBit clears a bit on the given field. -func (h *tHolder) SetBit(index, field string, rowID, columnID uint64) { - f := h.MustCreateFieldIfNotExists(index, field) - _, err := f.SetBit(rowID, columnID, nil) - if err != nil { - panic(err) - } -} - -// Row returns a Row for a given field. -func (h *tHolder) Row(index, field string, rowID uint64) *Row { - f := h.MustCreateFieldIfNotExists(index, field) - row, err := f.Row(rowID) - if err != nil { - panic(err) - } - return row -} - -func TestHolder_Optn(t *testing.T) { - t.Run("ErrViewPermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - availableShardFileFlushDuration.Set(100 * time.Millisecond) - h := newHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { - t.Fatal(err) - } - defer func() { - // we don't care about a failure here - _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0755) - }() - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - t.Run("ErrViewFragmentsMkdir", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := newHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := field.createViewIfNotExists(viewStandard); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { - t.Fatal(err) - } - defer func() { - // we don't care about a failure here - _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0755) - }() - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrFragmentCachePermission", func(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("Skipping permissions test since user is root.") - } - h := newHolder() - defer h.Close() - - if idx, err := h.CreateIndex("foo", IndexOptions{}); err != nil { - t.Fatal(err) - } else if field, err := idx.CreateField("bar", OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if view, err := field.createViewIfNotExists(viewStandard); err != nil { - t.Fatal(err) - } else if _, err := field.SetBit(0, 0, nil); err != nil { - t.Fatal(err) - } else if err := view.Fragment(0).FlushCache(); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { - t.Fatal(err) - } - defer func() { - _ = os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0644) - }() - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("unexpected error: %s", err) - } - }) - -} - -// Ensure holder can clean up orphaned fragments. -func TestHolderCleaner_CleanHolder(t *testing.T) { - availableShardFileFlushDuration.Set(100 * time.Millisecond) //shorten the default time to force a file write - cluster := NewTestCluster(2) - - // Create a local holder. - hldr0 := newHolder() - defer hldr0.Close() - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - cluster.nodes[0].URI = NewTestURIFromHostPort("localhost", 0) - - // Create fields on nodes. - for _, hldr := range []*tHolder{hldr0} { - hldr.MustCreateFieldIfNotExists("i", "f") - hldr.MustCreateFieldIfNotExists("i", "f0") - hldr.MustCreateFieldIfNotExists("y", "z") - } - - // Set data on the local holder. - hldr0.SetBit("i", "f", 0, 10) - hldr0.SetBit("i", "f", 0, 4000) - hldr0.SetBit("i", "f", 2, 20) - hldr0.SetBit("i", "f", 3, 10) - hldr0.SetBit("i", "f", 120, 10) - hldr0.SetBit("i", "f", 200, 4) - - hldr0.SetBit("i", "f0", 9, ShardWidth+5) - - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+4) - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+5) - hldr0.SetBit("y", "z", 10, (2*ShardWidth)+7) - - // Set highest shard. - err := hldr0.Field("i", "f").AddRemoteAvailableShards(roaring.NewBitmap(0, 1)) - if err != nil { - t.Fatalf("adding remote shards: %v", err) - } - err = hldr0.Field("y", "z").AddRemoteAvailableShards(roaring.NewBitmap(0, 1, 2)) - if err != nil { - t.Fatalf("adding remote shards: %v", err) - } - time.Sleep(2 * availableShardFileFlushDuration.Get()) - - // Keep replication the same and ensure we get the expected results. - cluster.ReplicaN = 2 - - // Set up cleaner for replication 2. - cleaner2 := holderCleaner{ - Node: cluster.nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner2.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*tHolder{hldr0} { - if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) - } - - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } - - // Change replication factor to ensure we have fragments to remove. - cluster.ReplicaN = 1 - - // Set up cleaner for replication 1. - cleaner1 := holderCleaner{ - Node: cluster.nodes[0], - Holder: hldr0.Holder, - Cluster: cluster, - } - - if err := cleaner1.CleanHolder(); err != nil { - t.Fatal(err) - } - - // Verify data is the same on both nodes. - for i, hldr := range []*tHolder{hldr0} { - if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) - } - - f := hldr.fragment("i", "f0", viewStandard, 1) - if f != nil { - t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) - } - - if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * ShardWidth) + 4, (2 * ShardWidth) + 5, (2 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) - } - } -} - -// Ensure holder can reopen. -func TestHolderCleaner_Reopen(t *testing.T) { h := NewHolder(DefaultPartitionN) - h.Path = "path" - err := h.Open() + + return h, path, nil +} + +func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { + idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { - t.Fatalf("couldn't open holder: %v", err) + t.Fatalf("creating index: %v", err) } - err = h.Close() + f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault()) if err != nil { - t.Fatalf("couldn't close holder: %v", err) + t.Fatalf("setting bit: %v", err) } - err = h.Open() + _, err = f.SetBit(rowID, columnID, nil) if err != nil { - t.Fatalf("couldn't open holder: %v", err) - } - err = h.Close() - if err != nil { - t.Fatalf("couldn't close holder: %v", err) + t.Fatalf("setting bit: %v", err) + } +} + +func TestHolderOperatorProcess(t *testing.T) { + h, path, err := makeHolder() + if err != nil { + t.Fatalf("creating holder: %v", err) + } + defer os.RemoveAll(path) + defer h.Close() + + // Write bits to separate indexes. + testSetBit(t, h, "i0", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 12345678) + + testOp := testHolderOperator{} + ctx := context.Background() + err = h.Process(ctx, &testOp) + if err != nil { + t.Fatalf("processing holder: %v", err) + } + expected := testHolderOperator{ + indexSeen: 2, indexProcessed: 2, + fieldSeen: 2, fieldProcessed: 2, + viewSeen: 2, viewProcessed: 2, + fragmentSeen: 3, fragmentProcessed: 3, + } + if testOp != expected { + t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) + } +} + +func TestHolderOperatorCancel(t *testing.T) { + h, path, err := makeHolder() + if err != nil { + t.Fatalf("creating holder: %v", err) + } + defer os.RemoveAll(path) + defer h.Close() + + // Write bits to separate indexes. + testSetBit(t, h, "i0", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 12345678) + + // Here, we want to ensure that the operation gets cancelled + // successfully. In practice we expect it to process one fragment, then + // end up blocked on the waitHere, then get cancelled... But the + // waitHere blockage isn't really something holder.Process can do + // anything about, so we close the channel, so two fragments are + // processed. But in theory you could end up with only one fragment + // processed if this goroutine managed to cancel before the processor + // gets to the next fragment. Point is, it shouldn't hit all three, + // because the checks against the cancellation should fire before it + // gets there. + testOp := testHolderOperator{waitHere: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + err = h.Process(ctx, &testOp) + close(done) + }() + testOp.waitHere <- struct{}{} + cancel() + close(testOp.waitHere) + <-done + if err != context.Canceled { + t.Fatalf("processing holder: expected context.Canceled, got %v", err) + } + testOp.waitHere = nil + expected := testHolderOperator{ + indexSeen: 2, indexProcessed: 2, + fieldSeen: 2, fieldProcessed: 2, + viewSeen: 2, viewProcessed: 2, + fragmentSeen: 3, fragmentProcessed: 3, + } + if testOp == expected { + t.Fatalf("holder processor did not cancel. expected something other than %#v", expected) } } diff --git a/http/handler.go b/http/handler.go index 67be9e297..06a62ebd7 100644 --- a/http/handler.go +++ b/http/handler.go @@ -221,6 +221,8 @@ func (h *Handler) populateValidators() { h.validators["GetTransaction"] = queryValidationSpecRequired() h.validators["PostTransaction"] = queryValidationSpecRequired() h.validators["PostFinishTransaction"] = queryValidationSpecRequired() + h.validators["Inspect"] = queryValidationSpecRequired().Optional("indexes", "fields", "views", "shards", "checksum", "containers") + } type contextKeyQuery int @@ -352,6 +354,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo") + router.HandleFunc("/inspect", handler.handleInspect).Methods("GET").Name("Inspect") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") @@ -590,6 +593,37 @@ func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { } } +func (h *Handler) handleInspect(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + q := r.URL.Query() + _, checksum := q["checksum"] + _, containers := q["containers"] + req := pilosa.InspectRequest{ + HolderFilterParams: pilosa.HolderFilterParams{ + Indexes: q.Get("indexes"), + Fields: q.Get("fields"), + Views: q.Get("views"), + Shards: q.Get("shards"), + }, + InspectRequestParams: pilosa.InspectRequestParams{ + Checksum: checksum, + Containers: containers, + }, + } + info, err := h.api.Inspect(r.Context(), &req) + if err != nil { + http.Error(w, fmt.Sprintf("inspect request: %v", err), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + h.logger.Printf("write inspect response error: %s", err) + } +} + type getSchemaResponse struct { Indexes []*pilosa.IndexInfo `json:"indexes"` } diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 64a6d217e..a43b3eee8 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -45,35 +45,47 @@ type Container struct { type containerFlags uint8 +var containerFlagStrings = [...]string{ + "", + "mapped", + "frozen", + "frozen/mapped", + "pristine", + "pristine/mapped", + "pristine/frozen", + "pristine/frozen/mapped", +} + +func (f containerFlags) String() string { + return containerFlagStrings[f&7] +} + const ( flagMapped = containerFlags(1 << iota) flagFrozen + flagPristine ) func (c *Container) String() string { if c == nil { return "" } - froze := "" - switch c.flags { - case flagFrozen: - froze = "frozen " - case flagMapped: - froze = "mapped " - case flagFrozen | flagMapped: - froze = "frozen/mapped" + var space, froze string + if c.flags != 0 { + space = " " + froze = c.flags.String() } switch c.typeID { case containerArray: - return fmt.Sprintf("<%sarray container, N=%d>", froze, c.N()) + return fmt.Sprintf("<%s%sarray container, N=%d>", froze, space, c.N()) case containerBitmap: - return fmt.Sprintf("<%sbitmap container, N=%d, len %dx uint64>", - froze, c.N(), len(c.bitmap())) + return fmt.Sprintf("<%s%sbitmap container, N=%d, len %dx uint64>", + froze, space, c.N(), len(c.bitmap())) case containerRun: - return fmt.Sprintf("<%srun container, N=%d, len %dx interval>", - froze, c.N(), len(c.runs())) + return fmt.Sprintf("<%s%srun container, N=%d, len %dx interval>", + froze, space, c.N(), len(c.runs())) default: - return fmt.Sprintf("", froze, c.typeID, c.N()) + return fmt.Sprintf("", froze, space, c.typeID, c.N()) } } @@ -292,6 +304,7 @@ func (c *Container) unmapOrClone() *Container { return c.Clone() } c.flags &^= flagMapped + c.flags &^= flagPristine // mapped: we want to unmap the storage. switch c.typeID { case containerArray: @@ -368,6 +381,7 @@ func (c *Container) setArrayMaybeCopy(array []uint16, doCopy bool) { if len(array) > 1<<16 { panic("impossibly large array") } + c.flags &^= flagPristine // array we can fit in data store: if len(array) <= stashedArraySize { copy(c.data[:stashedArraySize], array) @@ -497,6 +511,7 @@ func (c *Container) setBitmap(bitmap []uint64) { panic("illegal bitmap length") } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&bitmap[0])), bitmapN, bitmapN + c.flags &^= flagPristine } // runs yields the data viewed as a slice of intervals. @@ -531,6 +546,7 @@ func (c *Container) setRunsMaybeCopy(runs []interval16, doCopy bool) { if len(runs) > 1<<15 { panic("impossibly large run set") } + c.flags &^= flagPristine // array we can fit in data store: if len(runs) <= stashedRunSize { newRuns := (*[stashedRunSize]interval16)(unsafe.Pointer(&c.data))[:len(runs)] diff --git a/roaring/roaring.go b/roaring/roaring.go index f79b20c5b..b87f4d6af 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1721,6 +1721,8 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) { // bitmap and yield information about containers, including type, size, and // the location of their data structures. type roaringIterator interface { + // Len reports the number of containers total. + Len() (count int64) // Next yields the information about the next container Next() (key uint64, cType byte, n int, length int, pointer *uint16, err error) // Remaining yields the bytes left over past the end of the roaring data, @@ -1885,6 +1887,11 @@ func (r *baseRoaringIterator) Done(err error) { r.currentDataOffset = 0 } +// Len() indicates the total number of containers the iterator expects to have. +func (r *baseRoaringIterator) Len() int64 { + return r.keys +} + func (r *baseRoaringIterator) Remaining() ([]byte, int64) { if r.lastDataOffset == 0 { return nil, 0 @@ -2436,20 +2443,24 @@ func (b *Bitmap) roaringSize() (int64, int64) { } // Info returns stats for the bitmap. -func (b *Bitmap) Info() BitmapInfo { +func (b *Bitmap) Info(includeContainers bool) BitmapInfo { info := BitmapInfo{ - OpN: b.opN, - Ops: b.ops, - Containers: make([]ContainerInfo, 0, b.Containers.Size()), + OpN: b.opN, + Ops: b.ops, + ContainerCount: b.Containers.Size(), + } + if includeContainers { + info.Containers = make([]ContainerInfo, 0, info.ContainerCount) } - info.ContainerCount = cap(info.Containers) citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() ci := c.info() ci.Key = k info.BitCount += uint64(c.N()) - info.Containers = append(info.Containers, ci) + if includeContainers { + info.Containers = append(info.Containers, ci) + } } return info } @@ -2510,11 +2521,12 @@ func (b *Bitmap) Flip(start, end uint64) *Bitmap { type BitmapInfo struct { OpN int Ops int - OpDetails []OpInfo + OpDetails []OpInfo `json:"OpDetails,omitempty"` BitCount uint64 ContainerCount int - Containers []ContainerInfo // The containers found in the bitmap originally - OpContainers []ContainerInfo // The containers resulting from ops log changes. + Containers []ContainerInfo `json:"Containers,omitempty"` // The containers found in the bitmap originally + OpContainers []ContainerInfo `json:"OpContainers,omitempty"` // The containers resulting from ops log changes. + From, To uintptr // if set, indicates the address range used when unpacking } // Iterator represents an iterator over a Bitmap. @@ -3718,6 +3730,7 @@ func (c *Container) info() ContainerInfo { info.Alloc = 0 return info } + info.Flags = c.flags.String() if c.isArray() { info.Type = "array" @@ -3799,6 +3812,7 @@ func (c *Container) bitmapRepair() { type ContainerInfo struct { Key uint64 // container key Type string // container type (array, bitmap, or run) + Flags string // flag state N int32 // number of bits Alloc int // memory used Pointer uintptr // address diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 55395ea73..7dda0f5c6 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1733,7 +1733,7 @@ type benchmarkSampleData struct { var sampleData benchmarkSampleData func isAllType(b *roaring.Bitmap, typ string) bool { - bi := b.Info() + bi := b.Info(true) for _, c := range bi.Containers { if c.Type != typ { return false diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index ee566236a..aeab0ef2f 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -17,6 +17,7 @@ package roaring import ( "errors" "io" + "unsafe" ) // UnmarshalBinary reads Pilosa's format, or upstream roaring (mostly; @@ -99,9 +100,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) (err error) { // InspectBinary reads a roaring bitmap, plus a possible ops log, // and reports back on the contents, including distinguishing between // the original ops log and the post-ops-log contents. -func InspectBinary(data []byte) (info BitmapInfo, err error) { +func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappedAny bool, err error) { + b = NewFileBitmap() + b.PreferMapping(mapped) if data == nil { - return info, errors.New("no roaring bitmap provided") + return b, mappedAny, errors.New("no roaring bitmap provided") } var itr roaringIterator var itrKey uint64 @@ -113,13 +116,13 @@ func InspectBinary(data []byte) (info BitmapInfo, err error) { itr, err = newRoaringIterator(data) if err != nil { - return info, err + return b, mappedAny, err } if itr == nil { - return info, errors.New("failed to create roaring iterator, but don't know why") + return b, mappedAny, errors.New("failed to create roaring iterator, but don't know why") } - - b := NewFileBitmap() + keys := itr.Len() + info.Containers = make([]ContainerInfo, 0, keys) itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() for itrErr == nil { @@ -131,22 +134,54 @@ func InspectBinary(data []byte) (info BitmapInfo, err error) { pointer: itrPointer, flags: flagMapped, } + if !mapped { + newC.unmapOrClone() + } + newC.flags |= flagPristine + if newC.flags&flagMapped != 0 { + mappedAny = true + } + var size int b.Containers.Put(itrKey, newC) + switch itrCType { + case containerArray: + size = int(newC.n) * 2 + case containerBitmap: + size = 8192 + case containerRun: + size = itrLen*interval16Size + runCountHeaderSize + } + info.Containers = append(info.Containers, ContainerInfo{ + N: newC.n, + Mapped: newC.flags&flagMapped != 0, + Type: containerTypeNames[itrCType], + Alloc: size, + Pointer: uintptr(unsafe.Pointer(newC.pointer)), + Key: itrKey, + Flags: newC.flags.String(), + }) + info.ContainerCount++ + info.BitCount += uint64(newC.n) itrKey, itrCType, itrN, itrLen, itrPointer, itrErr = itr.Next() } // note: if we get a non-EOF err, it's possible that we made SOME // changes but didn't log them. I don't have a good solution to this. if itrErr != io.EOF { - return info, itrErr + return b, mappedAny, itrErr } - - // gather initial info - info = b.Info() + // stash pointer ranges + info.From = uintptr(unsafe.Pointer(&data[0])) + info.To = info.From + uintptr(len(data)) // Read ops log until the end of the file. b.ops = 0 b.opN = 0 buf, lastValidOffset := itr.Remaining() + // if there's no ops log, we're done and can just return the + // info so far. + if len(buf) == 0 { + return b, mappedAny, err + } for { // Exit when there are no more ops to parse. if len(buf) == 0 { @@ -155,36 +190,56 @@ func InspectBinary(data []byte) (info BitmapInfo, err error) { // Unmarshal the op and apply it. var opr op - if err := opr.UnmarshalBinary(buf); err != nil { - return info, err + if err = opr.UnmarshalBinary(buf); err != nil { + // we break out here, but we continue on to + // return the bitmap as-is, along with data about + // it, and the error. this lets us share the + // "is anything mapped" check with that code. + break } opr.apply(b) // Increase the op count. - info.Ops++ - info.OpN += opr.count() - info.OpDetails = append(info.OpDetails, opr.info()) - + if info != nil { + info.Ops++ + info.OpN += opr.count() + info.OpDetails = append(info.OpDetails, opr.info()) + } // Move the buffer forward. opSize := opr.size() buf = buf[opSize:] lastValidOffset += int64(opSize) } + citer, _ := b.Containers.Iterator(0) + // it's possible the ops log unmapped every mapped container, so we recheck. + mappedAny = false + if info == nil { + for citer.Next() { + _, c := citer.Value() + if c.Mapped() { + mappedAny = true + break + } + } + return b, mappedAny, err + } // now we want to compute the actual container and bit counts after // ops, and create a report of just the containers which got changed. info.ContainerCount = 0 info.BitCount = 0 - citer, _ := b.Containers.Iterator(0) for citer.Next() { k, c := citer.Value() + if c.Mapped() { + mappedAny = true + } info.ContainerCount++ info.BitCount += uint64(c.N()) - if c.Mapped() { + if c.flags&flagPristine != 0 { continue } ci := c.info() ci.Key = k info.OpContainers = append(info.OpContainers, ci) } - return info, nil + return b, mappedAny, err } diff --git a/server.go b/server.go index 75f5b2024..0afd2fa43 100644 --- a/server.go +++ b/server.go @@ -565,6 +565,7 @@ func (s *Server) Open() error { return errors.Wrap(err, "opening Holder") } // bring up the background tasks for the holder. + s.holder.SnapshotQueue = s.snapshotQueue s.holder.Activate() if err := s.cluster.setNodeState(nodeStateReady); err != nil { return errors.Wrap(err, "setting nodeState") diff --git a/snapshotqueue.go b/snapshotqueue.go index 5dcc5dbf8..1bdcdb7a9 100644 --- a/snapshotqueue.go +++ b/snapshotqueue.go @@ -15,7 +15,9 @@ package pilosa import ( + "context" "fmt" + "io" "math/bits" "os" "sync" @@ -64,7 +66,8 @@ type SnapshotQueue interface { type queuelessSnapshotQueue struct{} func (q *queuelessSnapshotQueue) Enqueue(f *fragment) { - _ = f.snapshot() + // We don't actually try to enqueue the snapshot; it breaks things + // if a snapshot gets caused during a transaction. } func (q *queuelessSnapshotQueue) Await(f *fragment) error { @@ -86,7 +89,16 @@ var defaultSnapshotQueue = &queuelessSnapshotQueue{} // newSnapshotQueue makes a new snapshot queue, of depth N, with // w worker threads. func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue { - sq := prioritySnapshotQueue{normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), done: make(chan struct{}), maxOpN: 10000, logger: l} + ctx, cancel := context.WithCancel(context.Background()) + sq := prioritySnapshotQueue{ + normal: make(chan snapshotRequest, n), + urgent: make(chan snapshotRequest), + background: make(chan snapshotRequest), + ctx: ctx, + cancel: cancel, + maxOpN: 10000, + logger: l, + } if sq.logger == nil { sq.logger = logger.NewStandardLogger(os.Stderr) } @@ -114,11 +126,12 @@ type prioritySnapshotQueue struct { urgent chan snapshotRequest normal chan snapshotRequest background chan snapshotRequest - done chan struct{} + ctx context.Context + cancel context.CancelFunc mu sync.RWMutex scanWG, workerWG sync.WaitGroup maxOpN int - observedOpN [16]int + observedOpN [16]uint32 stats struct { enqueued uint32 skipped uint32 @@ -128,38 +141,37 @@ type prioritySnapshotQueue struct { func (sq *prioritySnapshotQueue) spawnWorkers(w int) { sq.mu.Lock() defer sq.mu.Unlock() - if sq.done == nil { - sq.logger.Printf("prioritySnapshotQueue worker: no done channel, already done?") + if sq.ctx.Err() != nil { + sq.logger.Printf("prioritySnapshotQueue worker: already done") return } sq.workerWG.Add(w) for i := 0; i < w; i++ { - go sq.worker(sq.urgent, sq.normal, sq.background, sq.done) + go sq.worker(sq.ctx, sq.urgent, sq.normal, sq.background) } } -func (sq *prioritySnapshotQueue) worker(urgent, normal, background chan snapshotRequest, done chan struct{}) { - // We don't want a race condition on these. If they're non-nil when - // we get them, they should get closed at some point. If done is - // already nil, we shouldn't do anything. +func (sq *prioritySnapshotQueue) worker(ctx context.Context, urgent, normal, background chan snapshotRequest) { defer sq.workerWG.Done() + done := ctx.Done() ok := true var req snapshotRequest for ok { req.frag = nil - select { + case _, ok = <-done: case req, ok = <-urgent: default: select { + case _, ok = <-done: case req, ok = <-urgent: case req, ok = <-normal: default: select { + case _, ok = <-done: case req, ok = <-urgent: case req, ok = <-normal: case req, ok = <-background: - case _, ok = <-done: } } } @@ -193,10 +205,9 @@ func (sq *prioritySnapshotQueue) process(req snapshotRequest) { func (sq *prioritySnapshotQueue) Stop() { sq.mu.Lock() defer sq.mu.Unlock() - close(sq.done) + sq.cancel() // scanners need to be done before we close the other channels. sq.scanWG.Wait() - sq.done = nil close(sq.normal) sq.normal = nil close(sq.urgent) @@ -216,6 +227,7 @@ func (sq *prioritySnapshotQueue) Enqueue(f *fragment) { if f.snapshotPending { return } + sq.observeOpN(uint32(f.opN)) sq.mu.RLock() defer sq.mu.RUnlock() if sq.normal == nil { @@ -270,6 +282,7 @@ func (sq *prioritySnapshotQueue) Immediate(f *fragment) error { return errors.New("requested immediate snapshot after snapshot queue was closed") } f.snapshotPending = true + sq.observeOpN(uint32(f.opN)) req := snapshotRequest{frag: f, when: time.Now()} // if the fragment was already in the work queue, it's *possible* // that the only available worker just picked it off the queue, and @@ -286,35 +299,6 @@ func (sq *prioritySnapshotQueue) Immediate(f *fragment) error { return sq.Await(f) } -// needsSnapshot determines whether a fragment probably wants snapshotting. -// Specifically, it looks for fragments not already marked to receive -// snapshots, but which have a high enough opN to justify a snapshot. This -// is only used from the background scan. -func (sq *prioritySnapshotQueue) needsSnapshot(f *fragment) bool { - if f == nil { - return false - } - f.mu.Lock() - defer f.mu.Unlock() - if f.snapshotPending { - return false - } - if f.opN > sq.maxOpN { - return true - } - // aka "log2(n) + 1", or 0 for n==0 - pow2 := 32 - bits.LeadingZeros32(uint32(f.opN)) - // 15 == 16384. we assume that since 16384 is higher than our - // normal maxOpN, it's always a reasonable value. - if pow2 > 15 { - pow2 = 15 - } - // store in inverse order so the lowest slot in the array is the - // highest cardinality - sq.observedOpN[15-pow2]++ - return false -} - // ScanHolder spawns a goroutine which iterates through the holder's // indexes/fields/views/fragments, looking for fragments which have OpN // high enough to justify a snapshot but don't seem to have one pending. @@ -326,143 +310,191 @@ func (sq *prioritySnapshotQueue) ScanHolder(h *Holder, done chan struct{}) { sq.mu.Unlock() } +// observeOpN reports that a given value of opN was "observed", meaning, +// we encountered a fragment which had that value. This happens for every +// enqueue/immediate, including enqueue attempts which fail to actually +// enter the queue, and it also happens for fragments noticed by the background +// scan but which don't have high enough opN to trigger a snapshot. +func (sq *prioritySnapshotQueue) observeOpN(n uint32) { + // aka "log2(n) + 1", or 0 for n==0 + pow2 := 32 - bits.LeadingZeros32(n) + // 15 == 16384. Our usual fragment maxOpN is 10k, so most fragments + // should end up in the 8k-16k bucket, rather than the 16k+ bucket, + // unless we've got a lot of ingests with large batches going on, + // in which case the 16k bucket will win. + if pow2 > 15 { + pow2 = 15 + } + // store in inverse order so the lowest slot in the array is the + // highest cardinality + atomic.AddUint32(&sq.observedOpN[15-pow2], 1) +} + +// computeMaxOpN tries to pick a reasonable new maxOpN for the background +// scan to use. On a quiet system, we want to gradually lower opN, picking +// the fragments with the highest opN values first, because those offer the +// largest benefit. So, whenever we check a fragment in the background, if we +// *don't* snapshot it, we'll "observe" its OpN value, and then we pick a +// value which picks up at least 1/4 of them. +// +// If there's ingest activity, the Immediate and Enqueue operations will +// "observe" the OpN of fragments submitted to them. This can drive OpN back +// up, if those fragments frequently have very high opN values, which reflects +// the fact that we have enough of that activity that we don't need the +// background scanner adding more. +// +// If we have enough ingest activity that the background scanner never actually +// gets to submit work, we'll rarely get here, because the background scanner +// will block until there's no snapshots pending for the normal workload. +// When we do, we'll probably pick a MaxOpN which is dominated by the ingest +// workload's opN values. So for instance, if everything coming in from the +// ingest workload has 10k or more items, because that's the default fragment +// maxOpN, that will probably set the background snapshot queue value to 8k. +func (sq *prioritySnapshotQueue) computeMaxOpN() { + sq.logger.Debugf("observedOpN by power of 2: %d\n", sq.observedOpN[:]) + total := uint32(0) + for i := range sq.observedOpN { + total += atomic.LoadUint32(&sq.observedOpN[i]) + } + target := (total / 4) + 1 + subTotal := uint32(0) + for i := range sq.observedOpN { + v := atomic.LoadUint32(&sq.observedOpN[i]) + subTotal += v + if subTotal >= target { + prevMaxOpN := sq.maxOpN + sq.maxOpN = (1 << (15 - uint(i))) / 2 + if sq.maxOpN > 0 { + sq.maxOpN-- + } + if prevMaxOpN != sq.maxOpN { + sq.logger.Printf("background scan: %d/%d fragments considered have opN %d or higher\n", + subTotal, total, sq.maxOpN) + } + break + } + } + // It's conceptually possible that we'll miss a couple of observations + // here but that's not really important. This is all pretty approximate. + for i := range sq.observedOpN { + atomic.StoreUint32(&sq.observedOpN[i], 0) + } +} + +// prioritySnapshotQueueScanner is the data type that implements HolderOperator +// and represents a single scan of a holder, with a given maxOpN. +type prioritySnapshotQueueScanner struct { + HolderFilterAll + HolderProcessNone + sq *prioritySnapshotQueue + holder *Holder + queue chan snapshotRequest + ctx context.Context + maxOpN int + seen, hits, counter int +} + +func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error { + if f == nil { + return nil + } + s.seen++ + // we can't defer this reasonably, because otherwise we'll keep + // the fragment locked forever if we end up trying to send it + // to the queue, but the workers are busy on other fragments. + f.mu.Lock() + open := f.open + snapshotPending, opN := f.snapshotPending, f.opN + f.mu.Unlock() + + // a pending snapshot is one that is either in the normal or + // immediate queue, or is trying to get into the normal queue + // and about to fail, but either way, it already got observed + // there, so we don't need to observe it here. A closed fragment + // doesn't matter to us -- it should be a transient state that + // happens during a shutdown, or shouldn't happen, but we don't + // care about it. + if snapshotPending || !open { + return nil + } + if opN <= s.maxOpN { + // observe the value but don't do a snapshot + s.sq.observeOpN(uint32(opN)) + s.counter++ + if s.counter == 1000 { + select { + case <-time.After(1 * time.Second): + case <-s.ctx.Done(): + return io.EOF + } + s.counter = 0 + } + return nil + } + // we don't observe values when we decide to trigger a snapshot, + // because those values will be changing anyway. we could also + // observe them as zero, but that's also sort of wrong. + s.hits++ + select { + case s.queue <- snapshotRequest{frag: f, when: time.Now()}: + s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path) + case <-s.ctx.Done(): + return io.EOF + } + return nil + +} + +func contextMergedWithStructChan(ctx context.Context, ch chan struct{}) (context.Context, context.CancelFunc) { + canCancel, cancel := context.WithCancel(ctx) + go func() { + select { + case <-ctx.Done(): + cancel() + case <-ch: + cancel() + case <-canCancel.Done(): + // don't need to cancel, but do need to exit this + // function + } + }() + return canCancel, cancel +} + // scanHolderWorker is a background task that scans a holder looking for // fragments which need snapshots taken. It's the cleanup task for snapshots // that would have been requested by Enqueue, but the queue was full. func (sq *prioritySnapshotQueue) scanHolderWorker(h *Holder, background chan snapshotRequest, done chan struct{}) { - // queueDone is global to this snapshotQueue, done is specific to this holder - // scanner. - queueDone := sq.done defer sq.scanWG.Done() - var indexNames, fieldNames, viewNames []string - var fragNums []uint64 + ctx, cancel := contextMergedWithStructChan(sq.ctx, done) + defer cancel() + scanner := &prioritySnapshotQueueScanner{ + sq: sq, + holder: h, + queue: background, + ctx: sq.ctx, + maxOpN: sq.maxOpN, + } for { - // To avoid abusing things, cap activity rate; every time we finish - // the holder, or every couple hundred fragments considered, we - // pause for a bit. - counter := 0 - hits := 0 - h.mu.Lock() - indexNames = indexNames[:0] - for indexName := range h.indexes { - indexNames = append(indexNames, indexName) + err := h.Process(ctx, scanner) + if err != nil { + return } - h.mu.Unlock() - for _, indexName := range indexNames { - h.mu.Lock() - index := h.indexes[indexName] - h.mu.Unlock() - if index == nil { - continue - } - fieldNames = fieldNames[:0] - index.mu.Lock() - for fieldName := range index.fields { - fieldNames = append(fieldNames, fieldName) - } - index.mu.Unlock() - for _, fieldName := range fieldNames { - index.mu.Lock() - field := index.fields[fieldName] - index.mu.Unlock() - if field == nil { - continue - } - viewNames = viewNames[:0] - field.mu.Lock() - for viewName := range field.viewMap { - viewNames = append(viewNames, viewName) - } - field.mu.Unlock() - for _, viewName := range viewNames { - field.mu.Lock() - view := field.viewMap[viewName] - field.mu.Unlock() - if view == nil { - continue - } - fragNums := fragNums[:0] - view.mu.Lock() - for fragNum := range view.fragments { - fragNums = append(fragNums, fragNum) - } - view.mu.Unlock() - for _, fragNum := range fragNums { - view.mu.Lock() - frag := view.fragments[fragNum] - view.mu.Unlock() - if sq.needsSnapshot(frag) { - hits++ - select { - case background <- snapshotRequest{frag: frag, when: time.Now()}: - sq.logger.Debugf("found fragment needing snapshot: %s\n", frag.path) - case <-done: - return - } - } else { - // Count fragments examined *without* finding anything that - // needed a snapshot. When we find things that need snapshots, - // the time it takes the workers to respond to us is enough - // of a delay to keep us from eating every CPU. So, if a lot - // of things need snapshots, and the workers aren't doing - // anything else, ScanHolder will mostly keep them saturated. - // If they're busy, we'll block forever in the write to the - // background queue. If there's nothing that needs snapshots, - // we pause frequently for a second or so at a time. - counter++ - if counter == 1000 { - select { - case <-time.After(1 * time.Second): - case <-queueDone: - return - case <-done: - return - } - counter = 0 - } - } - } - } - } - } - if hits > 0 { - sq.logger.Printf("background scan: %d fragments needed snapshots\n", hits) - hits = 0 + + if scanner.hits > 0 { + sq.logger.Printf("background scan: %d/%d fragments needed snapshots\n", scanner.hits, scanner.seen) + scanner.hits = 0 } else { sq.logger.Debugf("background scan: no fragments needed snapshots, waiting\n") // No reason to be active if we're not finding anything. select { case <-time.After(60 * time.Second): - case <-queueDone: - return - case <-done: + case <-ctx.Done(): return } } - sq.logger.Debugf("observedOpN by power of 2: %d\n", sq.observedOpN[:]) - total := 0 - for _, v := range sq.observedOpN { - total += v - } - target := total / 4 - subTotal := 0 - for i, v := range sq.observedOpN { - subTotal += v - if subTotal >= target { - prevMaxOpN := sq.maxOpN - sq.maxOpN = (1 << (15 - uint(i))) / 2 - if sq.maxOpN > 0 { - sq.maxOpN-- - } - if prevMaxOpN != sq.maxOpN { - sq.logger.Printf("background scan: %d/%d fragments considered have opN %d or higher\n", - subTotal, total, sq.maxOpN) - } - break - } - } - for i := range sq.observedOpN { - sq.observedOpN[i] = 0 - } + scanner.seen = 0 + sq.computeMaxOpN() + scanner.maxOpN = sq.maxOpN } }