mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
ceaf5c15d1
This is logically two separate things, but the individual changes are thoroughly intertwined in the code. The first change is a logical change to the design of the snapshot queue, which is that it now adjusts the maxOpN the background scan targets, allowing it to lower that value over time when things are quiet. We do this because it turns out that on large data sets, this can make a factor-of-four difference in memory usage! So, in general, on a quiet system, each pass through the holder aims for about 1/4 of the existing fragments to get snapshotted. When there's more load, we adjust those values up. We also make the snapshot queue a bit less chatty, to make testing less annoying -- we only print stats if the queue enqueues at least two snapshots, or skips any. The second change is threading the holder through things. We've always threaded the logger through, and then added the snapshot queue, and some of the Inspect-related work led to wanting to have a way to thread options through, so what if we just threaded the holder itself through, and removed the direct copying around of the logger, snapshot queue, and so on. Similarly, everything can now use holder.PartitionN instead of having to get its own copy of PartitionN handed out to each index. This does imply ensuring that test cases always get a reasonable default holder. This is a precursor to adding additional information to the holder, such as whether it's in a special read-only mode, which would imply not modifying on-disk files. This is already semi-supported for the specific case of the background snapshot queue and cache flushing, which are attached to the (created in a previous commit) new holder Activate method, instead of being automatic on holder Open. The change to a snapshot queue can also cause races in tests, because the fragment.Clean method's "sanity check" accesses a fragment without a lock. Fix that. Since there's a couple of t.Fatalf(), but we need to release the lock before closing, we use an anonymous function with a defer to handle that. Whee!
117 lines
2.9 KiB
Go
117 lines
2.9 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// mustOpenView returns a new instance of View with a temporary path.
|
|
func mustOpenView(index, field, name string) *view {
|
|
path, err := ioutil.TempDir(*TempDir, "pilosa-view-")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
fo := FieldOptions{
|
|
CacheType: DefaultCacheType,
|
|
CacheSize: DefaultCacheSize,
|
|
}
|
|
|
|
v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo)
|
|
if err := v.open(); err != nil {
|
|
panic(err)
|
|
}
|
|
v.rowAttrStore = &memAttrStore{
|
|
store: make(map[uint64]map[string]interface{}),
|
|
}
|
|
return v
|
|
}
|
|
|
|
// Ensure view can open and retrieve a fragment.
|
|
func TestView_DeleteFragment(t *testing.T) {
|
|
v := mustOpenView("i", "f", "v")
|
|
defer v.close()
|
|
|
|
shard := uint64(9)
|
|
|
|
// Create fragment.
|
|
fragment, err := v.CreateFragmentIfNotExists(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if fragment == nil {
|
|
t.Fatal("expected fragment")
|
|
}
|
|
|
|
err = v.deleteFragment(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if v.Fragment(shard) != nil {
|
|
t.Fatal("fragment still exists in view")
|
|
}
|
|
|
|
// Recreate fragment with same shard, verify that the old fragment was not reused.
|
|
fragment2, err := v.CreateFragmentIfNotExists(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if fragment == fragment2 {
|
|
t.Fatal("failed to create new fragment")
|
|
}
|
|
}
|
|
|
|
// Ensure that simultaneous attempts to grab a new fragment don't clash even
|
|
// if the broadcast operation takes a bit of time.
|
|
func TestView_CreateFragmentRace(t *testing.T) {
|
|
var creates errgroup.Group
|
|
v := mustOpenView("i", "f", "v")
|
|
defer v.close()
|
|
|
|
// Use a broadcaster which intentionally fails.
|
|
v.broadcaster = delayBroadcaster{delay: 10 * time.Millisecond}
|
|
|
|
shard := uint64(0)
|
|
|
|
creates.Go(func() error {
|
|
_, err := v.CreateFragmentIfNotExists(shard)
|
|
return err
|
|
})
|
|
creates.Go(func() error {
|
|
_, err := v.CreateFragmentIfNotExists(shard)
|
|
return err
|
|
})
|
|
err := creates.Wait()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
// delayBroadcaster is a nopBroadcaster with a configurable delay.
|
|
type delayBroadcaster struct {
|
|
nopBroadcaster
|
|
delay time.Duration
|
|
}
|
|
|
|
// SendSync is an implementation of Broadcaster SendSync which delays for a
|
|
// specified interval before succeeding.
|
|
func (d delayBroadcaster) SendSync(Message) error {
|
|
time.Sleep(d.delay)
|
|
return nil
|
|
}
|