mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
dde6954de4
There existed a case where two goroutines would try to CreateIfNotExists the same fragment, and the first would create it, but not put it in the fragments table, then drop the lock, try to broadcast a message, and if it succeeded then populate the fragments table. The second would come along during the broadcast, not find an entry, try to create one, and fail because the file was already locked. Basic problem: At least one test in server/ will fail if we don't delay to send out broadcast messages. Everything will lock up if we can wait forever (or even just a very long time) for the message broadcast. We don't ever want to have an inconsistent state -- so we don't want to either fail to get a fragment when one's been created, or get one that's about to be deleted if the broadcast fails. So, creation and stashing in the fragments table is atomic and immediate. After that, we optimistically attempt to broadcast. If we fail, we fail. We delay up to about 50ms for the broadcast to be done, but after that return anyway. This way, if things are going well everything works, and if there's unexpected delays, things work except some nodes in a cluster may not know about available shards on other nodes sometimes. But that would have happened anyway. A proper fix is beyond the scope of this patch.
117 lines
2.8 KiB
Go
117 lines
2.8 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
// mustOpenView returns a new instance of View with a temporary path.
|
|
func mustOpenView(index, field, name string) *view {
|
|
path, err := ioutil.TempDir(*TempDir, "pilosa-view-")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
fo := FieldOptions{
|
|
CacheType: DefaultCacheType,
|
|
CacheSize: DefaultCacheSize,
|
|
}
|
|
|
|
v := newView(path, index, field, name, fo)
|
|
if err := v.open(); err != nil {
|
|
panic(err)
|
|
}
|
|
v.rowAttrStore = &memAttrStore{
|
|
store: make(map[uint64]map[string]interface{}),
|
|
}
|
|
return v
|
|
}
|
|
|
|
// Ensure view can open and retrieve a fragment.
|
|
func TestView_DeleteFragment(t *testing.T) {
|
|
v := mustOpenView("i", "f", "v")
|
|
defer v.close()
|
|
|
|
shard := uint64(9)
|
|
|
|
// Create fragment.
|
|
fragment, err := v.CreateFragmentIfNotExists(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if fragment == nil {
|
|
t.Fatal("expected fragment")
|
|
}
|
|
|
|
err = v.deleteFragment(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if v.Fragment(shard) != nil {
|
|
t.Fatal("fragment still exists in view")
|
|
}
|
|
|
|
// Recreate fragment with same shard, verify that the old fragment was not reused.
|
|
fragment2, err := v.CreateFragmentIfNotExists(shard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if fragment == fragment2 {
|
|
t.Fatal("failed to create new fragment")
|
|
}
|
|
}
|
|
|
|
// Ensure that simultaneous attempts to grab a new fragment don't clash even
|
|
// if the broadcast operation takes a bit of time.
|
|
func TestView_CreateFragmentRace(t *testing.T) {
|
|
var creates errgroup.Group
|
|
v := mustOpenView("i", "f", "v")
|
|
defer v.close()
|
|
|
|
// Use a broadcaster which intentionally fails.
|
|
v.broadcaster = delayBroadcaster{delay: 10 * time.Millisecond}
|
|
|
|
shard := uint64(0)
|
|
|
|
creates.Go(func() error {
|
|
_, err := v.CreateFragmentIfNotExists(shard)
|
|
return err
|
|
})
|
|
creates.Go(func() error {
|
|
_, err := v.CreateFragmentIfNotExists(shard)
|
|
return err
|
|
})
|
|
err := creates.Wait()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
// delayBroadcaster is a nopBroadcaster with a configurable delay.
|
|
type delayBroadcaster struct {
|
|
nopBroadcaster
|
|
delay time.Duration
|
|
}
|
|
|
|
// SendSync is an implementation of Broadcaster SendSync which delays for a
|
|
// specified interval before succeeding.
|
|
func (d delayBroadcaster) SendSync(Message) error {
|
|
time.Sleep(d.delay)
|
|
return nil
|
|
}
|