FragSpec as struct, fragment fields to avoid dynamic dispatch

- allow inlining of getters index(), view().
 - GOMAXPROCS set to 128

fragment.go

@@ -100,61 +100,49 @@ const (
 	roaringFlagBSIv2 = 0x01 // indicates version using low bit for existence
 )
 
-// FragProxy lets us use either fragProxy or testFragProxy.
-type FragProxy interface {
-	index() string
-	field() string
-	view() string
-	path() string
-}
-
-// fragProxy saves a ton of duplicated strings for
+// fragSpec saves a ton of duplicated strings for
 // path, and index, field, view name strings.
-// We already have the non _ versions as methods on FragProxy.
-// Thus the _shard, _index, _field, _view as names.
-type fragProxy struct {
-	_shard    uint64
-	_index    *Index
-	_field    *Field
-	_fieldstr string
-	_view     *view
+type fragSpec struct {
+	index    *Index
+	field    *Field
+	fieldstr string
+	view     *view
 }
 
-func (fb *fragProxy) index() string {
-	return fb._index.Name()
-
+func (f *fragment) index() string {
+	return f.idx.name
 }
-func (fb *fragProxy) field() string {
+
+func (f *fragment) field() string {
 	// initialization of _field *Field can
-	// be late, so we might not have it
-	// during the startup dance. Hence
-	// we keep _fieldstr as a backup
-	// until we've safely go _field filled.
-	if fb._field == nil {
-		return fb._fieldstr
+	// go missing during tests that do incomplete setup.
+	// Hence we must keep fieldstr as a backup.
+	if f.fld == nil {
+		return f.fieldstr
 	} else {
-		fb._fieldstr = ""
+		f.fieldstr = ""
 	}
-	return fb._field.Name()
+	return f.fld.name
 }
 
-func (fb *fragProxy) view() string {
-	return fb._view.name
+func (f *fragment) view() string {
+	return f._view.name
 }
-func (fb *fragProxy) path() string {
-	return filepath.Join(fb._view.path, "fragments", strconv.FormatUint(fb._shard, 10))
+func (f *fragment) path() string {
+	return filepath.Join(f._view.path, "fragments", strconv.FormatUint(f.shard, 10))
 }
 
 // fragment represents the intersection of a field and shard in an index.
 type fragment struct {
 	mu sync.RWMutex
 
-	// Composite identifiers
-
 	// We save 20GB worth strings on some data sets by not duplicating
 	// the path, index, field, view strings on every fragment.
-	// FragProxy assembles these on demand.
-	FragProxy
+	// Instead assemble strings on demand in field(), view(), path(), index().
+	fld      *Field
+	fieldstr string
+	_view    *view
+
 	shard uint64
 
 	// idx cached to avoid repeatedly looking it up everywhere.
@@ -217,19 +205,21 @@ type fragment struct {
 	bitmapInfo *roaring.BitmapInfo
 }
 
-// newFragment returns a new instance of Fragment.
-func newFragment(holder *Holder, fp FragProxy, shard uint64, flags byte) *fragment {
-	idx := holder.Index(fp.index())
+// newFragment returns a new instance of fragment.
+func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragment {
+	idx := holder.Index(spec.index.name)
 
 	if idx == nil {
-		panic(fmt.Sprintf("got nil idx back for '%v' from holder!", fp.index()))
+		panic(fmt.Sprintf("got nil idx back for '%v' from holder!", spec.index))
 	}
 
 	f := &fragment{
-		FragProxy: fp,
-		shard:     shard,
-		flags:     flags,
-		idx:       idx,
+		_view:    spec.view,
+		fieldstr: spec.fieldstr,
+		fld:      spec.field,
+		shard:    shard,
+		flags:    flags,
+		idx:      idx,
 
 		CacheType: DefaultCacheType,
 		CacheSize: DefaultCacheSize,

fragment_internal_test.go

@@ -1800,7 +1800,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
 
 	// Open the fragment specified by the path. Note that newFragment
 	// is overriding the usual holder-to-fragment path logic...
-	f := newFragment(th, makeTestFragProxy(*FragmentPath, "i", "f", viewStandard), 0, 0)
+	f := newFragment(th, makeTestFragSpec(*FragmentPath, "i", "f", viewStandard), 0, 0)
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
@@ -2741,35 +2741,14 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
 	}
 }
 
-// testFragProxy implements FragProxy because fragProxy
-// requires *view and *Field but not all tests have those.
-type testFragProxy struct {
-	_path  string
-	_index string
-	_field string
-	_view  string
-}
+func makeTestFragSpec(path, index, field, view0 string) fragSpec {
 
-func (fb *testFragProxy) index() string {
-	return fb._index
-
-}
-func (fb *testFragProxy) field() string {
-	return fb._field
-}
-func (fb *testFragProxy) view() string {
-	return fb._view
-}
-func (fb *testFragProxy) path() string {
-	return fb._path
-}
-
-func makeTestFragProxy(path, index, field, view string) *testFragProxy {
-	return &testFragProxy{
-		_path:  path,
-		_index: index,
-		_field: field,
-		_view:  view,
+	// /tmp/holder-dir199550572/i/f/views/standard/fragments/0 -> /tmp/holder-dir199550572/i/f/views/standard
+	splt := strings.Split(path, sep+"fragments"+sep)
+	return fragSpec{
+		index: &Index{path: splt[0], name: index},
+		field: &Field{name: field},
+		view:  &view{path: splt[0], name: view0},
 	}
 }
 
@@ -2780,7 +2759,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) {
 
 	b.ReportAllocs()
 	// Open the fragment specified by the path.
-	f := newFragment(newTestHolder(b), makeTestFragProxy(*FragmentPath, "i", "f", viewStandard), 0, 0)
+	f := newFragment(newTestHolder(b), makeTestFragSpec(*FragmentPath, "i", "f", viewStandard), 0, 0)
 	if err := f.Open(); err != nil {
 		b.Fatal(err)
 	}
@@ -3200,7 +3179,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		idx, err := h.CreateIndex("i", IndexOptions{})
 		panicOn(err)
 
-		f := newFragment(h, makeTestFragProxy(fi.Name(), "i", "f", viewStandard), 0, 0)
+		f := newFragment(h, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0)
 		err = f.Open()
 		if err != nil {
 			b.Fatalf("opening fragment: %v", err)
@@ -3253,7 +3232,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
 			th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
 		}
 		// XXX TODO: newFragment is using the wrong path here, we should fix that someday.
-		f := newFragment(th, makeTestFragProxy(fi.Name(), "i", "f", viewStandard), 0, 0)
+		f := newFragment(th, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0)
 		defer f.Clean(b)
 
 		tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
@@ -3532,7 +3511,7 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
 	fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view)
 	panicOn(os.MkdirAll(fragDir, 0777))
 	fragPath := fragDir + fmt.Sprintf("%v", shard)
-	f := newFragment(th, makeTestFragProxy(fragPath, index, field, view), shard, flags)
+	f := newFragment(th, makeTestFragSpec(fragPath, index, field, view), shard, flags)
 
 	tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
 	testhook.Cleanup(tb, func() {
@@ -5022,7 +5001,7 @@ func TestImportClearRestart(t *testing.T) {
 				panicOn(err)
 
 				// OVERWRITING the f.path with a new fragment
-				f2 := newFragment(h, makeTestFragProxy(f.path(), "i", "f", viewStandard), 0, 0)
+				f2 := newFragment(h, makeTestFragSpec(f.path(), "i", "f", viewStandard), 0, 0)
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
@@ -5074,7 +5053,7 @@ func TestImportClearRestart(t *testing.T) {
 				_ = idx3
 				panicOn(err)
 
-				f3 := newFragment(h3, makeTestFragProxy(f2.path(), "i", "f", viewStandard), 0, 0)
+				f3 := newFragment(h3, makeTestFragSpec(f2.path(), "i", "f", viewStandard), 0, 0)
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 

holder.go

@@ -22,6 +22,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -53,6 +54,9 @@ const (
 )
 
 func init() {
+	// needed to get the most I/O throughtpu.
+	runtime.GOMAXPROCS(128)
+
 	// For performance tuning, leave these readily available:
 	// CPUProfileForDur(time.Minute, "server.cpu.pprof")
 	// MemProfileForDur(2*time.Minute, "server.mem.pprof")

view.go

@@ -372,14 +372,18 @@ func (v *view) notifyIfNewShard(shard uint64) {
 
 func (v *view) newFragment(shard uint64) *fragment {
 	fld := v.idx.Field(v.field)
-	nb := &fragProxy{
-		_shard:    shard,
-		_index:    v.idx,
-		_field:    fld,
-		_fieldstr: v.field,
-		_view:     v,
+	spec := fragSpec{
+		index: v.idx,
+		field: fld,
+		view:  v,
 	}
-	frag := newFragment(v.holder, nb, shard, v.flags())
+	if fld == nil {
+		// The backup plan.
+		// For tests that do incomplete setup, like making
+		// a view without a field.
+		spec.fieldstr = v.field
+	}
+	frag := newFragment(v.holder, spec, shard, v.flags())
 	frag.CacheType = v.cacheType
 	frag.CacheSize = v.cacheSize
 	frag.stats = v.stats