diff --git a/fragment.go b/fragment.go index e0985999e..7b91d79c5 100644 --- a/fragment.go +++ b/fragment.go @@ -100,61 +100,49 @@ const ( roaringFlagBSIv2 = 0x01 // indicates version using low bit for existence ) -// FragProxy lets us use either fragProxy or testFragProxy. -type FragProxy interface { - index() string - field() string - view() string - path() string -} - -// fragProxy saves a ton of duplicated strings for +// fragSpec saves a ton of duplicated strings for // path, and index, field, view name strings. -// We already have the non _ versions as methods on FragProxy. -// Thus the _shard, _index, _field, _view as names. -type fragProxy struct { - _shard uint64 - _index *Index - _field *Field - _fieldstr string - _view *view +type fragSpec struct { + index *Index + field *Field + fieldstr string + view *view } -func (fb *fragProxy) index() string { - return fb._index.Name() - +func (f *fragment) index() string { + return f.idx.name } -func (fb *fragProxy) field() string { + +func (f *fragment) field() string { // initialization of _field *Field can - // be late, so we might not have it - // during the startup dance. Hence - // we keep _fieldstr as a backup - // until we've safely go _field filled. - if fb._field == nil { - return fb._fieldstr + // go missing during tests that do incomplete setup. + // Hence we must keep fieldstr as a backup. + if f.fld == nil { + return f.fieldstr } else { - fb._fieldstr = "" + f.fieldstr = "" } - return fb._field.Name() + return f.fld.name } -func (fb *fragProxy) view() string { - return fb._view.name +func (f *fragment) view() string { + return f._view.name } -func (fb *fragProxy) path() string { - return filepath.Join(fb._view.path, "fragments", strconv.FormatUint(fb._shard, 10)) +func (f *fragment) path() string { + return filepath.Join(f._view.path, "fragments", strconv.FormatUint(f.shard, 10)) } // fragment represents the intersection of a field and shard in an index. type fragment struct { mu sync.RWMutex - // Composite identifiers - // We save 20GB worth strings on some data sets by not duplicating // the path, index, field, view strings on every fragment. - // FragProxy assembles these on demand. - FragProxy + // Instead assemble strings on demand in field(), view(), path(), index(). + fld *Field + fieldstr string + _view *view + shard uint64 // idx cached to avoid repeatedly looking it up everywhere. @@ -217,19 +205,21 @@ type fragment struct { bitmapInfo *roaring.BitmapInfo } -// newFragment returns a new instance of Fragment. -func newFragment(holder *Holder, fp FragProxy, shard uint64, flags byte) *fragment { - idx := holder.Index(fp.index()) +// newFragment returns a new instance of fragment. +func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragment { + idx := holder.Index(spec.index.name) if idx == nil { - panic(fmt.Sprintf("got nil idx back for '%v' from holder!", fp.index())) + panic(fmt.Sprintf("got nil idx back for '%v' from holder!", spec.index)) } f := &fragment{ - FragProxy: fp, - shard: shard, - flags: flags, - idx: idx, + _view: spec.view, + fieldstr: spec.fieldstr, + fld: spec.field, + shard: shard, + flags: flags, + idx: idx, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 6fef34dac..970596705 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1800,7 +1800,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { // Open the fragment specified by the path. Note that newFragment // is overriding the usual holder-to-fragment path logic... - f := newFragment(th, makeTestFragProxy(*FragmentPath, "i", "f", viewStandard), 0, 0) + f := newFragment(th, makeTestFragSpec(*FragmentPath, "i", "f", viewStandard), 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -2741,35 +2741,14 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { } } -// testFragProxy implements FragProxy because fragProxy -// requires *view and *Field but not all tests have those. -type testFragProxy struct { - _path string - _index string - _field string - _view string -} +func makeTestFragSpec(path, index, field, view0 string) fragSpec { -func (fb *testFragProxy) index() string { - return fb._index - -} -func (fb *testFragProxy) field() string { - return fb._field -} -func (fb *testFragProxy) view() string { - return fb._view -} -func (fb *testFragProxy) path() string { - return fb._path -} - -func makeTestFragProxy(path, index, field, view string) *testFragProxy { - return &testFragProxy{ - _path: path, - _index: index, - _field: field, - _view: view, + // /tmp/holder-dir199550572/i/f/views/standard/fragments/0 -> /tmp/holder-dir199550572/i/f/views/standard + splt := strings.Split(path, sep+"fragments"+sep) + return fragSpec{ + index: &Index{path: splt[0], name: index}, + field: &Field{name: field}, + view: &view{path: splt[0], name: view0}, } } @@ -2780,7 +2759,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := newFragment(newTestHolder(b), makeTestFragProxy(*FragmentPath, "i", "f", viewStandard), 0, 0) + f := newFragment(newTestHolder(b), makeTestFragSpec(*FragmentPath, "i", "f", viewStandard), 0, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -3200,7 +3179,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { idx, err := h.CreateIndex("i", IndexOptions{}) panicOn(err) - f := newFragment(h, makeTestFragProxy(fi.Name(), "i", "f", viewStandard), 0, 0) + f := newFragment(h, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0) err = f.Open() if err != nil { b.Fatalf("opening fragment: %v", err) @@ -3253,7 +3232,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { th.SnapshotQueue = newSnapshotQueue(1, 1, nil) } // XXX TODO: newFragment is using the wrong path here, we should fix that someday. - f := newFragment(th, makeTestFragProxy(fi.Name(), "i", "f", viewStandard), 0, 0) + f := newFragment(th, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0) defer f.Clean(b) tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) @@ -3532,7 +3511,7 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6 fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view) panicOn(os.MkdirAll(fragDir, 0777)) fragPath := fragDir + fmt.Sprintf("%v", shard) - f := newFragment(th, makeTestFragProxy(fragPath, index, field, view), shard, flags) + f := newFragment(th, makeTestFragSpec(fragPath, index, field, view), shard, flags) tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) testhook.Cleanup(tb, func() { @@ -5022,7 +5001,7 @@ func TestImportClearRestart(t *testing.T) { panicOn(err) // OVERWRITING the f.path with a new fragment - f2 := newFragment(h, makeTestFragProxy(f.path(), "i", "f", viewStandard), 0, 0) + f2 := newFragment(h, makeTestFragSpec(f.path(), "i", "f", viewStandard), 0, 0) f2.MaxOpN = maxOpN f2.CacheType = f.CacheType @@ -5074,7 +5053,7 @@ func TestImportClearRestart(t *testing.T) { _ = idx3 panicOn(err) - f3 := newFragment(h3, makeTestFragProxy(f2.path(), "i", "f", viewStandard), 0, 0) + f3 := newFragment(h3, makeTestFragSpec(f2.path(), "i", "f", viewStandard), 0, 0) f3.MaxOpN = maxOpN f3.CacheType = f.CacheType diff --git a/holder.go b/holder.go index 9ac008b08..a6db1e76a 100644 --- a/holder.go +++ b/holder.go @@ -22,6 +22,7 @@ import ( "path" "path/filepath" "regexp" + "runtime" "sort" "strconv" "strings" @@ -53,6 +54,9 @@ const ( ) func init() { + // needed to get the most I/O throughtpu. + runtime.GOMAXPROCS(128) + // For performance tuning, leave these readily available: // CPUProfileForDur(time.Minute, "server.cpu.pprof") // MemProfileForDur(2*time.Minute, "server.mem.pprof") diff --git a/view.go b/view.go index 49e624685..0929827d6 100644 --- a/view.go +++ b/view.go @@ -372,14 +372,18 @@ func (v *view) notifyIfNewShard(shard uint64) { func (v *view) newFragment(shard uint64) *fragment { fld := v.idx.Field(v.field) - nb := &fragProxy{ - _shard: shard, - _index: v.idx, - _field: fld, - _fieldstr: v.field, - _view: v, + spec := fragSpec{ + index: v.idx, + field: fld, + view: v, } - frag := newFragment(v.holder, nb, shard, v.flags()) + if fld == nil { + // The backup plan. + // For tests that do incomplete setup, like making + // a view without a field. + spec.fieldstr = v.field + } + frag := newFragment(v.holder, spec, shard, v.flags()) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.stats = v.stats