diff --git a/bsi_test.go b/bsi_test.go index 57803a7f9..091daf409 100644 --- a/bsi_test.go +++ b/bsi_test.go @@ -64,7 +64,6 @@ func TestBSIAdd(t *testing.T) { if max < i { max = i } - t.Log("num: ", i) break } idToIndex[id] = int(i) @@ -99,8 +98,6 @@ func TestBSIAdd(t *testing.T) { } }) } - t.Log("min", min) - t.Log("max", max) } type bsiAddCase struct { diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index a5ae25348..a35e3cce1 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -25,7 +25,6 @@ import ( "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/shardwidth" txkey "github.com/pilosa/pilosa/v2/short_txkey" - //txkey "github.com/pilosa/pilosa/v2/txkey" ) // Shard per db evaluation @@ -95,8 +94,8 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index) panicOn(err) } - estd := "rick/_exists/views/standard" - std := "rick/f/views/standard" + estd := "rick/fields/_exists/views/standard" + std := "rick/fields/f/views/standard" shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false) panicOn(err) @@ -128,7 +127,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { expect0 := txkey.FieldView{Field: "_exists", View: "standard"} expect1 := txkey.FieldView{Field: "f", View: "standard"} if len(fvs) != 2 { - panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs)) + panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src)) } if fvs[0] != expect0 { panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])) @@ -155,7 +154,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { expect0 := txkey.FieldView{Field: "_exists", View: "standard"} expect1 := txkey.FieldView{Field: "f", View: "standard"} if len(fvs) != 2 { - panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs)) + panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src)) } if fvs[0] != expect0 { panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0])) @@ -173,24 +172,24 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { // data for Test_DBPerShard_GetShardsForIndex // var sampleRoaringDirList = map[string]string{"roaring": ` -rick/f/views/standard/fragments/215.cache -rick/f/views/standard/fragments/221.cache -rick/f/views/standard/fragments/223.cache -rick/f/views/standard/fragments/93.cache -rick/f/views/standard/fragments/217.cache -rick/f/views/standard/fragments/219.cache -rick/f/views/standard/fragments/217 -rick/f/views/standard/fragments/219 -rick/f/views/standard/fragments/215 -rick/f/views/standard/fragments/221 -rick/f/views/standard/fragments/223 -rick/f/views/standard/fragments/93 -rick/_exists/views/standard/fragments/221 -rick/_exists/views/standard/fragments/215 -rick/_exists/views/standard/fragments/217 -rick/_exists/views/standard/fragments/93 -rick/_exists/views/standard/fragments/219 -rick/_exists/views/standard/fragments/223 +rick/fields/f/views/standard/fragments/215.cache +rick/fields/f/views/standard/fragments/221.cache +rick/fields/f/views/standard/fragments/223.cache +rick/fields/f/views/standard/fragments/93.cache +rick/fields/f/views/standard/fragments/217.cache +rick/fields/f/views/standard/fragments/219.cache +rick/fields/f/views/standard/fragments/217 +rick/fields/f/views/standard/fragments/219 +rick/fields/f/views/standard/fragments/215 +rick/fields/f/views/standard/fragments/221 +rick/fields/f/views/standard/fragments/223 +rick/fields/f/views/standard/fragments/93 +rick/fields/_exists/views/standard/fragments/221 +rick/fields/_exists/views/standard/fragments/215 +rick/fields/_exists/views/standard/fragments/217 +rick/fields/_exists/views/standard/fragments/93 +rick/fields/_exists/views/standard/fragments/219 +rick/fields/_exists/views/standard/fragments/223 `, "bolt": ` rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db diff --git a/holder.go b/holder.go index 91c778361..f385f0b89 100644 --- a/holder.go +++ b/holder.go @@ -56,6 +56,9 @@ const ( // DefaultIndexesDir is the default indexes directory used by the holder. DefaultIndexesDir = "indexes" + + // DefaultFieldsDir is the default fields directory used by each index. + DefaultFieldsDir = "fields" ) func init() { @@ -1762,7 +1765,7 @@ func (s *holderSyncer) resetTranslationSync() error { //////////////////////////////////////////////////////////// -// translationSyncer provides an interface allowing a function +// TranslationSyncer provides an interface allowing a function // to notify the server that an action has occurred which requires // the translation sync process to be reset. In general, this // includes anything which modifies schema (add/remove index, etc), diff --git a/index.go b/index.go index 080fa6ba1..0f0725dc9 100644 --- a/index.go +++ b/index.go @@ -140,6 +140,11 @@ func (i *Index) Path() string { return i.path } +// FieldsPath returns the path of the fields directory. +func (i *Index) FieldsPath() string { + return filepath.Join(i.path, DefaultFieldsDir) +} + // TranslateStorePath returns the translation database path for a partition. func (i *Index) TranslateStorePath(partitionID int) string { return filepath.Join(i.path, translateStoreDir, strconv.Itoa(partitionID)) @@ -204,8 +209,8 @@ func (i *Index) OpenWithSchema(idx *disco.Index) error { // not validated against the schema as they are opened. func (i *Index) open(idx *disco.Index) (err error) { // Ensure the path exists. - i.holder.Logger.Debugf("ensure index path exists: %s", i.path) - if err := os.MkdirAll(i.path, 0777); err != nil { + i.holder.Logger.Debugf("ensure index path exists: %s", i.FieldsPath()) + if err := os.MkdirAll(i.FieldsPath(), 0777); err != nil { return errors.Wrap(err, "creating directory") } @@ -286,9 +291,9 @@ var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. func (i *Index) openFields(idx *disco.Index) error { - f, err := os.Open(i.path) + f, err := os.Open(i.FieldsPath()) if err != nil { - return errors.Wrap(err, "opening directory") + return errors.Wrap(err, "opening fields directory") } defer f.Close() @@ -508,7 +513,7 @@ func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) { } // fieldPath returns the path to a field in the index. -func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) } +func (i *Index) fieldPath(name string) string { return filepath.Join(i.FieldsPath(), name) } // Field returns a field in the index by name. func (i *Index) Field(name string) *Field { diff --git a/rrtx.go b/rrtx.go index 2a16807ae..64715fb50 100644 --- a/rrtx.go +++ b/rrtx.go @@ -430,7 +430,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { vs = NewFieldView2Shards() // A) open the index directory - f, err := os.Open(idx.path) + f, err := os.Open(idx.FieldsPath()) if err != nil { return nil, errors.Wrap(err, "opening directory") } @@ -453,7 +453,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { //vv("roaringGetFieldView2Shards B) on field '%v'", field) - fieldPath := filepath.Join(idx.path, field) + fieldPath := filepath.Join(idx.FieldsPath(), field) // Skip embedded db files too. if idx.holder.txf.IsTxDatabasePath(field) { @@ -506,7 +506,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { // A) open the index directory - f, err := os.Open(idx.path) + f, err := os.Open(idx.FieldsPath()) if err != nil { return nil, errors.Wrap(err, "opening directory") } @@ -529,7 +529,7 @@ func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txk //vv("B) on field '%v'", field) - fieldPath := filepath.Join(idx.path, field) + fieldPath := filepath.Join(idx.FieldsPath(), field) // Skip embedded db files too. if idx.holder.txf.IsTxDatabasePath(field) { diff --git a/server/server_test.go b/server/server_test.go index f14aa75e2..43e67f5ab 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -839,7 +839,7 @@ func TestMain_ImportTimestamp(t *testing.T) { t.Fatal(err) } // Ensure the correct views were created. - dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName) + dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, pilosa.DefaultFieldsDir, fieldName) files, err := ioutil.ReadDir(dir) if err != nil { t.Fatal(err) @@ -895,7 +895,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } // Ensure the correct views were created. - dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName) + dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, pilosa.DefaultFieldsDir, fieldName) files, err := ioutil.ReadDir(dir) if err != nil { t.Fatal(err) diff --git a/txfactory.go b/txfactory.go index 1cee1d286..b86566d71 100644 --- a/txfactory.go +++ b/txfactory.go @@ -708,7 +708,7 @@ func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error) } // field metadata, e.g. rowAttrs - fieldPath := path.Join(indexPath, field) + fieldPath := path.Join(indexPath, DefaultFieldsDir, field) metaBytes, err := directoryUsage(fieldPath, false) // this includes keys if err != nil { return fieldUsage, errors.Wrapf(err, "getting disk usage for field meta (%s)", field) @@ -1005,19 +1005,19 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, } // sample path: - // field view shard - // myfield/views/standard/fragments/0 + // field view shard + // fields/myfield/views/standard/fragments/0 s := strings.Split(path, "/") n := len(s) - if n != 5 { + if n != 6 { err = fmt.Errorf("len(s)=%v, but expected 5. path='%v'", n, path) return } - field = s[0] - view = s[2] - shard, err = strconv.ParseUint(s[4], 10, 64) + field = s[1] + view = s[3] + shard, err = strconv.ParseUint(s[5], 10, 64) if err != nil { - err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[4], err) + err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[5], err) } return }