Merge pull request #1517 from travisturner/datadir-take-3

Remove stutter and suffix from backend files

bolt.go

@@ -112,15 +112,6 @@ func DumpAllBolt() {
 	}
 }
 
-// boltPath is a helper for determining the full directory
-// in which the bolt database will be stored.
-func boltPath(path string) string {
-	if !strings.HasSuffix(path, "-boltdb") {
-		return path + "-boltdb"
-	}
-	return path
-}
-
 // openBoltDB opens the database in the bpath directoy
 // without deleting any prior content. Any BoltDB
 // database directory will have the "-bolt" suffix.
@@ -129,9 +120,7 @@ func boltPath(path string) string {
 // if one does not exist for its bpath. Otherwise it returns
 // the existing instance. This insures only one boltDB
 // per bpath in this pilosa node.
-func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
-	path := boltPath(path0)
-
+func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	w, ok := r.path2db[path]

bolt_test.go

@@ -85,7 +85,7 @@ func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shar
 
 func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) {
 	var err error
-	fn := boltPath(path)
+	fn := path
 	panicOn(os.RemoveAll(fn))
 	ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil)
 	panicOn(err)

dbshard.go

@@ -37,9 +37,6 @@ const (
 	// backendsDir is the default backends directory used to store the
 	// data for each backend.
 	backendsDir = "backends"
-
-	// backendDirPrefix is the default prefix of each backend directory.
-	backendDirPrefix = "backend"
 )
 
 // types to support a database file per shard
@@ -566,7 +563,7 @@ func (dbs *DBShard) pathForType(ty txtype) string {
 	// what here for roaring? well, roaringRegistrar.OpenDBWrapper()
 	// is a no-op anyhow. so doesn't need to be correct atm.
 
-	path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + backendDirPrefix + ty.FileSuffix() + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix())
+	path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard)
 	if ty == boltTxn {
 		// special case:
 		// bolt doesn't use a directory like the others, just a direct path.
@@ -579,7 +576,7 @@ func (dbs *DBShard) pathForType(ty txtype) string {
 // prefixForType and pathForType must be kept in sync!
 func (per *DBPerShard) prefixForType(idx *Index, ty txtype) string {
 	// top level paths will end in "@@"
-	return per.HolderDir + sep + idx.name + sep + backendsDir + sep + backendDirPrefix + ty.FileSuffix() + sep
+	return per.HolderDir + sep + idx.name + sep + backendsDir + sep + ty.DirectoryName() + sep
 }
 
 var ErrNoData = fmt.Errorf("no data")
@@ -807,31 +804,25 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
 	}
 	// INVAR: not-roaring.
 
-	requiredSuffix := ty.FileSuffix()
 	path := per.prefixForType(idx, ty)
 
 	ignoreEmpty := false
 	includeRoot := true
-	dbf, err := listDirUnderDir(path, includeRoot, requiredSuffix, ignoreEmpty)
+	dbf, err := listDirUnderDir(path, includeRoot, ignoreEmpty)
 	panicOn(err)
 
 	for _, nm := range dbf {
-
 		base := filepath.Base(nm)
 
-		splt := strings.Split(base, requiredSuffix)
-		if len(splt) != 2 {
-			panic(fmt.Sprintf("should have 2 parts: nm='%v', base(nm)='%v'; requiredSuffix='%v'", nm, base, requiredSuffix))
-		}
-		prefix := splt[0]
+		// We're only interested in "shard.*" files, so skip everything else.
 		const shardPrefix = "shard."
 		const lenOfShardPrefix = len(shardPrefix)
-		if !strings.HasPrefix(prefix, shardPrefix) {
+		if !strings.HasPrefix(base, shardPrefix) {
 			continue
 		}
 
 		// Parse filename into integer.
-		shard, err := strconv.ParseUint(prefix[lenOfShardPrefix:], 10, 64)
+		shard, err := strconv.ParseUint(base[lenOfShardPrefix:], 10, 64)
 		if err != nil {
 			panicOn(err)
 			continue
@@ -877,7 +868,7 @@ func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index,
 	return dbs.W[whichty].HasData()
 }
 
-func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
+func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) {
 	if !dirExists(root) {
 		return
 	}
@@ -901,9 +892,7 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor
 				if ignoreEmpty && info.Size() == 0 {
 					return nil
 				}
-				if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
-					files = append(files, path[n:])
-				}
+				files = append(files, path[n:])
 			}
 		}
 		return nil

rbf.go

@@ -141,15 +141,6 @@ func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) {
 	r.mu.Unlock()
 }
 
-// rbfPath is a helper for determining the full directory
-// in which the RBF database will be stored.
-func rbfPath(path string) string {
-	if !strings.HasSuffix(path, "-rbf") {
-		return path + "-rbf"
-	}
-	return path
-}
-
 // OpenDBWrapper opens the database in the path directory
 // without deleting any prior content. Any
 // database directory will have the "-rbf" suffix.
@@ -158,8 +149,7 @@ func rbfPath(path string) string {
 // if one does not exist for its path. Otherwise it returns
 // the existing instance. This insures only one RbfDBWrapper
 // per bpath in this pilosa node.
-func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
-	path := rbfPath(path0)
+func (r *rbfDBRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	w, ok := r.path2db[path]

txfactory.go

@@ -423,19 +423,18 @@ const (
 	boltTxn    txtype = 4
 )
 
-// FileSuffix is used to determine backend directory names.
-// We append '@' to be sure we never collide with a field name
-// inside the index directory. In the future for different
-// versions of the same backend, there might be version
-// identifier tacked on too.
-func (ty txtype) FileSuffix() string {
+// DirectoryName just returns a string version of the transaction type. We
+// really need to consolidate the storage backend and tx stuff because it's
+// currently rather confusing. This method should be addressed (i.e.
+// replaced/removed) during that refactor.
+func (ty txtype) DirectoryName() string {
 	switch ty {
 	case roaringTxn:
-		return ""
+		return "roaring"
 	case rbfTxn:
-		return "-rbf"
+		return "rbf"
 	case boltTxn:
-		return "-boltdb"
+		return "boltdb"
 	}
 	panic(fmt.Sprintf("unkown txtype %v", int(ty)))
 }