introduce "indexes" directory between datadir and index

holder.go

@@ -17,9 +17,7 @@ package pilosa
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -40,7 +38,6 @@ import (
 	"github.com/pilosa/pilosa/v2/topology"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
-	uuid "github.com/satori/go.uuid"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -56,6 +53,9 @@ const (
 
 	// DefaultDiscoDir is the default data directory used by the disco implementation.
 	DefaultDiscoDir = ".disco"
+
+	// DefaultIndexesDir is the default indexes directory used by the holder.
+	DefaultIndexesDir = "indexes"
 )
 
 func init() {
@@ -296,7 +296,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
 
 	storage.SetRowCacheOn(cfg.RowcacheOn)
 
-	txf, err := NewTxFactory(cfg.StorageConfig.Backend, path, h)
+	txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h)
 	panicOn(err)
 	h.txf = txf
 	h.txf.blueGreenOffIfRunningBlueGreen()
@@ -305,11 +305,16 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
 	return h
 }
 
-// Path() returns the path directory the holder was created with.
+// Path returns the path directory the holder was created with.
 func (h *Holder) Path() string {
 	return h.path
 }
 
+// IndexesPath returns the path of the indexes directory.
+func (h *Holder) IndexesPath() string {
+	return filepath.Join(h.path, DefaultIndexesDir)
+}
+
 type HolderInfo struct {
 	FragmentInfo  map[string]FragmentInfo
 	FragmentNames []string
@@ -590,7 +595,7 @@ func (h *Holder) Open() error {
 	defer func() { h.opening = false }()
 
 	if h.txf == nil {
-		txf, err := NewTxFactory(h.cfg.StorageConfig.Backend, h.path, h)
+		txf, err := NewTxFactory(h.cfg.StorageConfig.Backend, h.IndexesPath(), h)
 		if err != nil {
 			return errors.Wrap(err, "Holder.Open NewTxFactory()")
 		}
@@ -605,7 +610,7 @@ func (h *Holder) Open() error {
 	h.setFileLimit()
 
 	h.Logger.Printf("open holder path: %s", h.path)
-	if err := os.MkdirAll(h.path, 0777); err != nil {
+	if err := os.MkdirAll(h.IndexesPath(), 0777); err != nil {
 		return errors.Wrap(err, "creating directory")
 	}
 
@@ -629,7 +634,7 @@ func (h *Holder) Open() error {
 	}
 
 	// Open path to read all index directories.
-	f, err := os.Open(h.path)
+	f, err := os.Open(h.IndexesPath())
 	if err != nil {
 		return errors.Wrap(err, "opening directory")
 	}
@@ -850,13 +855,13 @@ func (h *Holder) HasData() (bool, error) {
 		return true, nil
 	}
 	// Open path to read all index directories.
-	if _, err := os.Stat(h.path); os.IsNotExist(err) {
+	if _, err := os.Stat(h.IndexesPath()); os.IsNotExist(err) {
 		return false, nil
 	} else if err != nil {
 		return false, errors.Wrap(err, "statting data dir")
 	}
 
-	f, err := os.Open(h.path)
+	f, err := os.Open(h.IndexesPath())
 	if err != nil {
 		return false, errors.Wrap(err, "opening data dir")
 	}
@@ -992,18 +997,7 @@ func (h *Holder) applySchema(schema *Schema) error {
 
 // IndexPath returns the path where a given index is stored.
 func (h *Holder) IndexPath(name string) string {
-	return filepath.Join(h.path, name)
-}
-
-// HolderPathFromIndexPath is
-// used by test/index.go:71 in test.Index.Reopen() to get the right
-// path into a test Holder that doesn't know its own proper path.
-// If the Holder changes index paths to being something other than
-// holderPath + "/" + indexName, this will need adjusting too.
-func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string {
-	n := len(indexPath)
-	hpath2 := indexPath[:n-(len(indexName)+1)]
-	return hpath2
+	return filepath.Join(h.IndexesPath(), name)
 }
 
 // Index returns the index by name.
@@ -1483,31 +1477,6 @@ func (h *Holder) setFileLimit() {
 	}
 }
 
-func (h *Holder) LoadNodeID() (string, error) {
-	idPath := path.Join(h.path, ".id")
-	h.Logger.Printf("load NodeID: %s", idPath)
-	if err := os.MkdirAll(h.path, 0777); err != nil {
-		return "", errors.Wrap(err, "creating directory")
-	}
-
-	nodeIDBytes, err := ioutil.ReadFile(idPath)
-	if err == nil {
-		nodeid := strings.TrimSpace(string(nodeIDBytes))
-		h.Logger.Printf("I am NodeID: %s", nodeid)
-		return nodeid, nil
-	}
-	if !os.IsNotExist(err) {
-		return "", errors.Wrap(err, "reading file")
-	}
-	nodeID := uuid.NewV4().String()
-	err = ioutil.WriteFile(idPath, []byte(nodeID), 0600)
-	if err != nil {
-		return "", errors.Wrap(err, "writing file")
-	}
-	h.Logger.Printf("I am NodeID: %s", nodeID)
-	return nodeID, nil
-}
-
 // Log startup time and version to $DATA_DIR/.startup.log
 func (h *Holder) logStartup() error {
 	RFC3339NanoFixedWidth := "2006-01-02T15:04:05.000000 07:00"
@@ -2270,14 +2239,13 @@ func (h *Holder) Txf() *TxFactory {
 	return h.txf
 }
 
-// Begin starts a transaction on the holder. The index and shard
+// BeginTx starts a transaction on the holder. The index and shard
 // must be specified.
 func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) {
 	return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil
 }
 
 func (h *Holder) HasRoaringData() (has bool, err error) {
-
 	idxs := h.Indexes()
 	for _, idx := range idxs {
 		paths, err := listFilesUnderDir(idx.path, false, "", true)

holder_test.go

@@ -240,7 +240,7 @@ func TestHolder_Open(t *testing.T) {
 			t.Fatal(err)
 		} else if err := h.Holder.Close(); err != nil {
 			t.Fatal(err)
-		} else if err := os.Truncate(filepath.Join(h.Path(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
+		} else if err := os.Truncate(filepath.Join(h.IndexesPath(), "foo", "bar", "views", "standard", "fragments", "0"), 20); err != nil {
 			t.Fatal(err)
 		}
 
@@ -353,7 +353,8 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek", func(t *testing.T) {
-		h := test.NewHolder(t)
+		h := test.MustOpenHolder(t)
+		defer h.Close()
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)

server/server_test.go

@@ -839,7 +839,7 @@ func TestMain_ImportTimestamp(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Ensure the correct views were created.
-	dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
+	dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName)
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		t.Fatal(err)
@@ -895,7 +895,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
 	}
 
 	// Ensure the correct views were created.
-	dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName)
+	dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName)
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		t.Fatal(err)

txfactory.go

@@ -594,6 +594,10 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) {
 	if err != nil {
 		return indexUsage, 0, errors.Wrap(err, "expanding data directory")
 	}
+	indexesPath, err := expandDirName(f.holder.IndexesPath())
+	if err != nil {
+		return indexUsage, 0, errors.Wrap(err, "expanding indexes directory")
+	}
 
 	idxs := f.holder.Indexes()
 
@@ -601,7 +605,7 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) {
 	defer qcx.Abort()
 	for _, idx := range idxs {
 		index := idx.name
-		indexPath := path.Join(holderPath, index)
+		indexPath := path.Join(indexesPath, index)
 
 		// field usage
 		fieldUsages := make(map[string]FieldUsage)