scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

introduce "fields" directory between index and field

Browse source
This commit is contained in:
Travis 2021-03-04 22:42:52 -06:00
parent 4e7ec34c6d
commit 345d076fdf
No known key found for this signature in database
GPG key ID: 37080CC2042BA34E
7 changed files with 50 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bsi_test.go
View file

@ -64,7 +64,6 @@ func TestBSIAdd(t *testing.T) {
if max < i {
max = i
}
t.Log("num: ", i)
break
}
idToIndex[id] = int(i)
@ -99,8 +98,6 @@ func TestBSIAdd(t *testing.T) {
}
})
}
t.Log("min", min)
t.Log("max", max)
}
type bsiAddCase struct {

45
dbshard_internal_test.go
View file

@ -25,7 +25,6 @@ import (
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/shardwidth"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
//txkey "github.com/pilosa/pilosa/v2/txkey"
)
// Shard per db evaluation
@ -95,8 +94,8 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
panicOn(err)
}
estd := "rick/_exists/views/standard"
std := "rick/f/views/standard"
estd := "rick/fields/_exists/views/standard"
std := "rick/fields/f/views/standard"
shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
panicOn(err)
@ -128,7 +127,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs))
panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
}
if fvs[0] != expect0 {
panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
@ -155,7 +154,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
if len(fvs) != 2 {
panic(fmt.Sprintf("fvs should be len 2, got '%#v'", fvs))
panic(fmt.Sprintf("fvs should be len 2, got '%#v' (%s)", fvs, src))
}
if fvs[0] != expect0 {
panic(fmt.Sprintf("expected fvs[0]='%#v', but got '%#v'", expect0, fvs[0]))
@ -173,24 +172,24 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
// data for Test_DBPerShard_GetShardsForIndex
//
var sampleRoaringDirList = map[string]string{"roaring": `
rick/f/views/standard/fragments/215.cache
rick/f/views/standard/fragments/221.cache
rick/f/views/standard/fragments/223.cache
rick/f/views/standard/fragments/93.cache
rick/f/views/standard/fragments/217.cache
rick/f/views/standard/fragments/219.cache
rick/f/views/standard/fragments/217
rick/f/views/standard/fragments/219
rick/f/views/standard/fragments/215
rick/f/views/standard/fragments/221
rick/f/views/standard/fragments/223
rick/f/views/standard/fragments/93
rick/_exists/views/standard/fragments/221
rick/_exists/views/standard/fragments/215
rick/_exists/views/standard/fragments/217
rick/_exists/views/standard/fragments/93
rick/_exists/views/standard/fragments/219
rick/_exists/views/standard/fragments/223
rick/fields/f/views/standard/fragments/215.cache
rick/fields/f/views/standard/fragments/221.cache
rick/fields/f/views/standard/fragments/223.cache
rick/fields/f/views/standard/fragments/93.cache
rick/fields/f/views/standard/fragments/217.cache
rick/fields/f/views/standard/fragments/219.cache
rick/fields/f/views/standard/fragments/217
rick/fields/f/views/standard/fragments/219
rick/fields/f/views/standard/fragments/215
rick/fields/f/views/standard/fragments/221
rick/fields/f/views/standard/fragments/223
rick/fields/f/views/standard/fragments/93
rick/fields/_exists/views/standard/fragments/221
rick/fields/_exists/views/standard/fragments/215
rick/fields/_exists/views/standard/fragments/217
rick/fields/_exists/views/standard/fragments/93
rick/fields/_exists/views/standard/fragments/219
rick/fields/_exists/views/standard/fragments/223
`,
"bolt": `
rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db

5
holder.go
View file

@ -56,6 +56,9 @@ const (
// DefaultIndexesDir is the default indexes directory used by the holder.
DefaultIndexesDir = "indexes"
// DefaultFieldsDir is the default fields directory used by each index.
DefaultFieldsDir = "fields"
)
func init() {
@ -1762,7 +1765,7 @@ func (s *holderSyncer) resetTranslationSync() error {
////////////////////////////////////////////////////////////
// translationSyncer provides an interface allowing a function
// TranslationSyncer provides an interface allowing a function
// to notify the server that an action has occurred which requires
// the translation sync process to be reset. In general, this
// includes anything which modifies schema (add/remove index, etc),

15
index.go
View file

@ -140,6 +140,11 @@ func (i *Index) Path() string {
return i.path
}
// FieldsPath returns the path of the fields directory.
func (i *Index) FieldsPath() string {
return filepath.Join(i.path, DefaultFieldsDir)
}
// TranslateStorePath returns the translation database path for a partition.
func (i *Index) TranslateStorePath(partitionID int) string {
return filepath.Join(i.path, translateStoreDir, strconv.Itoa(partitionID))
@ -204,8 +209,8 @@ func (i *Index) OpenWithSchema(idx *disco.Index) error {
// not validated against the schema as they are opened.
func (i *Index) open(idx *disco.Index) (err error) {
// Ensure the path exists.
i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
i.holder.Logger.Debugf("ensure index path exists: %s", i.FieldsPath())
if err := os.MkdirAll(i.FieldsPath(), 0777); err != nil {
return errors.Wrap(err, "creating directory")
}
@ -286,9 +291,9 @@ var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields(idx *disco.Index) error {
f, err := os.Open(i.path)
f, err := os.Open(i.FieldsPath())
if err != nil {
return errors.Wrap(err, "opening directory")
return errors.Wrap(err, "opening fields directory")
}
defer f.Close()
@ -508,7 +513,7 @@ func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) {
}
// fieldPath returns the path to a field in the index.
func (i *Index) fieldPath(name string) string { return filepath.Join(i.path, name) }
func (i *Index) fieldPath(name string) string { return filepath.Join(i.FieldsPath(), name) }
// Field returns a field in the index by name.
func (i *Index) Field(name string) *Field {

8
rrtx.go
View file

@ -430,7 +430,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) {
vs = NewFieldView2Shards()
// A) open the index directory
f, err := os.Open(idx.path)
f, err := os.Open(idx.FieldsPath())
if err != nil {
return nil, errors.Wrap(err, "opening directory")
}
@ -453,7 +453,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) {
//vv("roaringGetFieldView2Shards B) on field '%v'", field)
fieldPath := filepath.Join(idx.path, field)
fieldPath := filepath.Join(idx.FieldsPath(), field)
// Skip embedded db files too.
if idx.holder.txf.IsTxDatabasePath(field) {
@ -506,7 +506,7 @@ func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) {
func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
// A) open the index directory
f, err := os.Open(idx.path)
f, err := os.Open(idx.FieldsPath())
if err != nil {
return nil, errors.Wrap(err, "opening directory")
}
@ -529,7 +529,7 @@ func (tx *RoaringTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txk
//vv("B) on field '%v'", field)
fieldPath := filepath.Join(idx.path, field)
fieldPath := filepath.Join(idx.FieldsPath(), field)
// Skip embedded db files too.
if idx.holder.txf.IsTxDatabasePath(field) {

4
server/server_test.go
View file

@ -839,7 +839,7 @@ func TestMain_ImportTimestamp(t *testing.T) {
t.Fatal(err)
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName)
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, pilosa.DefaultFieldsDir, fieldName)
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)
@ -895,7 +895,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, fieldName)
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.DefaultIndexesDir, indexName, pilosa.DefaultFieldsDir, fieldName)
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatal(err)

16
txfactory.go
View file

@ -708,7 +708,7 @@ func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error)
}
// field metadata, e.g. rowAttrs
fieldPath := path.Join(indexPath, field)
fieldPath := path.Join(indexPath, DefaultFieldsDir, field)
metaBytes, err := directoryUsage(fieldPath, false) // this includes keys
if err != nil {
return fieldUsage, errors.Wrapf(err, "getting disk usage for field meta (%s)", field)
@ -1005,19 +1005,19 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
}
// sample path:
// field view shard
// myfield/views/standard/fragments/0
// field view shard
// fields/myfield/views/standard/fragments/0
s := strings.Split(path, "/")
n := len(s)
if n != 5 {
if n != 6 {
err = fmt.Errorf("len(s)=%v, but expected 5. path='%v'", n, path)
return
}
field = s[0]
view = s[2]
shard, err = strconv.ParseUint(s[4], 10, 64)
field = s[1]
view = s[3]
shard, err = strconv.ParseUint(s[5], 10, 64)
if err != nil {
err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[4], err)
err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[5], err)
}
return
}