linter fix

cmd/roaring-migrate/main.go

@@ -1,3 +1,16 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 package main
 
 import (
@@ -23,8 +36,8 @@ func main() {
 	var dataDir, backupPath string
 	cmdMigrate := &cobra.Command{
 		Use:   "roaring-migrate",
-		Short: "TBD",
-		Long:  ` TBD`,
+		Short: "convert roaring pilosa backup to rbf",
+		Long:  `roaring-migrate uses the pilosa data-dir for each node, and produces a new backup that is able to be restored from utilizing the new pilosa restore tool.`,
 		//Args:  cobra.MinimumNArgs(2),
 		Run: func(cmd *cobra.Command, args []string) {
 			nodes := strings.Split(dataDir, ",")
@@ -40,10 +53,18 @@ func main() {
 	}
 	cmdMigrate.Flags().StringVarP(&dataDir, "dataDir", "d", "", "source directories for each node seperated by commas")
 	cmdMigrate.Flags().StringVarP(&backupPath, "backupDir", "b", "", "location of backup directory")
-	cmdMigrate.MarkFlagRequired("dataDir")
-	cmdMigrate.MarkFlagRequired("backupDir")
+	err := cmdMigrate.MarkFlagRequired("dataDir")
+	if err != nil {
+		fmt.Println("Error setting flag dataDir")
+		return
+	}
+	err = cmdMigrate.MarkFlagRequired("backupDir")
+	if err != nil {
+		fmt.Println("Error setting flag backupDir")
+		return
+	}
 
-	err := cmdMigrate.Execute()
+	err = cmdMigrate.Execute()
 	if err != nil {
 		fmt.Println("exec error", err)
 	}
@@ -109,6 +130,9 @@ func BuildSchema(dataDir string) ([]byte, error) {
 				index := t[1]
 				src := dataDir + pathX
 				content, err := ioutil.ReadFile(src)
+				if err != nil {
+					return err
+				}
 				fstat, err := os.Stat(src)
 				stat := fstat.Sys().(*syscall.Stat_t)
 				if err != nil {
@@ -186,11 +210,20 @@ func (d *rbfFile) Close() error {
 	if d.last != "" {
 		d.working.Close()
 		//if d.last exists only keep the biggest
-		os.MkdirAll(filepath.Dir(d.last), 0777)
+		err := os.MkdirAll(filepath.Dir(d.last), 0777)
+		if err != nil {
+			return err
+		}
 		// move the datafile backup shard
-		os.Rename(d.temp+"/data", d.last)
+		err = os.Rename(d.temp+"/data", d.last)
+		if err != nil {
+			return err
+		}
 		//cleanup the tempdirectory
-		os.RemoveAll(d.temp)
+		err = os.RemoveAll(d.temp)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -208,8 +241,11 @@ func copyFile(src, dest string) error {
 }
 
 func Migrate(dataDir, backupPath string) error {
-	os.MkdirAll(backupPath, 0777)
-	err := copyFile(dataDir+"/idalloc.db", backupPath+"idalloc")
+	err := os.MkdirAll(backupPath, 0777)
+	if err != nil {
+		return err
+	}
+	err = copyFile(dataDir+"/idalloc.db", backupPath+"idalloc")
 	if err != nil {
 		return err
 	}
@@ -318,7 +354,10 @@ func Migrate(dataDir, backupPath string) error {
 
 func writeIfBigger(dst string, content []byte) error {
 	if stats, err := os.Stat(dst); os.IsNotExist(err) {
-		os.MkdirAll(filepath.Dir(dst), 0777)
+		err = os.MkdirAll(filepath.Dir(dst), 0777)
+		if err != nil {
+			return err
+		}
 		return ioutil.WriteFile(dst, content, 0644)
 	} else {
 		if stats.Size() < int64(len(content)) {
@@ -337,12 +376,6 @@ func ignore(path string, items ...string) bool {
 	return false
 }
 
-func panicOnErr(err error) {
-	if err != nil {
-		panic(err)
-	}
-}
-
 func FetchIndexKeys(base string) []string {
 	var directory []string
 

go.mod

@@ -52,7 +52,7 @@ require (
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
 	sigs.k8s.io/yaml v1.2.0 // indirect

hack.go

@@ -0,0 +1,77 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/v2/pb"
+	"github.com/pilosa/pilosa/v2/pql"
+)
+
+func UnmarshalIndexOptions(name string, createdAt int64, buf []byte) (*IndexOptions, error) {
+	var io pb.IndexMeta
+	// Read data from meta file.
+	if err := proto.Unmarshal(buf, &io); err != nil {
+		return nil, err
+	}
+	return &IndexOptions{
+		Keys:           io.Keys,
+		TrackExistence: io.TrackExistence,
+	}, nil
+
+}
+func UnmarshalFieldOptions(name string, createdAt int64, buf []byte) (*FieldInfo, error) {
+	var pbi pb.FieldOptions
+
+	// Read data from meta file.
+	if err := proto.Unmarshal(buf, &pbi); err != nil {
+		return nil, err
+	}
+	fi := &FieldInfo{}
+	fi.Name = name
+	fi.CreatedAt = createdAt
+	fi.Options = FieldOptions{}
+	// Initialize "base" to "min" when upgrading from v1 BSI format.
+	if pbi.BitDepth == 0 {
+		pbi.Base = bsiBase(pbi.OldMin, pbi.OldMax)
+		pbi.BitDepth = uint64(bitDepthInt64(pbi.OldMax - pbi.OldMin))
+		if pbi.BitDepth == 0 {
+			pbi.BitDepth = 1
+		}
+	}
+
+	// Copy metadata fields.
+	fi.Options.Type = pbi.Type
+	if pbi.Type == "decimal" {
+		fi.Options.Scale = 3
+	}
+	fi.Options.CacheType = pbi.CacheType
+	fi.Options.CacheSize = pbi.CacheSize
+	fi.Options.Min = pql.Decimal{
+		Value: pbi.OldMin,
+		Scale: 3, //what scale?
+	}
+	fi.Options.Max = pql.Decimal{
+		Value: pbi.OldMax,
+		Scale: 3, //what scale? its 3
+	}
+	fi.Options.Base = pbi.Base
+	fi.Options.BitDepth = pbi.BitDepth
+	fi.Options.TimeQuantum = TimeQuantum(pbi.TimeQuantum)
+	fi.Options.Keys = pbi.Keys
+	fi.Options.NoStandardView = pbi.NoStandardView
+
+	return fi, nil
+}