From 023d05aaf970e50474901524e214bbe19aeffa8b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 16 Jul 2021 14:22:14 -0500 Subject: [PATCH] linter fix --- cmd/roaring-migrate/main.go | 67 ++++++++++++++++++++++++-------- go.mod | 2 +- hack.go | 77 +++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 18 deletions(-) create mode 100644 hack.go diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 02cd7e875..3889ba9ef 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -1,3 +1,16 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package main import ( @@ -23,8 +36,8 @@ func main() { var dataDir, backupPath string cmdMigrate := &cobra.Command{ Use: "roaring-migrate", - Short: "TBD", - Long: ` TBD`, + Short: "convert roaring pilosa backup to rbf", + Long: `roaring-migrate uses the pilosa data-dir for each node, and produces a new backup that is able to be restored from utilizing the new pilosa restore tool.`, //Args: cobra.MinimumNArgs(2), Run: func(cmd *cobra.Command, args []string) { nodes := strings.Split(dataDir, ",") @@ -40,10 +53,18 @@ func main() { } cmdMigrate.Flags().StringVarP(&dataDir, "dataDir", "d", "", "source directories for each node seperated by commas") cmdMigrate.Flags().StringVarP(&backupPath, "backupDir", "b", "", "location of backup directory") - cmdMigrate.MarkFlagRequired("dataDir") - cmdMigrate.MarkFlagRequired("backupDir") + err := cmdMigrate.MarkFlagRequired("dataDir") + if err != nil { + fmt.Println("Error setting flag dataDir") + return + } + err = cmdMigrate.MarkFlagRequired("backupDir") + if err != nil { + fmt.Println("Error setting flag backupDir") + return + } - err := cmdMigrate.Execute() + err = cmdMigrate.Execute() if err != nil { fmt.Println("exec error", err) } @@ -109,6 +130,9 @@ func BuildSchema(dataDir string) ([]byte, error) { index := t[1] src := dataDir + pathX content, err := ioutil.ReadFile(src) + if err != nil { + return err + } fstat, err := os.Stat(src) stat := fstat.Sys().(*syscall.Stat_t) if err != nil { @@ -186,11 +210,20 @@ func (d *rbfFile) Close() error { if d.last != "" { d.working.Close() //if d.last exists only keep the biggest - os.MkdirAll(filepath.Dir(d.last), 0777) + err := os.MkdirAll(filepath.Dir(d.last), 0777) + if err != nil { + return err + } // move the datafile backup shard - os.Rename(d.temp+"/data", d.last) + err = os.Rename(d.temp+"/data", d.last) + if err != nil { + return err + } //cleanup the tempdirectory - os.RemoveAll(d.temp) + err = os.RemoveAll(d.temp) + if err != nil { + return err + } } return nil } @@ -208,8 +241,11 @@ func copyFile(src, dest string) error { } func Migrate(dataDir, backupPath string) error { - os.MkdirAll(backupPath, 0777) - err := copyFile(dataDir+"/idalloc.db", backupPath+"idalloc") + err := os.MkdirAll(backupPath, 0777) + if err != nil { + return err + } + err = copyFile(dataDir+"/idalloc.db", backupPath+"idalloc") if err != nil { return err } @@ -318,7 +354,10 @@ func Migrate(dataDir, backupPath string) error { func writeIfBigger(dst string, content []byte) error { if stats, err := os.Stat(dst); os.IsNotExist(err) { - os.MkdirAll(filepath.Dir(dst), 0777) + err = os.MkdirAll(filepath.Dir(dst), 0777) + if err != nil { + return err + } return ioutil.WriteFile(dst, content, 0644) } else { if stats.Size() < int64(len(content)) { @@ -337,12 +376,6 @@ func ignore(path string, items ...string) bool { return false } -func panicOnErr(err error) { - if err != nil { - panic(err) - } -} - func FetchIndexKeys(base string) []string { var directory []string diff --git a/go.mod b/go.mod index ccb21d162..3eb1fdbe6 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.3.0 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/hack.go b/hack.go new file mode 100644 index 000000000..c464cbf6e --- /dev/null +++ b/hack.go @@ -0,0 +1,77 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2/pb" + "github.com/pilosa/pilosa/v2/pql" +) + +func UnmarshalIndexOptions(name string, createdAt int64, buf []byte) (*IndexOptions, error) { + var io pb.IndexMeta + // Read data from meta file. + if err := proto.Unmarshal(buf, &io); err != nil { + return nil, err + } + return &IndexOptions{ + Keys: io.Keys, + TrackExistence: io.TrackExistence, + }, nil + +} +func UnmarshalFieldOptions(name string, createdAt int64, buf []byte) (*FieldInfo, error) { + var pbi pb.FieldOptions + + // Read data from meta file. + if err := proto.Unmarshal(buf, &pbi); err != nil { + return nil, err + } + fi := &FieldInfo{} + fi.Name = name + fi.CreatedAt = createdAt + fi.Options = FieldOptions{} + // Initialize "base" to "min" when upgrading from v1 BSI format. + if pbi.BitDepth == 0 { + pbi.Base = bsiBase(pbi.OldMin, pbi.OldMax) + pbi.BitDepth = uint64(bitDepthInt64(pbi.OldMax - pbi.OldMin)) + if pbi.BitDepth == 0 { + pbi.BitDepth = 1 + } + } + + // Copy metadata fields. + fi.Options.Type = pbi.Type + if pbi.Type == "decimal" { + fi.Options.Scale = 3 + } + fi.Options.CacheType = pbi.CacheType + fi.Options.CacheSize = pbi.CacheSize + fi.Options.Min = pql.Decimal{ + Value: pbi.OldMin, + Scale: 3, //what scale? + } + fi.Options.Max = pql.Decimal{ + Value: pbi.OldMax, + Scale: 3, //what scale? its 3 + } + fi.Options.Base = pbi.Base + fi.Options.BitDepth = pbi.BitDepth + fi.Options.TimeQuantum = TimeQuantum(pbi.TimeQuantum) + fi.Options.Keys = pbi.Keys + fi.Options.NoStandardView = pbi.NoStandardView + + return fi, nil +}