diff --git a/boltdb/translate.go b/boltdb/translate.go index 397027dab..55f4c43fd 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -60,13 +60,14 @@ var _ pilosa.TranslateStore = &TranslateStore{} // TranslateStore is an on-disk storage engine for translating string-to-uint64 values. // An empty string will be converted into the sentinel byte slice: -// var emptyKey = []byte{ -// 0x00, 0x00, 0x00, -// 0x4d, 0x54, 0x4d, 0x54, // MTMT -// 0x00, -// 0xc2, 0xa0, // NO-BREAK SPACE -// 0x00, -// } +// +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore struct { mu sync.RWMutex db *bolt.DB diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 93477bb27..46227401d 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -6,8 +6,6 @@ import ( "archive/tar" "compress/gzip" "context" - "time" - "fmt" "io" gohttp "net/http" @@ -40,8 +38,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { tarReader = tar.NewReader(f) } viewData := make(map[string][]byte) - //given ordered by index/field/view - //trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255 + // given ordered by index/field/view + // trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255 lastIndex := "" lastField := "" lastShard := uint64(0) @@ -52,7 +50,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { if header != nil { vprint.PanicOn("header should not be nil on err io.EOF") } - //submit any stuff we have left + // submit any stuff we have left if len(viewData) > 0 { request := &pilosa.ImportRoaringRequest{ Views: viewData, @@ -69,7 +67,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { vprint.VV("n = %v, progress, elapsed '%v'", n, time.Since(t0)) } parts := strings.Split(header.Name, "/") - //vv("parts = '%#v'", parts) + // vv("parts = '%#v'", parts) index := parts[1] field := parts[2] view := parts[4] @@ -83,11 +81,11 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { request := &pilosa.ImportRoaringRequest{ Views: viewData, } - //vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) + // vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) uri := GetImportRoaringURI(lastIndex, lastShard) vprint.PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)) viewData = make(map[string][]byte) - //vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0)) + // vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0)) } } @@ -102,8 +100,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { lastIndex = index lastField = field - //lastShard = shard - //vv("bottom of loop") + // lastShard = shard + // vv("bottom of loop") } } @@ -111,7 +109,6 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { // the new "good" loader, and should always be preferred now // when not trying to repro that bug. pulled from 85fa67e8 func main() { - host := "127.0.0.1:10101" h := &gohttp.Client{} c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 81e6d4278..afe826c72 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -23,8 +23,10 @@ import ( "github.com/spf13/cobra" ) -var visited map[string]int64 -var glogger = logger.NewStandardLogger(os.Stdout) +var ( + visited map[string]int64 + glogger = logger.NewStandardLogger(os.Stdout) +) const ( Version = "1.0" @@ -33,6 +35,7 @@ const ( func main() { os.Exit(realMain()) } + func realMain() int { visited = make(map[string]int64) var dataDir, backupPath string @@ -82,7 +85,6 @@ func FetchFragments(base string) []string { var fragments []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -98,7 +100,6 @@ func FetchFragments(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } @@ -121,14 +122,13 @@ func fileExists(filename string) (bool, int64) { } func BuildSchema(dataDir string) ([]byte, error) { - //need to find all the ".meta" files and load as field options + // need to find all the ".meta" files and load as field options schemaSerializer := struct { Indexes []*local `json:"indexes,omitempty"` }{Indexes: make([]*local, 0)} var l *local ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Infof("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -136,10 +136,10 @@ func BuildSchema(dataDir string) ([]byte, error) { } pathX = pathX[len(dataDir):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { if strings.Contains(pathX, ".meta") { - //convert the file to a fieldOptions + // convert the file to a fieldOptions // ex: metaPath /trait_store/aba/.meta glogger.Infof("PATHX %v", pathX) t := strings.Split(pathX, "/") @@ -165,7 +165,7 @@ func BuildSchema(dataDir string) ([]byte, error) { CreatedAt: uint64(CTimeNano(stat)), Options: *io, } - //index options + // index options schemaSerializer.Indexes = append(schemaSerializer.Indexes, l) return nil } @@ -184,14 +184,14 @@ func BuildSchema(dataDir string) ([]byte, error) { } err := filepath.Walk(dataDir, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", dataDir, err) } return json.MarshalIndent(schemaSerializer, "", " ") } + func Extract(filename string) (index, field, view string, shard uint64) { - //trait_store/aba/views/standard/fragments + // trait_store/aba/views/standard/fragments parts := strings.Split(filename, "/") shard, _ = strconv.ParseUint(parts[6], 10, 64) return parts[1], parts[2], parts[4], shard @@ -222,6 +222,7 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) { } return d.working, nil } + func (d *rbfFile) Close() error { defer func() error { // clean up the temp directory @@ -235,11 +236,11 @@ func (d *rbfFile) Close() error { if d.last != "" { d.working.Close() - //if d.last exists only keep the biggest + // if d.last exists only keep the biggest exists, sz := fileExists(d.last) src := filepath.Join(d.temp, "data") if !exists { - err := os.MkdirAll(filepath.Dir(d.last), 0750) + err := os.MkdirAll(filepath.Dir(d.last), 0o750) if err != nil { return err } @@ -257,6 +258,7 @@ func (d *rbfFile) Close() error { } return nil } + func copyFile(src, dest string) error { from, err := os.Open(src) if err != nil { @@ -264,7 +266,7 @@ func copyFile(src, dest string) error { } defer from.Close() - to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644) + to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0o644) if err != nil { return err } @@ -280,7 +282,7 @@ func copyFile(src, dest string) error { func Migrate(dataDir, backupPath string, verbose bool) error { dataDir = strings.TrimSuffix(dataDir, "/") - err := os.MkdirAll(backupPath, 0750) + err := os.MkdirAll(backupPath, 0o750) if err != nil { return err } @@ -300,7 +302,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { raw := FetchFragments(dataDir) sort.Slice(raw, func(i, j int) bool { - //trait_store/zip_code/views/standard/fragments/ + // trait_store/zip_code/views/standard/fragments/ ti := strings.LastIndex(raw[i], "/") + 1 shardi, err := strconv.ParseUint(raw[i][ti:], 10, 16) if err != nil { @@ -318,7 +320,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } return false }) - //raw is now sorted by shard + // raw is now sorted by shard cache := &rbfFile{ temp: filepath.Join(backupPath, "_SCRATCH"), @@ -356,8 +358,14 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } key := string(txkey.Prefix(index, field, view, shard)) tx, err := db.Begin(true) + if err != nil { + return err + } tx.AddRoaring(key, bm) err = tx.Commit() + if err != nil { + return err + } } cache.Close() keys := FetchIndexKeys(dataDir) @@ -372,7 +380,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } } - //deal with index field(row)keys + // deal with index field(row)keys keys = FetchRowkeys(dataDir) for _, filename := range keys { glogger.Infof("field %v", filename) @@ -389,7 +397,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { func writeIfBigger(dst string, srcFile string) error { if stats, err := os.Stat(dst); os.IsNotExist(err) { - err = os.MkdirAll(filepath.Dir(dst), 0750) + err = os.MkdirAll(filepath.Dir(dst), 0o750) if err != nil { return err } @@ -404,8 +412,9 @@ func writeIfBigger(dst string, srcFile string) error { return copyFile(srcFile, dst) } } - return nil //simply skip it + return nil // simply skip it } + func ignore(path string, items ...string) bool { f := filepath.Base(path) for i := range items { @@ -420,7 +429,6 @@ func FetchIndexKeys(base string) []string { var directory []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -428,7 +436,7 @@ func FetchIndexKeys(base string) []string { } pathX = pathX[len(base):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { if strings.Contains(pathX, "_keys") && !ignore(pathX, ".data", "keys") { directory = append(directory, pathX) @@ -438,7 +446,6 @@ func FetchIndexKeys(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } @@ -449,7 +456,6 @@ func FetchRowkeys(base string) []string { var directory []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -457,13 +463,13 @@ func FetchRowkeys(base string) []string { } pathX = pathX[len(base):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { fp := filepath.Base(pathX) if fp == "keys" { p := strings.Split(pathX, "/") if len(p) != 4 { - return nil //skip all but field/key files + return nil // skip all but field/key files } directory = append(directory, pathX) } @@ -472,7 +478,6 @@ func FetchRowkeys(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } diff --git a/cmd/roaring-migrate/main_test.go b/cmd/roaring-migrate/main_test.go index f83a9bf0c..0e8740d3a 100644 --- a/cmd/roaring-migrate/main_test.go +++ b/cmd/roaring-migrate/main_test.go @@ -26,13 +26,15 @@ func TestFileExists(t *testing.T) { } func TestMainProgram(t *testing.T) { - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose", } if realMain() == 0 { t.Fatal("should fail and it succeeded") } - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose", } if realMain() == 0 { @@ -43,7 +45,8 @@ func TestMainProgram(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(dir) // clean up - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose=true", "--data-dir=testdata/data-dir/", "--backup-dir=" + dir, @@ -51,5 +54,4 @@ func TestMainProgram(t *testing.T) { if realMain() == 1 { t.Fatal("shouldn't fail") } - } diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 340af3c97..21e30e484 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -85,9 +85,8 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { index := parts[1] fieldName := parts[2] if fieldName == "_keys" { - //skip index keys are not not real fields so will have no need for field keys + // skip index keys are not not real fields so will have no need for field keys return nil - } byteData, err := io.ReadAll(tr) @@ -121,6 +120,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { r.state = parts[0] return nil } + func (r *stateMachine) Upload() error { if len(r.viewData) > 0 { request := &pilosa.ImportRoaringRequest{ @@ -242,7 +242,6 @@ func stopProfile(host, outfile string) { defer fd.Close() _, err = io.Copy(fd, resp.Body) vprint.PanicOn(err) - } var globURI *pnet.URI diff --git a/dbshard.go b/dbshard.go index c1d10b070..95a6d7788 100644 --- a/dbshard.go +++ b/dbshard.go @@ -457,7 +457,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir // // when a new DBShard is made, we will update the list of shards then. Thus // the per.index2shard should always be up to date AFTER the first call here. -// func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]struct{}, err error) { // use the cache, always diff --git a/idk/api/source_test.go b/idk/api/source_test.go index c8108240d..03a32b5d5 100644 --- a/idk/api/source_test.go +++ b/idk/api/source_test.go @@ -46,19 +46,19 @@ func TestIngest(t *testing.T) { unkeyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) unkeyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true)) unkeyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) - //unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) + // unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) unkeyedIdx.Field("int", pilosaclient.OptFieldTypeInt()) unkeyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2)) - //unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) + // unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) keyedIdx := schema.Index("apitestingest_keyed", pilosaclient.OptIndexTrackExistence(true), pilosaclient.OptIndexKeys(true)) keyedIdx.Field("idset") keyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) keyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true)) keyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) - //keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) + // keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) keyedIdx.Field("int", pilosaclient.OptFieldTypeInt()) keyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2)) - //keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) + // keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) err = client.SyncSchema(schema) if !assert.NoError(t, err) { diff --git a/idk/cmd/molecula-consumer-github/main.go b/idk/cmd/molecula-consumer-github/main.go index 2af592adc..edc664c7d 100644 --- a/idk/cmd/molecula-consumer-github/main.go +++ b/idk/cmd/molecula-consumer-github/main.go @@ -285,6 +285,7 @@ func (s *Source) issueSchema() []idk.Field { idk.RecordTimeField{NameVal: "created_at", Layout: time.RFC3339}, } } + func (s *Source) Close() error { return nil } diff --git a/idk/common/configMap.go b/idk/common/configMap.go index 3d7d138e2..bfdb0700e 100644 --- a/idk/common/configMap.go +++ b/idk/common/configMap.go @@ -43,6 +43,7 @@ func LaunchKafkaEventConfirmer(producer *confluent.Producer, finished *int32, it }() return doneChan } + func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { var err error configMap := &confluent.ConfigMap{} @@ -84,7 +85,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { } } - //SSL + // SSL if m.KafkaSslCaLocation != "" { err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation) if err != nil { @@ -121,7 +122,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { } } - //SSL + // SSL if m.KafkaSslCaLocation != "" { err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation) if err != nil { diff --git a/idk/csv/csv_test.go b/idk/csv/csv_test.go index b0f6f361d..3902f358d 100644 --- a/idk/csv/csv_test.go +++ b/idk/csv/csv_test.go @@ -325,7 +325,7 @@ func testDirTree(tree string) (string, error) { return "", err } - err = os.MkdirAll(filepath.Join(tmp, tree), 0755) + err = os.MkdirAll(filepath.Join(tmp, tree), 0o755) if err != nil { os.RemoveAll(tmp) return "", err @@ -489,7 +489,6 @@ func testTimestampRunner(t *testing.T, m *Main, testCase TimestampTestCase) { check := testCase.expect.([]interface{}) testExtractRowsQuery(t, m.Index, testCase.fieldName, check) - } // Test that out of range int values are ingested as nil when AllowIntOutOfRange is true. @@ -531,7 +530,6 @@ id__ID,negneg__Int_-10_-5,negpos__Int_-10_10,pospos__Int_5_10,negzero__Int_-10_0 } }) } - } // Test that out of range timestamp values are ingested as nil when AllowTimestampOutOfRange is true. @@ -555,7 +553,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15: checker["ts4"] = []interface{}{nil, "0001-01-01T00:00:01Z", "0001-01-01T00:00:02Z", "9999-12-31T23:59:58Z", "9999-12-31T23:59:59Z", nil} batchSizes := []int{3, 1, 4, 10} for _, bsize := range batchSizes { - t.Run(fmt.Sprintf("batchsize=%d", bsize), func(t *testing.T) { m := newMainOORFactory(t, file, false, false, true) m.BatchSize = bsize @@ -572,7 +569,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15: } }) } - } // Tests various conditions that should halt ingest @@ -617,7 +613,8 @@ func TestFailureConditions(t *testing.T) { {name: "int string overflow", csv: `id__ID,pospos__Int 0,"89273948723984729387492387492987" -`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}} +`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}, + } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { diff --git a/idk/datagen/common.go b/idk/datagen/common.go index 5e58daa46..cee21f986 100644 --- a/idk/datagen/common.go +++ b/idk/datagen/common.go @@ -8,7 +8,9 @@ import ( // data by providing one or more idk.Source. It also contains // a method for getting information about the supported sources. // Info() - return information about the generator along with -// a list of supported types +// +// a list of supported types +// // Sources() - provided a string key and configuration, returns // a list of primary key fields and a list of sources. type SourceGenerator interface { diff --git a/idk/datagen/custom.go b/idk/datagen/custom.go index d81a422a0..444e0fbe0 100644 --- a/idk/datagen/custom.go +++ b/idk/datagen/custom.go @@ -44,7 +44,6 @@ func NewCustom(cfg SourceGeneratorConfig) Sourcer { IDKAndGenFields: ig, } } - } // CustomConfig represents the JSON/yaml configuration for the "custom" datagen source. @@ -373,7 +372,7 @@ func (cs *CustomSource) Record() (idk.Record, error) { last = cs.record[i] } if cs.recordsToGenerate > 0 && cs.recordCounter >= cs.recordsToGenerate { - //break when number records produced + // break when number records produced return nil, io.EOF } else { cs.recordCounter++ diff --git a/idk/datagen/custom_test.go b/idk/datagen/custom_test.go index 9a07fe29a..e636c5afa 100644 --- a/idk/datagen/custom_test.go +++ b/idk/datagen/custom_test.go @@ -143,7 +143,6 @@ func TestGetIDKFields(t *testing.T) { t.Fatalf("mismatched genFields at %d: got:\n%+v\nexp:\n%+v", i, gotGenField, expGenField) } } - } func TestNewCustomUnmarshalling(t *testing.T) { @@ -180,7 +179,6 @@ fields: if cc.CustomConfig.Fields[0].TimeFormat != unixFormat { t.Fatalf("wrong time format: %s", cc.CustomConfig.Fields[0].TimeFormat) } - } func TestIncreasingTimestampGenerator(t *testing.T) { @@ -232,7 +230,6 @@ func TestIncreasingTimestampGenerator(t *testing.T) { } else if recInt < testcase.min || recInt > testcase.max { t.Fatalf("unexpected value for time: %v", recInt) } - }) } } diff --git a/idk/header.go b/idk/header.go index 93a7fd4a5..74581effa 100644 --- a/idk/header.go +++ b/idk/header.go @@ -42,12 +42,12 @@ var ( // destname are separated by "___", triple underscore) and converts it // to an idk Field like: // -// FieldTypeField { -// NameVal: sourcename, -// DestNameVal: destname, -// Thing1: Arg, -// Thing2: Arg2, -// } +// FieldTypeField { +// NameVal: sourcename, +// DestNameVal: destname, +// Thing1: Arg, +// Thing2: Arg2, +// } // // It does this using a variety of reflective magic. The unwritten // rules are that all idk Fields must be structs and have their first diff --git a/idk/interfaces.go b/idk/interfaces.go index c1454cfa4..950a651e0 100644 --- a/idk/interfaces.go +++ b/idk/interfaces.go @@ -893,10 +893,10 @@ func validateTimestamp(unit Unit, ts time.Time) error { return nil } -// validateDuration checks if the duration will overflow. Users can provide a custom epoch but -// Featurebase will ultimately convert this to some duration relative to the Unix epoch. -// So if the custom epoch + the provided value in the desired units is too far from -// Unix epoch such that it causes an interger overflow, this will return an error. +// validateDuration checks if the duration will overflow. Users can provide a custom epoch but +// Featurebase will ultimately convert this to some duration relative to the Unix epoch. +// So if the custom epoch + the provided value in the desired units is too far from +// Unix epoch such that it causes an interger overflow, this will return an error. func validateDuration(dur int64, offset int64, granularity Unit) error { var minInt, maxInt int64 switch granularity { diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index c4d445552..f3cd26477 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -24,12 +24,14 @@ import ( liavro "github.com/linkedin/goavro/v2" ) -var pilosaHost string -var pilosaTLSHost string -var pilosaGrpcHost string -var kafkaHost string -var registryHost string -var certPath string +var ( + pilosaHost string + pilosaTLSHost string + pilosaGrpcHost string + kafkaHost string + registryHost string + certPath string +) func init() { var ok bool @@ -181,7 +183,6 @@ func TestConfigOptions(t *testing.T) { if val, err := cfg.Get("auto.offset.reset", nil); err != nil || val.(string) != "latest" { t.Fatalf("unexpected val for auto.offset.reset val: %v, err: %v", val, err) } - } func TestCmdMainOne(t *testing.T) { @@ -652,6 +653,7 @@ func (s sortableCRI) Less(i, j int) bool { } return true } + func (s sortableCRI) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/idk/kafka/put.go b/idk/kafka/put.go index 1f6c943e9..ae56c1986 100644 --- a/idk/kafka/put.go +++ b/idk/kafka/put.go @@ -59,7 +59,6 @@ func (p *PutCmd) Run() (err error) { } var auth *csrc.BasicAuth if p.SchemaRegistryUsername != "" { - auth = &csrc.BasicAuth{ KafkaSchemaApiKey: p.SchemaRegistryUsername, KafkaSchemaApiSecret: p.SchemaRegistryPassword, @@ -128,6 +127,7 @@ func (p *PutCmd) Run() (err error) { return nil } + func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, numPartitions int, replicationFactor int) error { a, err := confluent.NewAdminClientFromProducer(p) if err != nil { @@ -152,7 +152,8 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, []confluent.TopicSpecification{{ Topic: topic, NumPartitions: numPartitions, - ReplicationFactor: replicationFactor}}, + ReplicationFactor: replicationFactor, + }}, // Admin options confluent.SetAdminOperationTimeout(maxDur)) if err != nil { @@ -165,6 +166,7 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, } return nil } + func (p *PutCmd) getSchema() (string, error) { if p.Schema == "" && p.SchemaFile == "" { return "", errors.New("need a string schema or schema file") diff --git a/idk/kafka/source_test.go b/idk/kafka/source_test.go index eaeccca3c..27911b3a5 100644 --- a/idk/kafka/source_test.go +++ b/idk/kafka/source_test.go @@ -298,7 +298,8 @@ var tests = []struct { }, exp: [][]interface{}{ {"a", true, int64(101), []byte{9, 196}, float64(9.4921)}, - {nil, nil, nil, nil, nil}}, + {nil, nil, nil, nil, nil}, + }, }, { schemaFile: "floatscale.json", @@ -434,27 +435,28 @@ func TestKafkaSourceSchemaChangeCommitRegression(t *testing.T) { } go func() { - topic := "xyzzy" - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, - Partition: 0, - Offset: confluent.Offset(0), + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + Partition: 0, + Offset: confluent.Offset(0), + }, + Timestamp: time.Now(), + Value: data1, }, - Timestamp: time.Now(), - Value: data1, - }, } - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, - Partition: 0, - Offset: confluent.Offset(1), + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + Partition: 0, + Offset: confluent.Offset(1), + }, + Timestamp: time.Now(), + Value: data2, }, - Timestamp: time.Now(), - Value: data2, - }, } close(src.recordChannel) }() @@ -508,14 +510,14 @@ func TestKafkaSourceTimeout(t *testing.T) { go func() { topic := "test" - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + }, + Value: buf, }, - Value: buf, - }, } - }() // ensure we can get a message if one is available @@ -621,7 +623,6 @@ func TestKafkaSourceIntegration(t *testing.T) { } }) } - } func mustNewProducer(t *testing.T, kafkaHost string) *confluent.Producer { @@ -767,7 +768,6 @@ func TestKafkaSourceNotAutoCommitting(t *testing.T) { } else if off := offsets[0]; int64(off.Offset) != numRecords { t.Fatalf("after commit, offset is not %d: %d", numRecords-1, off.Offset) } - } func TestRegistryURLParsing(t *testing.T) { @@ -844,10 +844,8 @@ func TestRegistryURLParsing(t *testing.T) { if codecURL != test.expectedCodecURL { t.Errorf("codec URL exp:\n%s\ngot:\n%s", test.expectedCodecURL, codecURL) } - }) } - } func postSchema(t *testing.T, schemaFile, subj, regURL string, tlsConfig *tls.Config) (schemaID int) { @@ -883,7 +881,8 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) { []confluent.TopicSpecification{{ Topic: topic, NumPartitions: 64, - ReplicationFactor: 1}}, + ReplicationFactor: 1, + }}, // Admin options confluent.SetAdminOperationTimeout(maxDur)) if err != nil { @@ -895,7 +894,6 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) { } } a.Close() - } func tPutRecordsKafka(t *testing.T, p *confluent.Producer, topic string, schemaID int, schema *liavro.Codec, key string, records ...map[string]interface{}) { diff --git a/idk/kafka_static/source_test.go b/idk/kafka_static/source_test.go index e1fc37cb7..82b1afc37 100644 --- a/idk/kafka_static/source_test.go +++ b/idk/kafka_static/source_test.go @@ -84,7 +84,6 @@ func TestKafkaStaticSourceLocal(t *testing.T) { for _, exp := range test.exp { rec, err := src.Record() - if err != nil { t.Fatalf("unexpected error getting record: %v", err) } @@ -171,7 +170,6 @@ func TestKafkaStaticSourceIntegration(t *testing.T) { } }) } - } func tCreateTopic(t *testing.T, topic string, addr net.Addr) { diff --git a/idk/kafkagen/cmd.go b/idk/kafkagen/cmd.go index 2c6e29ca1..e7ee3fa15 100644 --- a/idk/kafkagen/cmd.go +++ b/idk/kafkagen/cmd.go @@ -57,7 +57,7 @@ func (m *Main) Run() (err error) { {"2", "1", 159, map[string]interface{}{"boolean": true}, map[string]interface{}{"boolean": false}, map[string]interface{}{"string": "cgr"}, map[string]interface{}{"array": []string{"a", "b"}}, nil, map[string]interface{}{"int": 7}, nil, nil, map[string]interface{}{"float": 5.4}, nil, map[string]interface{}{"org.test.survey1234": "yes"}, map[string]interface{}{"float": 8.0}, nil}, } - //kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) { + // kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) { p, err := confluent.NewProducer(m.configMap) if err != nil { @@ -101,7 +101,7 @@ func (m *Main) Run() (err error) { return errors.Wrap(err, "putting record") } } - <-doneChan //wait till all messages are acked + <-doneChan // wait till all messages are acked p.Flush(10 * 1000) return nil diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index 9f24525fd..cc0724327 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -6,7 +6,7 @@ import ( "context" "fmt" "hash/fnv" - "io/ioutil" + "io" "math/rand" "os" "os/exec" @@ -102,7 +102,7 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, if err != nil { return err } - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { return err } @@ -148,7 +148,7 @@ func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, // filter out non-file entries filePaths := make([]string, 0, len(dirEntries)) for _, entry := range dirEntries { - if entry.Mode().IsDir() { + if entry.IsDir() { continue } filePath := filepath.Join(dirPath, entry.Name()) @@ -189,7 +189,7 @@ func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, ke // create dir to store boltdbs for this node nodeDirPath := filepath.Join(dirPath, node) - err = os.Mkdir(nodeDirPath, 0755) + err = os.Mkdir(nodeDirPath, 0o755) if err != nil { return err } @@ -388,7 +388,7 @@ func TestPauseReplica(t *testing.T) { t.Fatal(err) } dirPath = filepath.Join(dirPath, keysDirName) - err = os.Mkdir(dirPath, 0755) + err = os.Mkdir(dirPath, 0o755) if err != nil { t.Fatal(err) } diff --git a/net/uri.go b/net/uri.go index 2278113cd..44f2d9fbe 100644 --- a/net/uri.go +++ b/net/uri.go @@ -30,12 +30,13 @@ var ( // 3) Port: Port of the URI. Default: 10101. // // All parts of the URI are optional. The following are equivalent: -// http://localhost:10101 -// http://localhost -// http://:10101 -// localhost:10101 -// localhost -// :10101 +// +// http://localhost:10101 +// http://localhost +// http://:10101 +// localhost:10101 +// localhost +// :10101 type URI struct { Scheme string `json:"scheme"` Host string `json:"host"` diff --git a/pql/parser_test.go b/pql/parser_test.go index 670e9adff..aac2ba5da 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -239,7 +239,6 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) - } func TestUnquote(t *testing.T) { @@ -287,7 +286,6 @@ func TestUnquote(t *testing.T) { } }) } - } func testErr(t *testing.T, exp string, actual error) (done bool) { diff --git a/rbf/cursorx.go b/rbf/cursorx.go index a3f2b561c..264409c24 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -13,7 +13,7 @@ import ( "github.com/pkg/errors" ) -//probably should just implement the container interface +// probably should just implement the container interface // but for now i'll do it func (c *Cursor) Rows() ([]uint64, error) { shardVsContainerExponent := uint(4) //needs constant exported from roaring package diff --git a/rbf/db.go b/rbf/db.go index 4204d2720..2cdaab7d7 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -20,9 +20,7 @@ import ( "github.com/featurebasedb/featurebase/v3/syswrap" ) -var ( - ErrClosed = errors.New("rbf: database closed") -) +var ErrClosed = errors.New("rbf: database closed") // shared cursor pool across all DB instances. // Cursors are returned on Cursor.Close(). @@ -119,14 +117,14 @@ func (db *DB) Open() (err error) { db.mu.Lock() defer db.mu.Unlock() - if err := os.MkdirAll(db.Path, 0755); err != nil { + if err := os.MkdirAll(db.Path, 0o755); err != nil { return err - } else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil { + } else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil { return fmt.Errorf("open file: %w", err) } // Open read-only database mmap. - if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0600); err != nil { + if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0o600); err != nil { return fmt.Errorf("open mmap file: %w", err) } else if db.data, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -164,12 +162,12 @@ func (db *DB) Open() (err error) { func (db *DB) openWAL() (err error) { // Open WAL file writer. - if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil { + if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil { return fmt.Errorf("open wal file: %w", err) } // Open read-only mmap. - if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0600); err != nil { + if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0o600); err != nil { return fmt.Errorf("open wal mmap file: %w", err) } else if db.wal, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxWALSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -502,7 +500,6 @@ func (db *DB) Close() (err error) { // We will internally create and rollback a read-only // transaction to answer this query. func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { - // Read a list of all bitmaps in Tx. tx, err := db.Begin(false) if err != nil { @@ -579,7 +576,6 @@ func (db *DB) init() error { // initMetaPage initializes the meta page. func (db *DB) initMetaPage() error { - page := allocPage() writeMetaMagic(page) writeMetaPageN(page, 3) @@ -591,7 +587,6 @@ func (db *DB) initMetaPage() error { // initRootRecordPage initializes the initial root record page. func (db *DB) initRootRecordPage() error { - page := allocPage() writePageNo(page, 1) writeFlags(page, PageTypeRootRecord) @@ -601,7 +596,6 @@ func (db *DB) initRootRecordPage() error { // initFreelistPage initializes the initial freelist btree page. func (db *DB) initFreelistPage() error { - page := allocPage() writePageNo(page, 2) writeFlags(page, PageTypeLeaf) @@ -787,7 +781,6 @@ func (db *DB) removeTx(tx *Tx) error { // Check performs an integrity check. func (db *DB) Check() error { - tx, err := db.Begin(false) if err != nil { return err diff --git a/rbf/page_map.go b/rbf/page_map.go index 3ba5f27e7..75a397cf9 100644 --- a/rbf/page_map.go +++ b/rbf/page_map.go @@ -1,7 +1,7 @@ /* start with https://github.com/benbjohnson/immutable and specialize Map -Copyright 2019 Ben Johnson +# Copyright 2019 Ben Johnson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/roaring/btree.go b/roaring/btree.go index bbc429704..6ff64c42e 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -635,9 +635,9 @@ func (t *tree) Set(k uint64, v *Container) { // (whatever, false) if it decides not to create or not to update the value of // the KV pair. // -// tree.Set(k, v) call conceptually equals calling +// tree.Set(k, v) call conceptually equals calling // -// tree.Put(k, func(uint64, bool){ return v, true }) +// tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Container, write bool)) (oldV *Container, written bool) { diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index d72d2d36c..fff06d5db 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -8,7 +8,7 @@ import "sync" const containerWidth = 1 << 16 -////////////////// array +// //////////////// array func arrayEmpty() []uint16 { return make([]uint16, 0) } @@ -77,7 +77,7 @@ func arrayEvenBitsSet() []uint16 { return array } -////////////////// bitmap +// //////////////// bitmap func bitmapEmpty() []uint64 { return make([]uint64, bitmapN) } @@ -162,7 +162,7 @@ func bitmapEvenBitsSet() []uint64 { return bitmap } -////////////////// run +// //////////////// run func runEmpty() []Interval16 { return make([]Interval16, 0) } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c4373f93e..70b7f6836 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1156,7 +1156,7 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) { } -//empty array test +// empty array test func TestBitmap_Xor_Empty(t *testing.T) { bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) empty := roaring.NewFileBitmap() diff --git a/short_txkey/txkey.go b/short_txkey/txkey.go index 18c738dea..aee4c80c1 100644 --- a/short_txkey/txkey.go +++ b/short_txkey/txkey.go @@ -33,7 +33,7 @@ func FieldViewFromFullKey(fullKey []byte) FieldView { // The roaringContainerKey argument to Key() is a container key into a roaring Container. // The return value from Key() is constructed as follows: // -// ~field;view', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} +// +// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} // // The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable // base-64 encoding. // // The shortest possible key is 14 bytes. It would be laid out like this: -// ~f;v<12345678# -// 12345678901234 +// +// ~f;v<12345678# +// 12345678901234 // // keys starting with '~' are regular value keys. // keys starting with '>' are symlink keys. // // NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -// func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) { prefix := Prefix(index, field, view, shard) @@ -101,9 +102,10 @@ func MustValidateKey(bkey []byte) { // KeyExtractContainerKey extracts the containerKey from bkey. // key example: field;view', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} +// +// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} // // The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable // base-64 encoding. // // The shortest possible key is 25 bytes. It would be laid out like this: -// ~i%f;v:12345678<12345678# -// 1234567890123456789012345 +// +// ~i%f;v:12345678<12345678# +// 1234567890123456789012345 // // keys starting with '~' are regular value keys. // keys starting with '>' are symlink keys. // // NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -// func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) { prefix := Prefix(index, field, view, shard) @@ -68,9 +69,12 @@ func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r } // ShardFromKey key example: index/field;view:shard