From 0aa5efcc51d96de31c4fe35eaa577c18305829fb Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 7 Nov 2022 10:51:55 -0600 Subject: [PATCH] staticcheck fixes (#2278) --- authz/authorization_test.go | 2 +- boltdb/translate.go | 15 +++--- cmd/badloader/badloader.go | 31 +++++------ cmd/pilosa-bench/main.go | 4 +- cmd/roaring-migrate/main.go | 68 +++++++++++++----------- cmd/roaring-migrate/main_test.go | 13 ++--- cmd/slurp/slurp.go | 12 ++--- ctl/auth_token.go | 2 +- dbshard.go | 1 - idk/api/source_test.go | 21 ++++---- idk/cmd/molecula-consumer-github/main.go | 6 +-- idk/common/configMap.go | 9 ++-- idk/csv/csv_test.go | 14 ++--- idk/datagen/common.go | 4 +- idk/datagen/custom.go | 6 +-- idk/datagen/custom_test.go | 6 +-- idk/datagen/dell.go | 4 +- idk/datagen/hughes.go | 4 +- idk/datagen/merck.go | 12 ++--- idk/header.go | 12 ++--- idk/idktest/idktest.go | 4 +- idk/interfaces.go | 8 +-- idk/internal/s3.go | 5 +- idk/kafka/cmd_test.go | 20 +++---- idk/kafka/csrc/csrc.go | 4 +- idk/kafka/put.go | 9 ++-- idk/kafka/source.go | 7 ++- idk/kafka/source_test.go | 63 +++++++++++----------- idk/kafka_sasl/source.go | 4 +- idk/kafka_static/source.go | 4 +- idk/kafka_static/source_test.go | 6 +-- idk/kafkagen/cmd.go | 7 ++- internal/clustertests/pause_node_test.go | 12 ++--- net/uri.go | 13 ++--- pql/parser_test.go | 4 +- rbf/cursorx.go | 2 +- rbf/db.go | 21 +++----- rbf/page_map.go | 2 +- roaring/btree.go | 4 +- roaring/roaring_helpers_test.go | 6 +-- roaring/roaring_test.go | 2 +- short_txkey/txkey.go | 26 +++++---- sql/extract.go | 14 +++-- sql3/parser/token.go | 2 +- sql3/sql_defs_between_test.go | 4 +- sql3/sql_defs_groupby_test.go | 2 +- sql3/sql_defs_like_test.go | 4 +- sql3/sql_defs_timequantum_test.go | 4 +- statik/filesystem.go | 4 +- time.go | 1 + txkey/txkey.go | 33 +++++++----- 51 files changed, 271 insertions(+), 276 deletions(-) diff --git a/authz/authorization_test.go b/authz/authorization_test.go index 5bc0602dc..c2aa0b8ce 100644 --- a/authz/authorization_test.go +++ b/authz/authorization_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/boltdb/translate.go b/boltdb/translate.go index 9f1bc4f5e..b5541f9ac 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -59,13 +59,14 @@ var _ pilosa.TranslateStore = &TranslateStore{} // TranslateStore is an on-disk storage engine for translating string-to-uint64 values. // An empty string will be converted into the sentinel byte slice: -// var emptyKey = []byte{ -// 0x00, 0x00, 0x00, -// 0x4d, 0x54, 0x4d, 0x54, // MTMT -// 0x00, -// 0xc2, 0xa0, // NO-BREAK SPACE -// 0x00, -// } +// +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore struct { mu sync.RWMutex db *bolt.DB diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 035b70dea..d46909089 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -5,21 +5,19 @@ import ( "archive/tar" "compress/gzip" "context" - "time" - "fmt" "io" - "io/ioutil" + "os" + "strconv" + "strings" + "time" + gohttp "net/http" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/encoding/proto" pnet "github.com/molecula/featurebase/v3/net" "github.com/molecula/featurebase/v3/vprint" - - "os" - "strconv" - "strings" ) func UploadTar(srcFile string, client *pilosa.InternalClient) error { @@ -40,8 +38,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { tarReader = tar.NewReader(f) } viewData := make(map[string][]byte) - //given ordered by index/field/view - //trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255 + // given ordered by index/field/view + // trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255 lastIndex := "" lastField := "" lastShard := uint64(0) @@ -52,7 +50,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { if header != nil { vprint.PanicOn("header should not be nil on err io.EOF") } - //submit any stuff we have left + // submit any stuff we have left if len(viewData) > 0 { request := &pilosa.ImportRoaringRequest{ Views: viewData, @@ -69,7 +67,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { vprint.VV("n = %v, progress, elapsed '%v'", n, time.Since(t0)) } parts := strings.Split(header.Name, "/") - //vv("parts = '%#v'", parts) + // vv("parts = '%#v'", parts) index := parts[1] field := parts[2] view := parts[4] @@ -83,15 +81,15 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { request := &pilosa.ImportRoaringRequest{ Views: viewData, } - //vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) + // vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) uri := GetImportRoaringURI(lastIndex, lastShard) vprint.PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)) viewData = make(map[string][]byte) - //vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0)) + // vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0)) } } - roaringData, err := ioutil.ReadAll(tarReader) + roaringData, err := io.ReadAll(tarReader) if err != nil { return err } @@ -102,8 +100,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { lastIndex = index lastField = field - //lastShard = shard - //vv("bottom of loop") + // lastShard = shard + // vv("bottom of loop") } } @@ -111,7 +109,6 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { // the new "good" loader, and should always be preferred now // when not trying to repro that bug. pulled from 85fa67e8 func main() { - host := "127.0.0.1:10101" h := &gohttp.Client{} c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{})) diff --git a/cmd/pilosa-bench/main.go b/cmd/pilosa-bench/main.go index c68e97e14..57c0ee301 100644 --- a/cmd/pilosa-bench/main.go +++ b/cmd/pilosa-bench/main.go @@ -6,7 +6,7 @@ import ( "expvar" "flag" "fmt" - "io/ioutil" + "io" "log" "math/rand" "net/http" @@ -71,7 +71,7 @@ func run(ctx context.Context, args []string) (err error) { // Clear time prefix on log. log.SetFlags(0) if !*verbose { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) } // Setup PRNG to have consistent values for the same set of data. diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index f4bc6459e..0e56d1baf 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sort" @@ -23,8 +22,10 @@ import ( "github.com/spf13/cobra" ) -var visited map[string]int64 -var glogger = logger.NewStandardLogger(os.Stdout) +var ( + visited map[string]int64 + glogger = logger.NewStandardLogger(os.Stdout) +) const ( Version = "1.0" @@ -33,6 +34,7 @@ const ( func main() { os.Exit(realMain()) } + func realMain() int { visited = make(map[string]int64) var dataDir, backupPath string @@ -82,7 +84,6 @@ func FetchFragments(base string) []string { var fragments []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -98,7 +99,6 @@ func FetchFragments(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } @@ -121,14 +121,13 @@ func fileExists(filename string) (bool, int64) { } func BuildSchema(dataDir string) ([]byte, error) { - //need to find all the ".meta" files and load as field options + // need to find all the ".meta" files and load as field options schemaSerializer := struct { Indexes []*local `json:"indexes,omitempty"` }{Indexes: make([]*local, 0)} var l *local ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Infof("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -136,16 +135,16 @@ func BuildSchema(dataDir string) ([]byte, error) { } pathX = pathX[len(dataDir):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { if strings.Contains(pathX, ".meta") { - //convert the file to a fieldOptions + // convert the file to a fieldOptions // ex: metaPath /trait_store/aba/.meta glogger.Infof("PATHX %v", pathX) t := strings.Split(pathX, "/") index := t[1] src := dataDir + pathX - content, err := ioutil.ReadFile(src) + content, err := os.ReadFile(src) if err != nil { return err } @@ -165,7 +164,7 @@ func BuildSchema(dataDir string) ([]byte, error) { CreatedAt: uint64(CTimeNano(stat)), Options: *io, } - //index options + // index options schemaSerializer.Indexes = append(schemaSerializer.Indexes, l) return nil } @@ -184,20 +183,20 @@ func BuildSchema(dataDir string) ([]byte, error) { } err := filepath.Walk(dataDir, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", dataDir, err) } return json.MarshalIndent(schemaSerializer, "", " ") } + func Extract(filename string) (index, field, view string, shard uint64) { - //trait_store/aba/views/standard/fragments + // trait_store/aba/views/standard/fragments parts := strings.Split(filename, "/") shard, _ = strconv.ParseUint(parts[6], 10, 64) return parts[1], parts[2], parts[4], shard } -//just a way to collect all the open dbs +// just a way to collect all the open dbs type rbfFile struct { working *rbf.DB last string @@ -222,6 +221,7 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) { } return d.working, nil } + func (d *rbfFile) Close() error { defer func() error { // clean up the temp directory @@ -235,11 +235,11 @@ func (d *rbfFile) Close() error { if d.last != "" { d.working.Close() - //if d.last exists only keep the biggest + // if d.last exists only keep the biggest exists, sz := fileExists(d.last) src := filepath.Join(d.temp, "data") if !exists { - err := os.MkdirAll(filepath.Dir(d.last), 0750) + err := os.MkdirAll(filepath.Dir(d.last), 0o750) if err != nil { return err } @@ -257,6 +257,7 @@ func (d *rbfFile) Close() error { } return nil } + func copyFile(src, dest string) error { from, err := os.Open(src) if err != nil { @@ -264,7 +265,7 @@ func copyFile(src, dest string) error { } defer from.Close() - to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644) + to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0o644) if err != nil { return err } @@ -280,7 +281,7 @@ func copyFile(src, dest string) error { func Migrate(dataDir, backupPath string, verbose bool) error { dataDir = strings.TrimSuffix(dataDir, "/") - err := os.MkdirAll(backupPath, 0750) + err := os.MkdirAll(backupPath, 0o750) if err != nil { return err } @@ -293,14 +294,14 @@ func Migrate(dataDir, backupPath string, verbose bool) error { return err } - err = ioutil.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644) + err = os.WriteFile(filepath.Join(backupPath, "schema"), schema, 0o644) if err != nil { return err } raw := FetchFragments(dataDir) sort.Slice(raw, func(i, j int) bool { - //trait_store/zip_code/views/standard/fragments/ + // trait_store/zip_code/views/standard/fragments/ ti := strings.LastIndex(raw[i], "/") + 1 shardi, err := strconv.ParseUint(raw[i][ti:], 10, 16) if err != nil { @@ -318,7 +319,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } return false }) - //raw is now sorted by shard + // raw is now sorted by shard cache := &rbfFile{ temp: filepath.Join(backupPath, "_SCRATCH"), @@ -341,7 +342,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { if verbose { glogger.Infof("processing: %v", dataDir+filename) } - content, err := ioutil.ReadFile(dataDir + filename) + content, err := os.ReadFile(dataDir + filename) if err != nil { return err } @@ -356,8 +357,14 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } key := string(txkey.Prefix(index, field, view, shard)) tx, err := db.Begin(true) + if err != nil { + return err + } tx.AddRoaring(key, bm) err = tx.Commit() + if err != nil { + return err + } } cache.Close() keys := FetchIndexKeys(dataDir) @@ -372,7 +379,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { } } - //deal with index field(row)keys + // deal with index field(row)keys keys = FetchRowkeys(dataDir) for _, filename := range keys { glogger.Infof("field %v", filename) @@ -389,7 +396,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { func writeIfBigger(dst string, srcFile string) error { if stats, err := os.Stat(dst); os.IsNotExist(err) { - err = os.MkdirAll(filepath.Dir(dst), 0750) + err = os.MkdirAll(filepath.Dir(dst), 0o750) if err != nil { return err } @@ -404,8 +411,9 @@ func writeIfBigger(dst string, srcFile string) error { return copyFile(srcFile, dst) } } - return nil //simply skip it + return nil // simply skip it } + func ignore(path string, items ...string) bool { f := filepath.Base(path) for i := range items { @@ -420,7 +428,6 @@ func FetchIndexKeys(base string) []string { var directory []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -428,7 +435,7 @@ func FetchIndexKeys(base string) []string { } pathX = pathX[len(base):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { if strings.Contains(pathX, "_keys") && !ignore(pathX, ".data", "keys") { directory = append(directory, pathX) @@ -438,7 +445,6 @@ func FetchIndexKeys(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } @@ -449,7 +455,6 @@ func FetchRowkeys(base string) []string { var directory []string ff := func(pathX string, infoX os.FileInfo, errX error) error { - // first thing to do, check error. and decide what to do about it if errX != nil { glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) @@ -457,13 +462,13 @@ func FetchRowkeys(base string) []string { } pathX = pathX[len(base):] if infoX.IsDir() { - //filepath.Walk(pathX, ff) + // filepath.Walk(pathX, ff) } else { fp := filepath.Base(pathX) if fp == "keys" { p := strings.Split(pathX, "/") if len(p) != 4 { - return nil //skip all but field/key files + return nil // skip all but field/key files } directory = append(directory, pathX) } @@ -472,7 +477,6 @@ func FetchRowkeys(base string) []string { } err := filepath.Walk(base, ff) - if err != nil { glogger.Errorf("error walking the path %q: %v\n", base, err) } diff --git a/cmd/roaring-migrate/main_test.go b/cmd/roaring-migrate/main_test.go index 8f452c178..0e8740d3a 100644 --- a/cmd/roaring-migrate/main_test.go +++ b/cmd/roaring-migrate/main_test.go @@ -1,7 +1,6 @@ package main import ( - "io/ioutil" "os" "testing" ) @@ -27,24 +26,27 @@ func TestFileExists(t *testing.T) { } func TestMainProgram(t *testing.T) { - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose", } if realMain() == 0 { t.Fatal("should fail and it succeeded") } - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose", } if realMain() == 0 { t.Fatal("should fail and it succeeded") } - dir, err := ioutil.TempDir("", "backup") + dir, err := os.MkdirTemp("", "backup") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) // clean up - os.Args = []string{"roaring-migrate", + os.Args = []string{ + "roaring-migrate", "--verbose=true", "--data-dir=testdata/data-dir/", "--backup-dir=" + dir, @@ -52,5 +54,4 @@ func TestMainProgram(t *testing.T) { if realMain() == 1 { t.Fatal("shouldn't fail") } - } diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 931ab910b..37b939aad 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -9,7 +9,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" gohttp "net/http" "net/url" "os" @@ -54,7 +53,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { return err } } - roaringData, err := ioutil.ReadAll(tr) + roaringData, err := io.ReadAll(tr) if err != nil { return err } @@ -85,12 +84,11 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { index := parts[1] fieldName := parts[2] if fieldName == "_keys" { - //skip index keys are not not real fields so will have no need for field keys + // skip index keys are not not real fields so will have no need for field keys return nil - } - byteData, err := ioutil.ReadAll(tr) + byteData, err := io.ReadAll(tr) vprint.PanicOn(err) readerFunc := func() (io.Reader, error) { return bytes.NewReader(byteData), nil @@ -106,7 +104,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { if err != nil { return err } - byteData, err := ioutil.ReadAll(tr) + byteData, err := io.ReadAll(tr) vprint.PanicOn(err) readerFunc := func() (io.Reader, error) { return bytes.NewReader(byteData), nil @@ -121,6 +119,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { r.state = parts[0] return nil } + func (r *stateMachine) Upload() error { if len(r.viewData) > 0 { request := &pilosa.ImportRoaringRequest{ @@ -242,7 +241,6 @@ func stopProfile(host, outfile string) { defer fd.Close() _, err = io.Copy(fd, resp.Body) vprint.PanicOn(err) - } var globURI *pnet.URI diff --git a/ctl/auth_token.go b/ctl/auth_token.go index a7a2de138..11ba7c9c3 100644 --- a/ctl/auth_token.go +++ b/ctl/auth_token.go @@ -168,7 +168,7 @@ func (cmd *AuthTokenCommand) Run(ctx context.Context) (err error) { } // Prompt the user to visit verification_uri and enter code. - fmt.Printf(formatPromptBox(dar.VerificationURI, dar.UserCode)) + fmt.Print(formatPromptBox(dar.VerificationURI, dar.UserCode)) // Request a token until success or error response, slowing down if requested. interval := dar.Interval diff --git a/dbshard.go b/dbshard.go index 8e7d5e7bf..76c483591 100644 --- a/dbshard.go +++ b/dbshard.go @@ -456,7 +456,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir // // when a new DBShard is made, we will update the list of shards then. Thus // the per.index2shard should always be up to date AFTER the first call here. -// func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]struct{}, err error) { // use the cache, always diff --git a/idk/api/source_test.go b/idk/api/source_test.go index 3843ccf8b..b09f85155 100644 --- a/idk/api/source_test.go +++ b/idk/api/source_test.go @@ -2,7 +2,6 @@ package api import ( "io" - "io/ioutil" "net/http" "strings" "testing" @@ -47,19 +46,19 @@ func TestIngest(t *testing.T) { unkeyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) unkeyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true)) unkeyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) - //unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) + // unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) unkeyedIdx.Field("int", pilosaclient.OptFieldTypeInt()) unkeyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2)) - //unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) + // unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) keyedIdx := schema.Index("apitestingest_keyed", pilosaclient.OptIndexTrackExistence(true), pilosaclient.OptIndexKeys(true)) keyedIdx.Field("idset") keyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) keyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true)) keyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0)) - //keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) + // keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool()) keyedIdx.Field("int", pilosaclient.OptFieldTypeInt()) keyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2)) - //keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) + // keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s")) err = client.SyncSchema(schema) if !assert.NoError(t, err) { @@ -119,13 +118,13 @@ func TestIngest(t *testing.T) { return } defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck + defer io.Copy(io.Discard, resp.Body) //nolint: errcheck if !assert.Equal(t, 200, resp.StatusCode) { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) t.Logf("request error: %s", body) return } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { return } @@ -190,13 +189,13 @@ func TestIngest(t *testing.T) { return } defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck + defer io.Copy(io.Discard, resp.Body) //nolint: errcheck if !assert.Equal(t, 200, resp.StatusCode) { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) t.Logf("request error: %s", body) return } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { return } diff --git a/idk/cmd/molecula-consumer-github/main.go b/idk/cmd/molecula-consumer-github/main.go index 1bc665e73..6a0388416 100644 --- a/idk/cmd/molecula-consumer-github/main.go +++ b/idk/cmd/molecula-consumer-github/main.go @@ -7,7 +7,6 @@ import ( "expvar" "fmt" "io" - "io/ioutil" "log" "net/http" _ "net/http/pprof" @@ -286,6 +285,7 @@ func (s *Source) issueSchema() []idk.Field { idk.RecordTimeField{NameVal: "created_at", Layout: time.RFC3339}, } } + func (s *Source) Close() error { return nil } @@ -384,9 +384,9 @@ func (m *Main) openURLReader(t time.Time) (io.ReadCloser, error) { } // If cache enabled, write to file first and then return. - if buf, err := ioutil.ReadAll(resp.Body); err != nil { + if buf, err := io.ReadAll(resp.Body); err != nil { return nil, err - } else if err := ioutil.WriteFile(cachePath+".tmp", buf, 0666); err != nil { + } else if err := os.WriteFile(cachePath+".tmp", buf, 0o666); err != nil { return nil, err } else if err := os.Rename(cachePath+".tmp", cachePath); err != nil { return nil, err diff --git a/idk/common/configMap.go b/idk/common/configMap.go index 20a1084fa..9b6b80e24 100644 --- a/idk/common/configMap.go +++ b/idk/common/configMap.go @@ -3,7 +3,7 @@ package common import ( "encoding/json" "fmt" - "io/ioutil" + "os" "strings" "sync/atomic" @@ -43,11 +43,12 @@ func LaunchKafkaEventConfirmer(producer *confluent.Producer, finished *int32, it }() return doneChan } + func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { var err error configMap := &confluent.ConfigMap{} if m.KafkaConfiguration != "" { - file, er := ioutil.ReadFile(m.KafkaConfiguration) + file, er := os.ReadFile(m.KafkaConfiguration) if er != nil { return nil, er } @@ -84,7 +85,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { } } - //SSL + // SSL if m.KafkaSslCaLocation != "" { err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation) if err != nil { @@ -121,7 +122,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { } } - //SSL + // SSL if m.KafkaSslCaLocation != "" { err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation) if err != nil { diff --git a/idk/csv/csv_test.go b/idk/csv/csv_test.go index a4c822204..b2d646125 100644 --- a/idk/csv/csv_test.go +++ b/idk/csv/csv_test.go @@ -3,7 +3,6 @@ package csv import ( "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -257,7 +256,7 @@ func tim(t *testing.T, tstr string) time.Time { } func writeTempFile(t *testing.T, data string) string { - f, err := ioutil.TempFile("", "") + f, err := os.CreateTemp("", "") if err != nil { t.Fatalf("getting temp file: %v", err) } @@ -321,12 +320,12 @@ func TestStreamFileNames(t *testing.T) { } func testDirTree(tree string) (string, error) { - tmp, err := ioutil.TempDir("", "") + tmp, err := os.MkdirTemp("", "") if err != nil { return "", err } - err = os.MkdirAll(filepath.Join(tmp, tree), 0755) + err = os.MkdirAll(filepath.Join(tmp, tree), 0o755) if err != nil { os.RemoveAll(tmp) return "", err @@ -490,7 +489,6 @@ func testTimestampRunner(t *testing.T, m *Main, testCase TimestampTestCase) { check := testCase.expect.([]interface{}) testExtractRowsQuery(t, m.Index, testCase.fieldName, check) - } // Test that out of range int values are ingested as nil when AllowIntOutOfRange is true. @@ -532,7 +530,6 @@ id__ID,negneg__Int_-10_-5,negpos__Int_-10_10,pospos__Int_5_10,negzero__Int_-10_0 } }) } - } // Test that out of range timestamp values are ingested as nil when AllowTimestampOutOfRange is true. @@ -556,7 +553,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15: checker["ts4"] = []interface{}{nil, "0001-01-01T00:00:01Z", "0001-01-01T00:00:02Z", "9999-12-31T23:59:58Z", "9999-12-31T23:59:59Z", nil} batchSizes := []int{3, 1, 4, 10} for _, bsize := range batchSizes { - t.Run(fmt.Sprintf("batchsize=%d", bsize), func(t *testing.T) { m := newMainOORFactory(t, file, false, false, true) m.BatchSize = bsize @@ -573,7 +569,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15: } }) } - } // Tests various conditions that should halt ingest @@ -618,7 +613,8 @@ func TestFailureConditions(t *testing.T) { {name: "int string overflow", csv: `id__ID,pospos__Int 0,"89273948723984729387492387492987" -`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}} +`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}, + } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { diff --git a/idk/datagen/common.go b/idk/datagen/common.go index 60d7fb6d2..ff95c061e 100644 --- a/idk/datagen/common.go +++ b/idk/datagen/common.go @@ -8,7 +8,9 @@ import ( // data by providing one or more idk.Source. It also contains // a method for getting information about the supported sources. // Info() - return information about the generator along with -// a list of supported types +// +// a list of supported types +// // Sources() - provided a string key and configuration, returns // a list of primary key fields and a list of sources. type SourceGenerator interface { diff --git a/idk/datagen/custom.go b/idk/datagen/custom.go index 3db2943cf..2425edab9 100644 --- a/idk/datagen/custom.go +++ b/idk/datagen/custom.go @@ -5,7 +5,6 @@ import ( "encoding/csv" "fmt" "io" - "io/ioutil" "log" "math/rand" "os" @@ -31,7 +30,7 @@ type Custom struct { // NewCustom returns a new instance of Custom. func NewCustom(cfg SourceGeneratorConfig) Sourcer { conf := &CustomConfig{} - if bytes, err := ioutil.ReadFile(cfg.CustomConfig); err != nil { + if bytes, err := os.ReadFile(cfg.CustomConfig); err != nil { return &Custom{err: errors.Wrap(err, "reading custom config file")} } else if err = yaml.Unmarshal(bytes, conf); err != nil { return &Custom{err: errors.Wrap(err, "unmarshaling custom config file")} @@ -45,7 +44,6 @@ func NewCustom(cfg SourceGeneratorConfig) Sourcer { IDKAndGenFields: ig, } } - } // CustomConfig represents the JSON/yaml configuration for the "custom" datagen source. @@ -374,7 +372,7 @@ func (cs *CustomSource) Record() (idk.Record, error) { last = cs.record[i] } if cs.recordsToGenerate > 0 && cs.recordCounter >= cs.recordsToGenerate { - //break when number records produced + // break when number records produced return nil, io.EOF } else { cs.recordCounter++ diff --git a/idk/datagen/custom_test.go b/idk/datagen/custom_test.go index 6370a3f17..b51ef5945 100644 --- a/idk/datagen/custom_test.go +++ b/idk/datagen/custom_test.go @@ -1,7 +1,6 @@ package datagen import ( - "io/ioutil" "math/rand" "os" "strings" @@ -144,11 +143,10 @@ func TestGetIDKFields(t *testing.T) { t.Fatalf("mismatched genFields at %d: got:\n%+v\nexp:\n%+v", i, gotGenField, expGenField) } } - } func TestNewCustomUnmarshalling(t *testing.T) { - tmp, err := ioutil.TempFile("", "") + tmp, err := os.CreateTemp("", "") if err != nil { t.Fatalf("creating temp file: %v", err) } @@ -181,7 +179,6 @@ fields: if cc.CustomConfig.Fields[0].TimeFormat != unixFormat { t.Fatalf("wrong time format: %s", cc.CustomConfig.Fields[0].TimeFormat) } - } func TestIncreasingTimestampGenerator(t *testing.T) { @@ -233,7 +230,6 @@ func TestIncreasingTimestampGenerator(t *testing.T) { } else if recInt < testcase.min || recInt > testcase.max { t.Fatalf("unexpected value for time: %v", recInt) } - }) } } diff --git a/idk/datagen/dell.go b/idk/datagen/dell.go index 556f7a849..29b164418 100644 --- a/idk/datagen/dell.go +++ b/idk/datagen/dell.go @@ -156,12 +156,12 @@ func (e *DellSource) StringWithCharset(length int, charset string) string { return string(b) } -//returns a string with random English alphabets of the specified length +// returns a string with random English alphabets of the specified length func (e *DellSource) StringDell(length int) string { return e.StringWithCharset(length, charsetDell) } -//generates a random int between range max and min (inclusive) +// generates a random int between range max and min (inclusive) func (e *DellSource) generateRandomInt(max int, min int) int { return e.rand.Intn(max-min) + min } diff --git a/idk/datagen/hughes.go b/idk/datagen/hughes.go index a8cfb91c9..f52cffc61 100644 --- a/idk/datagen/hughes.go +++ b/idk/datagen/hughes.go @@ -122,7 +122,7 @@ func (e *HughesSource) Schema() []idk.Field { return e.schema } -//returns a random float between a range +// returns a random float between a range func (e *HughesSource) floatRand(max float64, min float64) float64 { return min + e.rand.Float64()*(max-min) } @@ -138,7 +138,7 @@ func (e *HughesSource) StringWithCharset(length int, charset string) string { return string(b) } -//returns a string with random English alphabets of the specified length +// returns a string with random English alphabets of the specified length func (e *HughesSource) SIDHughes(length int) string { return e.StringWithCharset(length, charsetHughes) } diff --git a/idk/datagen/merck.go b/idk/datagen/merck.go index d16afcce6..71d92f732 100644 --- a/idk/datagen/merck.go +++ b/idk/datagen/merck.go @@ -665,12 +665,12 @@ func (e *MerckSource) StringWithCharset(length int, charset string) string { return string(b) } -//returns a string with random English alphabets of the specified length +// returns a string with random English alphabets of the specified length func (e *MerckSource) String(length int) string { return e.StringWithCharset(length, charset) } -//returns a random int between a range with a 50% chance of returning a zero. +// returns a random int between a range with a 50% chance of returning a zero. func (e *MerckSource) generate50(max int, min int) int { num := e.rand.Intn(100) numbertoReturn := e.rand.Intn(max-min) + min @@ -682,7 +682,7 @@ func (e *MerckSource) generate50(max int, min int) int { } } -//returns a random int between a range with a 10% chance of returning a zero. +// returns a random int between a range with a 10% chance of returning a zero. func (e *MerckSource) generate10(max int, min int) int { num := e.rand.Intn(100) numbertoReturn := e.rand.Intn(max-min) + min @@ -694,7 +694,7 @@ func (e *MerckSource) generate10(max int, min int) int { } } -//returns a random float between a range with a 50% chance of returning a zero. +// returns a random float between a range with a 50% chance of returning a zero. func (e *MerckSource) float50(max float64, min float64) float64 { numbertoReturn := min + e.rand.Float64()*(max-min) num := e.rand.Intn(100) @@ -706,12 +706,12 @@ func (e *MerckSource) float50(max float64, min float64) float64 { } } -//returns a random float between a range +// returns a random float between a range func (e *MerckSource) floatRand(max float64, min float64) float64 { return min + e.rand.Float64()*(max-min) } -//returns a random int between a range +// returns a random int between a range func (e *MerckSource) generateRandomInt(max int, min int) int { return e.rand.Intn(max-min) + min } diff --git a/idk/header.go b/idk/header.go index cb0e37188..86667b64d 100644 --- a/idk/header.go +++ b/idk/header.go @@ -42,12 +42,12 @@ var ( // destname are separated by "___", triple underscore) and converts it // to an idk Field like: // -// FieldTypeField { -// NameVal: sourcename, -// DestNameVal: destname, -// Thing1: Arg, -// Thing2: Arg2, -// } +// FieldTypeField { +// NameVal: sourcename, +// DestNameVal: destname, +// Thing1: Arg, +// Thing2: Arg2, +// } // // It does this using a variety of reflective magic. The unwritten // rules are that all idk Fields must be structs and have their first diff --git a/idk/idktest/idktest.go b/idk/idktest/idktest.go index 8ba55f0e6..48473820b 100644 --- a/idk/idktest/idktest.go +++ b/idk/idktest/idktest.go @@ -3,7 +3,7 @@ package idktest import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "strings" @@ -62,7 +62,7 @@ func DoExtractQuery(pql, index string) (ExtractResponse, error) { } defer resp.Body.Close() - s, err := ioutil.ReadAll(resp.Body) + s, err := io.ReadAll(resp.Body) if err != nil { return eResp, errors.Errorf("reading response: %v", err) } diff --git a/idk/interfaces.go b/idk/interfaces.go index 469d7db64..c5bbd3cf9 100644 --- a/idk/interfaces.go +++ b/idk/interfaces.go @@ -893,10 +893,10 @@ func validateTimestamp(unit Unit, ts time.Time) error { return nil } -// validateDuration checks if the duration will overflow. Users can provide a custom epoch but -// Featurebase will ultimately convert this to some duration relative to the Unix epoch. -// So if the custom epoch + the provided value in the desired units is too far from -// Unix epoch such that it causes an interger overflow, this will return an error. +// validateDuration checks if the duration will overflow. Users can provide a custom epoch but +// Featurebase will ultimately convert this to some duration relative to the Unix epoch. +// So if the custom epoch + the provided value in the desired units is too far from +// Unix epoch such that it causes an interger overflow, this will return an error. func validateDuration(dur int64, offset int64, granularity Unit) error { var minInt, maxInt int64 switch granularity { diff --git a/idk/internal/s3.go b/idk/internal/s3.go index fb2172021..df5aa2605 100644 --- a/idk/internal/s3.go +++ b/idk/internal/s3.go @@ -2,7 +2,6 @@ package internal import ( "bytes" - "io/ioutil" "net/url" "os" "strings" @@ -57,7 +56,7 @@ func ReadFileOrURL(name string, s3client s3iface.S3API) ([]byte, error) { } content = buf.Bytes() } else { - content, err = ioutil.ReadFile(name) + content, err = os.ReadFile(name) if err != nil { if os.IsNotExist(err) { return nil, FileOrURLNotFound @@ -91,7 +90,7 @@ func WriteFileOrURL(name string, contents []byte, s3client s3iface.S3API) error return errors.Wrapf(err, "putting S3 object %v", name) } } else { - err = ioutil.WriteFile(name, contents, 0644) + err = os.WriteFile(name, contents, 0o644) if err != nil { return errors.Wrapf(err, "reading file %v", name) } diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index 9256faaf1..bccf7233a 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -5,7 +5,7 @@ package kafka import ( "fmt" - "io/ioutil" + "io" "math/rand" "net/http" "os" @@ -24,12 +24,14 @@ import ( "github.com/molecula/featurebase/v3/logger" ) -var pilosaHost string -var pilosaTLSHost string -var pilosaGrpcHost string -var kafkaHost string -var registryHost string -var certPath string +var ( + pilosaHost string + pilosaTLSHost string + pilosaGrpcHost string + kafkaHost string + registryHost string + certPath string +) func init() { var ok bool @@ -181,7 +183,6 @@ func TestConfigOptions(t *testing.T) { if val, err := cfg.Get("auto.offset.reset", nil); err != nil || val.(string) != "latest" { t.Fatalf("unexpected val for auto.offset.reset val: %v, err: %v", val, err) } - } func TestCmdMainOne(t *testing.T) { @@ -652,6 +653,7 @@ func (s sortableCRI) Less(i, j int) bool { } return true } + func (s sortableCRI) Swap(i, j int) { s[i], s[j] = s[j], s[i] } @@ -673,7 +675,7 @@ func tDoHTTPPost(t *testing.T, url, contentType, body string) string { t.Fatalf("making POST request: %v", err) } - bod, err := ioutil.ReadAll(resp.Body) + bod, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading POST response bdoy: %v", err) } diff --git a/idk/kafka/csrc/csrc.go b/idk/kafka/csrc/csrc.go index 822377f23..f24eff3d1 100644 --- a/idk/kafka/csrc/csrc.go +++ b/idk/kafka/csrc/csrc.go @@ -4,7 +4,7 @@ import ( "crypto/tls" "encoding/json" "fmt" - "io/ioutil" + "io" "log" "net/http" "strings" @@ -114,7 +114,7 @@ func unmarshalRespErr(resp *http.Response, err error, into interface{}) error { return errors.Wrap(err, "making http request") } if resp.StatusCode != 200 { - bod, err := ioutil.ReadAll(resp.Body) + bod, err := io.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "reading body") } diff --git a/idk/kafka/put.go b/idk/kafka/put.go index afca94762..d5e26c0da 100644 --- a/idk/kafka/put.go +++ b/idk/kafka/put.go @@ -3,7 +3,6 @@ package kafka import ( "context" "encoding/binary" - "io/ioutil" "os" "time" @@ -60,7 +59,6 @@ func (p *PutCmd) Run() (err error) { } var auth *csrc.BasicAuth if p.SchemaRegistryUsername != "" { - auth = &csrc.BasicAuth{ KafkaSchemaApiKey: p.SchemaRegistryUsername, KafkaSchemaApiSecret: p.SchemaRegistryPassword, @@ -129,6 +127,7 @@ func (p *PutCmd) Run() (err error) { return nil } + func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, numPartitions int, replicationFactor int) error { a, err := confluent.NewAdminClientFromProducer(p) if err != nil { @@ -153,7 +152,8 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, []confluent.TopicSpecification{{ Topic: topic, NumPartitions: numPartitions, - ReplicationFactor: replicationFactor}}, + ReplicationFactor: replicationFactor, + }}, // Admin options confluent.SetAdminOperationTimeout(maxDur)) if err != nil { @@ -166,6 +166,7 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, } return nil } + func (p *PutCmd) getSchema() (string, error) { if p.Schema == "" && p.SchemaFile == "" { return "", errors.New("need a string schema or schema file") @@ -173,7 +174,7 @@ func (p *PutCmd) getSchema() (string, error) { if p.Schema != "" { return p.Schema, nil } - bytes, err := ioutil.ReadFile(p.SchemaFile) + bytes, err := os.ReadFile(p.SchemaFile) if err != nil { return "", errors.Wrap(err, "reading schema file") } diff --git a/idk/kafka/source.go b/idk/kafka/source.go index a7728e473..4ab1252bb 100644 --- a/idk/kafka/source.go +++ b/idk/kafka/source.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -1009,7 +1008,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { } defer schemaUrlResponse.Body.Close() if schemaUrlResponse.StatusCode >= 300 { - bod, err := ioutil.ReadAll(schemaUrlResponse.Body) + bod, err := io.ReadAll(schemaUrlResponse.Body) if err != nil { return nil, errors.Wrapf(err, "Failed to get schema, code: %d, no body", schemaUrlResponse.StatusCode) } @@ -1043,7 +1042,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { s.Log.Infof("Problem getting subject/version info for schema: %v", err) } else { if subVerResponse.StatusCode >= 300 { - bod, err := ioutil.ReadAll(subVerResponse.Body) + bod, err := io.ReadAll(subVerResponse.Body) s.Log.Infof("Problem getting subject/version info for schema, response: %s. Err reading body: %v", bod, err) } defer subVerResponse.Body.Close() @@ -1053,7 +1052,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { Version int `json:"version"` } - if bod, err := ioutil.ReadAll(subVerResponse.Body); err != nil { + if bod, err := io.ReadAll(subVerResponse.Body); err != nil { s.Log.Infof("decoding subj/version %s body: %v", schemaSubVerUrl, err) } else if err := json.Unmarshal(bod, &tempSchemaStruct); err != nil { s.Log.Infof("decoding schema subject & version from registry: %v", err) diff --git a/idk/kafka/source_test.go b/idk/kafka/source_test.go index 2574939fd..403169a4c 100644 --- a/idk/kafka/source_test.go +++ b/idk/kafka/source_test.go @@ -9,7 +9,6 @@ import ( "encoding/binary" "encoding/json" "fmt" - "io/ioutil" "math/big" "math/rand" "os" @@ -81,7 +80,7 @@ func TestAvroToPDKSchema(t *testing.T) { } // check that we've covered all the test schemas - files, err := ioutil.ReadDir("./testdata/schemas") + files, err := os.ReadDir("./testdata/schemas") if err != nil { t.Fatalf("reading directory: %v", err) } @@ -251,7 +250,7 @@ func decodeTestSchema(t *testing.T, filename string) avro.Schema { } func readTestSchema(t *testing.T, filename string) string { - bytes, err := ioutil.ReadFile("./testdata/schemas/" + filename) + bytes, err := os.ReadFile("./testdata/schemas/" + filename) if err != nil { t.Fatalf("reading schema file: %v", err) } @@ -299,7 +298,8 @@ var tests = []struct { }, exp: [][]interface{}{ {"a", true, int64(101), []byte{9, 196}, float64(9.4921)}, - {nil, nil, nil, nil, nil}}, + {nil, nil, nil, nil, nil}, + }, }, { schemaFile: "floatscale.json", @@ -435,27 +435,28 @@ func TestKafkaSourceSchemaChangeCommitRegression(t *testing.T) { } go func() { - topic := "xyzzy" - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, - Partition: 0, - Offset: confluent.Offset(0), + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + Partition: 0, + Offset: confluent.Offset(0), + }, + Timestamp: time.Now(), + Value: data1, }, - Timestamp: time.Now(), - Value: data1, - }, } - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, - Partition: 0, - Offset: confluent.Offset(1), + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + Partition: 0, + Offset: confluent.Offset(1), + }, + Timestamp: time.Now(), + Value: data2, }, - Timestamp: time.Now(), - Value: data2, - }, } close(src.recordChannel) }() @@ -509,14 +510,14 @@ func TestKafkaSourceTimeout(t *testing.T) { go func() { topic := "test" - src.recordChannel <- recordWithError{Record: &confluent.Message{ - TopicPartition: confluent.TopicPartition{ - Topic: &topic, + src.recordChannel <- recordWithError{ + Record: &confluent.Message{ + TopicPartition: confluent.TopicPartition{ + Topic: &topic, + }, + Value: buf, }, - Value: buf, - }, } - }() // ensure we can get a message if one is available @@ -622,7 +623,6 @@ func TestKafkaSourceIntegration(t *testing.T) { } }) } - } func mustNewProducer(t *testing.T, kafkaHost string) *confluent.Producer { @@ -768,7 +768,6 @@ func TestKafkaSourceNotAutoCommitting(t *testing.T) { } else if off := offsets[0]; int64(off.Offset) != numRecords { t.Fatalf("after commit, offset is not %d: %d", numRecords-1, off.Offset) } - } func TestRegistryURLParsing(t *testing.T) { @@ -845,10 +844,8 @@ func TestRegistryURLParsing(t *testing.T) { if codecURL != test.expectedCodecURL { t.Errorf("codec URL exp:\n%s\ngot:\n%s", test.expectedCodecURL, codecURL) } - }) } - } func postSchema(t *testing.T, schemaFile, subj, regURL string, tlsConfig *tls.Config) (schemaID int) { @@ -884,7 +881,8 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) { []confluent.TopicSpecification{{ Topic: topic, NumPartitions: 64, - ReplicationFactor: 1}}, + ReplicationFactor: 1, + }}, // Admin options confluent.SetAdminOperationTimeout(maxDur)) if err != nil { @@ -896,7 +894,6 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) { } } a.Close() - } func tPutRecordsKafka(t *testing.T, p *confluent.Producer, topic string, schemaID int, schema *liavro.Codec, key string, records ...map[string]interface{}) { diff --git a/idk/kafka_sasl/source.go b/idk/kafka_sasl/source.go index 69d681a47..e374aa4dc 100644 --- a/idk/kafka_sasl/source.go +++ b/idk/kafka_sasl/source.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "io" - "io/ioutil" + "os" "sort" "strconv" "sync" @@ -200,7 +200,7 @@ func (s *Source) Open() error { return errors.New("needs header specification file") } - headerData, err := ioutil.ReadFile(s.Header) + headerData, err := os.ReadFile(s.Header) if err != nil { return errors.Wrap(err, "reading header file") } diff --git a/idk/kafka_static/source.go b/idk/kafka_static/source.go index dbdf49378..5dac00f23 100644 --- a/idk/kafka_static/source.go +++ b/idk/kafka_static/source.go @@ -5,8 +5,8 @@ import ( "context" "encoding/json" "io" - "io/ioutil" "net/url" + "os" "strconv" "strings" "time" @@ -262,7 +262,7 @@ func (s *Source) readFileOrURL(name string) ([]byte, error) { s.Log.Printf("read %d bytes from %s\n", bytesRead, name) content = buf.Bytes() } else { - content, err = ioutil.ReadFile(name) + content, err = os.ReadFile(name) if err != nil { return nil, errors.Wrapf(err, "reading file %v", name) } diff --git a/idk/kafka_static/source_test.go b/idk/kafka_static/source_test.go index 1d273f204..4a7b395fe 100644 --- a/idk/kafka_static/source_test.go +++ b/idk/kafka_static/source_test.go @@ -6,9 +6,9 @@ package kafka_static import ( "context" "fmt" - "io/ioutil" "math/rand" "net" + "os" "reflect" "testing" "time" @@ -68,7 +68,7 @@ func TestKafkaStaticSourceLocal(t *testing.T) { src := NewSource() configureSourceTestFlags(src) { - headerData, err := ioutil.ReadFile(test.header) + headerData, err := os.ReadFile(test.header) if err != nil { t.Fatalf("reading header file: %v", err) } @@ -84,7 +84,6 @@ func TestKafkaStaticSourceLocal(t *testing.T) { for _, exp := range test.exp { rec, err := src.Record() - if err != nil { t.Fatalf("unexpected error getting record: %v", err) } @@ -171,7 +170,6 @@ func TestKafkaStaticSourceIntegration(t *testing.T) { } }) } - } func tCreateTopic(t *testing.T, topic string, addr net.Addr) { diff --git a/idk/kafkagen/cmd.go b/idk/kafkagen/cmd.go index f2a069652..cec0e0ca8 100644 --- a/idk/kafkagen/cmd.go +++ b/idk/kafkagen/cmd.go @@ -3,7 +3,6 @@ package kafkagen import ( "encoding/binary" "fmt" - "io/ioutil" "log" "os" @@ -58,7 +57,7 @@ func (m *Main) Run() (err error) { {"2", "1", 159, map[string]interface{}{"boolean": true}, map[string]interface{}{"boolean": false}, map[string]interface{}{"string": "cgr"}, map[string]interface{}{"array": []string{"a", "b"}}, nil, map[string]interface{}{"int": 7}, nil, nil, map[string]interface{}{"float": 5.4}, nil, map[string]interface{}{"org.test.survey1234": "yes"}, map[string]interface{}{"float": 8.0}, nil}, } - //kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) { + // kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) { p, err := confluent.NewProducer(m.configMap) if err != nil { @@ -102,7 +101,7 @@ func (m *Main) Run() (err error) { return errors.Wrap(err, "putting record") } } - <-doneChan //wait till all messages are acked + <-doneChan // wait till all messages are acked p.Flush(10 * 1000) return nil @@ -125,7 +124,7 @@ func (m *Main) putRecordKafka(p *confluent.Producer, schemaID int, schema *liavr } func readSchema(filename string) (string, error) { - bytes, err := ioutil.ReadFile(filename) + bytes, err := os.ReadFile(filename) if err != nil { return "", err } diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index 8c1d207a9..e0ca18559 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -5,7 +5,7 @@ import ( "context" "fmt" "hash/fnv" - "io/ioutil" + "io" "math/rand" "os" "os/exec" @@ -101,7 +101,7 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, if err != nil { return err } - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { return err } @@ -129,7 +129,7 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, } func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) { - dirEntries, err := ioutil.ReadDir(dirPath) + dirEntries, err := os.ReadDir(dirPath) if err != nil { return nil, err } @@ -145,7 +145,7 @@ func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, // filter out non-file entries filePaths := make([]string, 0, len(dirEntries)) for _, entry := range dirEntries { - if entry.Mode().IsDir() { + if entry.IsDir() { continue } filePath := filepath.Join(dirPath, entry.Name()) @@ -186,7 +186,7 @@ func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, ke // create dir to store boltdbs for this node nodeDirPath := filepath.Join(dirPath, node) - err = os.Mkdir(nodeDirPath, 0755) + err = os.Mkdir(nodeDirPath, 0o755) if err != nil { return err } @@ -385,7 +385,7 @@ func TestPauseReplica(t *testing.T) { t.Fatal(err) } dirPath = filepath.Join(dirPath, keysDirName) - err = os.Mkdir(dirPath, 0755) + err = os.Mkdir(dirPath, 0o755) if err != nil { t.Fatal(err) } diff --git a/net/uri.go b/net/uri.go index ea7e16e7e..c60e48283 100644 --- a/net/uri.go +++ b/net/uri.go @@ -29,12 +29,13 @@ var ( // 3) Port: Port of the URI. Default: 10101. // // All parts of the URI are optional. The following are equivalent: -// http://localhost:10101 -// http://localhost -// http://:10101 -// localhost:10101 -// localhost -// :10101 +// +// http://localhost:10101 +// http://localhost +// http://:10101 +// localhost:10101 +// localhost +// :10101 type URI struct { Scheme string `json:"scheme"` Host string `json:"host"` diff --git a/pql/parser_test.go b/pql/parser_test.go index ebadbbeff..2375361f6 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -217,7 +217,7 @@ func TestParser_Parse(t *testing.T) { ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } - q, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`) + _, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`) if err == nil { t.Fatal("no error parsing invalid date") } else if !strings.Contains(err.Error(), "not a valid timestamp") { @@ -238,7 +238,6 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) - } func TestUnquote(t *testing.T) { @@ -286,7 +285,6 @@ func TestUnquote(t *testing.T) { } }) } - } func testErr(t *testing.T, exp string, actual error) (done bool) { diff --git a/rbf/cursorx.go b/rbf/cursorx.go index fca85647a..80e7afbc1 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -12,7 +12,7 @@ import ( "github.com/pkg/errors" ) -//probably should just implement the container interface +// probably should just implement the container interface // but for now i'll do it func (c *Cursor) Rows() ([]uint64, error) { shardVsContainerExponent := uint(4) //needs constant exported from roaring package diff --git a/rbf/db.go b/rbf/db.go index 1ecbb30fd..6453cb43d 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -19,9 +19,7 @@ import ( "github.com/molecula/featurebase/v3/syswrap" ) -var ( - ErrClosed = errors.New("rbf: database closed") -) +var ErrClosed = errors.New("rbf: database closed") // shared cursor pool across all DB instances. // Cursors are returned on Cursor.Close(). @@ -118,14 +116,14 @@ func (db *DB) Open() (err error) { db.mu.Lock() defer db.mu.Unlock() - if err := os.MkdirAll(db.Path, 0755); err != nil { + if err := os.MkdirAll(db.Path, 0o755); err != nil { return err - } else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil { + } else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil { return fmt.Errorf("open file: %w", err) } // Open read-only database mmap. - if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0600); err != nil { + if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0o600); err != nil { return fmt.Errorf("open mmap file: %w", err) } else if db.data, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -163,12 +161,12 @@ func (db *DB) Open() (err error) { func (db *DB) openWAL() (err error) { // Open WAL file writer. - if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil { + if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil { return fmt.Errorf("open wal file: %w", err) } // Open read-only mmap. - if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0600); err != nil { + if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0o600); err != nil { return fmt.Errorf("open wal mmap file: %w", err) } else if db.wal, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxWALSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -332,7 +330,7 @@ func (db *DB) checkpoint() (err error) { if IsBitmapHeader(page) { pgno = readPageNo(page) if i+1 < db.walPageN { - if page, err = db.readWALPageAt(i + 1); err != nil { + if _, err = db.readWALPageAt(i + 1); err != nil { return err } } else { @@ -501,7 +499,6 @@ func (db *DB) Close() (err error) { // We will internally create and rollback a read-only // transaction to answer this query. func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { - // Read a list of all bitmaps in Tx. tx, err := db.Begin(false) if err != nil { @@ -578,7 +575,6 @@ func (db *DB) init() error { // initMetaPage initializes the meta page. func (db *DB) initMetaPage() error { - page := allocPage() writeMetaMagic(page) writeMetaPageN(page, 3) @@ -590,7 +586,6 @@ func (db *DB) initMetaPage() error { // initRootRecordPage initializes the initial root record page. func (db *DB) initRootRecordPage() error { - page := allocPage() writePageNo(page, 1) writeFlags(page, PageTypeRootRecord) @@ -600,7 +595,6 @@ func (db *DB) initRootRecordPage() error { // initFreelistPage initializes the initial freelist btree page. func (db *DB) initFreelistPage() error { - page := allocPage() writePageNo(page, 2) writeFlags(page, PageTypeLeaf) @@ -786,7 +780,6 @@ func (db *DB) removeTx(tx *Tx) error { // Check performs an integrity check. func (db *DB) Check() error { - tx, err := db.Begin(false) if err != nil { return err diff --git a/rbf/page_map.go b/rbf/page_map.go index 3ba5f27e7..75a397cf9 100644 --- a/rbf/page_map.go +++ b/rbf/page_map.go @@ -1,7 +1,7 @@ /* start with https://github.com/benbjohnson/immutable and specialize Map -Copyright 2019 Ben Johnson +# Copyright 2019 Ben Johnson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/roaring/btree.go b/roaring/btree.go index bbc429704..6ff64c42e 100644 --- a/roaring/btree.go +++ b/roaring/btree.go @@ -635,9 +635,9 @@ func (t *tree) Set(k uint64, v *Container) { // (whatever, false) if it decides not to create or not to update the value of // the KV pair. // -// tree.Set(k, v) call conceptually equals calling +// tree.Set(k, v) call conceptually equals calling // -// tree.Put(k, func(uint64, bool){ return v, true }) +// tree.Put(k, func(uint64, bool){ return v, true }) // // modulo the differing return values. func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Container, write bool)) (oldV *Container, written bool) { diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index c1d8c54eb..f9a52a19d 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -7,7 +7,7 @@ import "sync" const containerWidth = 1 << 16 -////////////////// array +// //////////////// array func arrayEmpty() []uint16 { return make([]uint16, 0) } @@ -76,7 +76,7 @@ func arrayEvenBitsSet() []uint16 { return array } -////////////////// bitmap +// //////////////// bitmap func bitmapEmpty() []uint64 { return make([]uint64, bitmapN) } @@ -161,7 +161,7 @@ func bitmapEvenBitsSet() []uint64 { return bitmap } -////////////////// run +// //////////////// run func runEmpty() []Interval16 { return make([]Interval16, 0) } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 1e25cc958..eedc41025 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1155,7 +1155,7 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) { } -//empty array test +// empty array test func TestBitmap_Xor_Empty(t *testing.T) { bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) empty := roaring.NewFileBitmap() diff --git a/short_txkey/txkey.go b/short_txkey/txkey.go index d5d7345ec..66c4c072d 100644 --- a/short_txkey/txkey.go +++ b/short_txkey/txkey.go @@ -32,7 +32,7 @@ func FieldViewFromFullKey(fullKey []byte) FieldView { // The roaringContainerKey argument to Key() is a container key into a roaring Container. // The return value from Key() is constructed as follows: // -// ~field;view', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} +// +// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} // // The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable // base-64 encoding. // // The shortest possible key is 14 bytes. It would be laid out like this: -// ~f;v<12345678# -// 12345678901234 +// +// ~f;v<12345678# +// 12345678901234 // // keys starting with '~' are regular value keys. // keys starting with '>' are symlink keys. // // NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -// func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) { prefix := Prefix(index, field, view, shard) @@ -100,9 +101,10 @@ func MustValidateKey(bkey []byte) { // KeyExtractContainerKey extracts the containerKey from bkey. // key example: field;view', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} +// +// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'} // // The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable // base-64 encoding. // // The shortest possible key is 25 bytes. It would be laid out like this: -// ~i%f;v:12345678<12345678# -// 1234567890123456789012345 +// +// ~i%f;v:12345678<12345678# +// 1234567890123456789012345 // // keys starting with '~' are regular value keys. // keys starting with '>' are symlink keys. // // NB must be kept in sync with Prefix() and KeyExtractContainerKey(). -// func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) { prefix := Prefix(index, field, view, shard) @@ -67,9 +68,12 @@ func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r } // ShardFromKey key example: index/field;view:shard