From f9ddb5d5c144a5fc7e77f80673c2d3808e0f2258 Mon Sep 17 00:00:00 2001 From: CLoZengineer <108562102+CLoZengineer@users.noreply.github.com> Date: Thu, 29 Sep 2022 12:34:29 -0400 Subject: [PATCH] fix: updating code to meet linting requirements (#2171) * removes unused filesize function * removes ioutil usage * updates ioutil.ReadAll to io.ReadAll * updates ioutil.TempFile to os.CreateTemp * updates ioutil.TempDir to os.MkdirTemp * updates ioutil.ReadAll to os.ReadAll * update ioutil.WriteFile to os.WriteFile * updates ioutil.Discard to io.Discard * updates ioutil.ReadDir to os.ReadDir where applicable * removes unused code in idk * creates type to use for context value keys * replaces assert.Nil with assert.NoError for error checks --- api.go | 27 ++++--- authn/authenticate.go | 13 ++-- authn/authenticate_internal_test.go | 7 +- authz/authorization.go | 3 +- client/batch_test.go | 15 ++-- client/client.go | 4 +- client/client_it_test.go | 6 +- client/ingest_api_batch_test.go | 4 +- client/logimport_test.go | 4 +- cluster_internal_test.go | 40 +--------- cmd/badloader/badloader.go | 3 +- cmd/pilosa-bench/main.go | 4 +- cmd/roaring-migrate/main.go | 9 +-- cmd/roaring-migrate/main_test.go | 3 +- cmd/root_test.go | 5 +- cmd/slurp/slurp.go | 7 +- ctl/auth_token.go | 2 +- ctl/backup.go | 3 +- ctl/import_test.go | 5 +- ctl/server.go | 4 +- etcd/embed.go | 2 +- etcd/external.go | 4 +- etcd/leasedkv_test.go | 18 ++++- executor_internal_test.go | 2 + executor_test.go | 12 +-- field_test.go | 4 +- fragment.go | 19 +++-- fragment_internal_test.go | 3 +- hash/blake3_test.go | 7 +- http_handler.go | 19 ++--- http_handler_internal_test.go | 10 +-- http_handler_test.go | 13 ++-- http_translator.go | 3 +- idk/api/source_test.go | 13 ++-- idk/cmd/molecula-consumer-github/main.go | 7 +- idk/common/configMap.go | 4 +- idk/csv/csv_test.go | 5 +- idk/datagen/custom.go | 3 +- idk/datagen/custom_test.go | 3 +- idk/idallocator.go | 5 +- idk/idktest/idktest.go | 4 +- idk/ingest.go | 4 +- idk/ingest_test.go | 5 +- idk/internal/s3.go | 5 +- idk/kafka/cmd_test.go | 6 +- idk/kafka/csrc/csrc.go | 4 +- idk/kafka/put.go | 5 +- idk/kafka/source.go | 14 +--- idk/kafka/source_test.go | 11 ++- idk/kafka_sasl/source.go | 9 +-- idk/kafka_static/source.go | 4 +- idk/kafka_static/source_test.go | 4 +- idk/kafkagen/cmd.go | 5 +- idk/kinesis/logger_test.go | 21 ++---- idk/kinesis/reader_test.go | 18 ++--- idk/tls.go | 3 +- ingest/codec.go | 3 +- ingest_test.go | 3 +- internal/clustertests/pause_node_test.go | 2 + internal_client.go | 41 +++++------ logger/filewriter_test.go | 20 +++-- logger/logger.go | 3 +- pb/pb.go | 3 +- pg/protocol.go | 3 +- pql/parser.go | 3 +- pql/parser_test.go | 2 +- rbf.go | 6 -- rbf/cursor_test.go | 6 +- rbf/db.go | 2 +- rbf/db_test.go | 4 +- rbf/tx_test.go | 41 ++++++----- roaring/fuzzer.go | 3 +- roaring/roaring.go | 57 +++++++++------ roaring/roaring_internal_test.go | 44 +++++------ server/grpc.go | 4 +- server/grpc_test.go | 22 +++--- server/handler_test.go | 93 +----------------------- server/pg_internal_test.go | 5 +- server/server.go | 3 +- server/server_test.go | 6 +- server/tlsconfig.go | 3 +- test/pilosa.go | 12 +-- testhook/hook.go | 9 +-- translate.go | 18 ++--- txfactory.go | 25 ++----- 85 files changed, 363 insertions(+), 529 deletions(-) diff --git a/api.go b/api.go index b0101655a..b29bae334 100644 --- a/api.go +++ b/api.go @@ -12,7 +12,6 @@ import ( "encoding/csv" "fmt" "io" - "io/ioutil" "math" "net/url" "os" @@ -27,7 +26,6 @@ import ( "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/rbf" - //"github.com/featurebasedb/featurebase/v3/pg" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/featurebasedb/featurebase/v3/stats" @@ -803,7 +801,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) (_ []byte return nil, errors.Wrap(err, "validating api method") } - reqBytes, err := ioutil.ReadAll(body) + reqBytes, err := io.ReadAll(body) if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read body error")) } @@ -1016,7 +1014,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { } // Read entire body. - body, err := ioutil.ReadAll(reqBody) + body, err := io.ReadAll(reqBody) if err != nil { return errors.Wrap(err, "reading body") } @@ -2398,7 +2396,7 @@ func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []u // ErrTranslatingKeyNotFound error will be swallowed here, so the empty response will be returned. func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) { var req TranslateKeysRequest - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error")) } else if err := api.Serializer.Unmarshal(buf, &req); err != nil { @@ -2435,7 +2433,7 @@ func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err e // TranslateIDs handles a TranslateIDRequest. func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) { var req TranslateIDsRequest - if buf, err := ioutil.ReadAll(r); err != nil { + if buf, err := io.ReadAll(r); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error")) } else if err := api.Serializer.Unmarshal(buf, &req); err != nil { return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error")) @@ -2884,14 +2882,15 @@ func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string, // MutexCheck checks a named field for mutex violations, returning a // map of record IDs to values for records that have multiple values in the // field. The return will be one of: -// details true: -// map[uint64][]uint64 // unkeyed index, unkeyed field -// map[uint64][]string // unkeyed index, keyed field -// map[string][]uint64 // keyed index, unkeyed field -// map[string][]string // keyed index, keyed field -// details false: -// []uint64 // unkeyed index -// []string // keyed index +// +// details true: +// map[uint64][]uint64 // unkeyed index, unkeyed field +// map[uint64][]string // unkeyed index, keyed field +// map[string][]uint64 // keyed index, unkeyed field +// map[string][]string // keyed index, keyed field +// details false: +// []uint64 // unkeyed index +// []string // keyed index func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) { if err = api.validate(apiMutexCheck); err != nil { return nil, errors.Wrap(err, "validating api method") diff --git a/authn/authenticate.go b/authn/authenticate.go index 6bf2415d3..1c6a05a43 100644 --- a/authn/authenticate.go +++ b/authn/authenticate.go @@ -25,6 +25,9 @@ import ( "golang.org/x/oauth2" ) +// AuthContextKey is a unique type to prevent collisions when using context.WithValue() +type AuthContextKey string + const ( // AccessCookieName is the name of the cookie that holds the access token. AccessCookieName = "molecula-chip" @@ -36,10 +39,10 @@ const ( RefreshHeaderName = "X-Molecula-Refresh-Token" // ContextValueAccessToken is the key used to set AccessTokens in a ctx. - ContextValueAccessToken = "Access" + ContextValueAccessToken = AuthContextKey("Access") // ContextValueRefreshToken is the key used to set RefreshTokens in a ctx. - ContextValueRefreshToken = "Refresh" + ContextValueRefreshToken = AuthContextKey("Refresh") ) // cachedGroups is used to hold groups and when they were last cached @@ -171,7 +174,7 @@ func (a *Auth) refreshToken(access, refresh string) (string, string, error) { // it is caller's responsibility to inform the user that the access token has been refreshed func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) { // clean up the cache every 30 minutes or so - if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute { + if time.Since(a.lastCacheClean) >= 30*time.Minute { a.cleanCache() } @@ -237,7 +240,7 @@ func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) { func (a *Auth) cleanCache() { for access, tkn := range a.groupsCache { // if it's been more than 24 hours since the groups were cached - if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour { + if time.Since(tkn.cacheTime) >= 24*time.Hour { // remove it from our cache delete(a.groupsCache, access) } @@ -300,7 +303,7 @@ func (a *Auth) getGroups(token string) ([]Group, error) { var groups Groups gc, ok := a.groupsCache[token] - if ok && (time.Now().Sub(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 { + if ok && (time.Since(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 { return gc.groups, nil } diff --git a/authn/authenticate_internal_test.go b/authn/authenticate_internal_test.go index 5d10bcdcd..e88aa9050 100644 --- a/authn/authenticate_internal_test.go +++ b/authn/authenticate_internal_test.go @@ -16,8 +16,8 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/golang-jwt/jwt" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -67,7 +67,7 @@ func TestSetGRPCMetadata(t *testing.T) { "otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}}, } { t.Run(name, func(t *testing.T) { - ogCookies, _ := md["cookie"] + ogCookies := md["cookie"] ctx := grpc.NewContextWithServerTransportStream( metadata.NewIncomingContext(context.TODO(), md, @@ -280,8 +280,7 @@ func TestAuthenticate(t *testing.T) { a.groupsCache[token] = cachedGroups{time.Now(), test.groups} } if test.refresh { - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/authz/authorization.go b/authz/authorization.go index 8558cc8ca..86db15d97 100644 --- a/authz/authorization.go +++ b/authz/authorization.go @@ -6,7 +6,6 @@ package authz import ( "fmt" "io" - "io/ioutil" "github.com/featurebasedb/featurebase/v3/authn" @@ -43,7 +42,7 @@ func (p Permission) Satisfies(b Permission) bool { } func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) { - permsData, err := ioutil.ReadAll(permsFile) + permsData, err := io.ReadAll(permsFile) if err != nil { return fmt.Errorf("reading permissions failed with error: %s", err) diff --git a/client/batch_test.go b/client/batch_test.go index 2aadb4e4e..b78f7a616 100644 --- a/client/batch_test.go +++ b/client/batch_test.go @@ -14,6 +14,7 @@ import ( featurebase "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/test" + "github.com/stretchr/testify/assert" "github.com/pkg/errors" ) @@ -1662,8 +1663,7 @@ func mutexClearRegression(t *testing.T, c *test.Cluster, client *Client) { t.Fatalf("getting batch: %v", err) } - col := uint64(0) - row := uint64(1) + var col, row uint64 for i := uint64(0); i <= 21; i++ { col = (i%2+1)*featurebase.ShardWidth + i%5 row = i % 3 @@ -1720,8 +1720,7 @@ func mutexNilClearID(t *testing.T, c *test.Cluster, client *Client) { t.Fatalf("getting batch: %v", err) } - col := uint64(0) - row := uint64(1) + var col, row uint64 // populate mutex with some data for i := uint64(0); i < 11; i++ { col = (i%2+1)*featurebase.ShardWidth + i%5 @@ -1746,14 +1745,17 @@ func mutexNilClearID(t *testing.T, c *test.Cluster, client *Client) { } items := resp.Result().Row().Columns // delete item 0 - b.Add( + err = b.Add( Row{ ID: items[0], Values: []interface{}{nil}, Clears: map[int]interface{}{0: nil}, }, ) - b.Import() + assert.NoError(t, err) + + err = b.Import() + assert.NoError(t, err) items = items[1:] // confirm record removed resp, err = client.Query(idx.RawQuery("Row(mut=0)")) @@ -1820,6 +1822,7 @@ func mutexNilClearKey(t *testing.T, c *test.Cluster, client *Client) { t.Fatalf("importing: %v", err) } resp, err := client.Query(idx.RawQuery(`Row(mut="a")`)) + assert.NoError(t, err) errorIfNotEqual(t, resp.Result().Row().Keys, []string{"0", "2"}) r.ID = "2" diff --git a/client/client.go b/client/client.go index 40fb623c6..9a92c2f73 100644 --- a/client/client.go +++ b/client/client.go @@ -20,7 +20,6 @@ import ( "sync" "time" - "github.com/golang/protobuf/proto" //nolint:staticcheck pilosa "github.com/featurebasedb/featurebase/v3" fbproto "github.com/featurebasedb/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import "github.com/featurebasedb/featurebase/v3/logger" @@ -29,6 +28,7 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/featurebasedb/featurebase/v3/stats" + "github.com/golang/protobuf/proto" //nolint:staticcheck "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -40,8 +40,6 @@ const PQLVersion = "1.0" // DefaultShardWidth is used if an index doesn't have it defined. const DefaultShardWidth = pilosa.ShardWidth -const maxHosts = 10 - // Client is the HTTP client for Pilosa server. type Client struct { cluster *Cluster diff --git a/client/client_it_test.go b/client/client_it_test.go index 9eaf1afbb..3fff0b003 100644 --- a/client/client_it_test.go +++ b/client/client_it_test.go @@ -5,7 +5,7 @@ package client import ( "bytes" "fmt" - "io/ioutil" + "io" "testing" "time" @@ -671,7 +671,7 @@ func TestClientAgainstCluster(t *testing.T) { r, err := cli.ExportField(testFieldExport) require.NoErrorf(t, err, "ExportField") - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) require.NoError(t, err) target := "1,1\n1,10\n2,1048577\n" @@ -696,7 +696,7 @@ func TestClientAgainstCluster(t *testing.T) { r, err := cli.ExportField(testFieldExport) require.NoErrorf(t, err, "ExportField") - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) require.NoError(t, err) target := "1,one\n1,ten\n2,big-number\n" diff --git a/client/ingest_api_batch_test.go b/client/ingest_api_batch_test.go index 7c0770603..b0d4a8b2d 100644 --- a/client/ingest_api_batch_test.go +++ b/client/ingest_api_batch_test.go @@ -7,6 +7,7 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/test" + "github.com/stretchr/testify/assert" ) func TestIngestAPIBatchAdd(t *testing.T) { @@ -148,7 +149,7 @@ func TestIngestAPIBatch(t *testing.T) { } defer cli.Close() - cli.IngestSchema(map[string]interface{}{ + _, err = cli.IngestSchema(map[string]interface{}{ "index-name": "test-1", "index-action": "create", "primary-key-type": "uint", @@ -197,6 +198,7 @@ func TestIngestAPIBatch(t *testing.T) { }, }, }) + assert.NoError(t, err) schema, err := cli.Schema() if err != nil { diff --git a/client/logimport_test.go b/client/logimport_test.go index 6db0f41c4..bea2b1e72 100644 --- a/client/logimport_test.go +++ b/client/logimport_test.go @@ -5,7 +5,6 @@ package client import ( "bytes" "fmt" - "io/ioutil" "os" "reflect" "testing" @@ -93,7 +92,8 @@ func TestEncodeDecode(t *testing.T) { }) } - buf, err := ioutil.TempFile("", "") + + buf, err := os.CreateTemp("", "") if err != nil { t.Fatalf("getting temp file: %v", err) } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index bd52fc1d9..528c9b7be 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -14,47 +14,9 @@ import ( "github.com/featurebasedb/featurebase/v3/disco" pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/testhook" - . "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck + _ "github.com/featurebasedb/featurebase/v3/vprint" ) -// newHolderWithTempPath returns a new instance of Holder. -func newHolderWithTempPath(tb testing.TB, backend string) *Holder { - path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-") - if err != nil { - panic(err) - } - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = backend - h := NewHolder(path, cfg) - PanicOn(h.Open()) - testhook.Cleanup(tb, func() { - h.Close() - }) - return h -} - -// newIndexWithTempPath returns a new instance of Index. -func newIndexWithTempPath(tb testing.TB, name string) *Index { - path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-") - if err != nil { - panic(err) - } - cfg := DefaultHolderConfig() - cfg.StorageConfig.FsyncEnabled = false - cfg.RBFConfig.FsyncEnabled = false - h := NewHolder(path, cfg) - PanicOn(h.Open()) - index, err := h.CreateIndex(name, IndexOptions{}) - testhook.Cleanup(tb, func() { - h.Close() - }) - if err != nil { - panic(err) - } - return index -} - // Ensure the cluster can fairly distribute partitions across the nodes. func TestCluster_Owners(t *testing.T) { c := cluster{ diff --git a/cmd/badloader/badloader.go b/cmd/badloader/badloader.go index 35fbadcc5..93477bb27 100644 --- a/cmd/badloader/badloader.go +++ b/cmd/badloader/badloader.go @@ -10,7 +10,6 @@ import ( "fmt" "io" - "io/ioutil" gohttp "net/http" pilosa "github.com/featurebasedb/featurebase/v3" @@ -92,7 +91,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error { } } - roaringData, err := ioutil.ReadAll(tarReader) + roaringData, err := io.ReadAll(tarReader) if err != nil { return err } diff --git a/cmd/pilosa-bench/main.go b/cmd/pilosa-bench/main.go index 6529c8fea..a89c33937 100644 --- a/cmd/pilosa-bench/main.go +++ b/cmd/pilosa-bench/main.go @@ -7,7 +7,7 @@ import ( "expvar" "flag" "fmt" - "io/ioutil" + "io" "log" "math/rand" "net/http" @@ -72,7 +72,7 @@ func run(ctx context.Context, args []string) (err error) { // Clear time prefix on log. log.SetFlags(0) if !*verbose { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) } // Setup PRNG to have consistent values for the same set of data. diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index ed6eb16fd..81e6d4278 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "os" "path/filepath" "sort" @@ -146,7 +145,7 @@ func BuildSchema(dataDir string) ([]byte, error) { t := strings.Split(pathX, "/") index := t[1] src := dataDir + pathX - content, err := ioutil.ReadFile(src) + content, err := os.ReadFile(src) if err != nil { return err } @@ -198,7 +197,7 @@ func Extract(filename string) (index, field, view string, shard uint64) { return parts[1], parts[2], parts[4], shard } -//just a way to collect all the open dbs +// just a way to collect all the open dbs type rbfFile struct { working *rbf.DB last string @@ -294,7 +293,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { return err } - err = ioutil.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644) + err = os.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644) if err != nil { return err } @@ -342,7 +341,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error { if verbose { glogger.Infof("processing: %v", dataDir+filename) } - content, err := ioutil.ReadFile(dataDir + filename) + content, err := os.ReadFile(dataDir + filename) if err != nil { return err } diff --git a/cmd/roaring-migrate/main_test.go b/cmd/roaring-migrate/main_test.go index 8f452c178..f83a9bf0c 100644 --- a/cmd/roaring-migrate/main_test.go +++ b/cmd/roaring-migrate/main_test.go @@ -1,7 +1,6 @@ package main import ( - "io/ioutil" "os" "testing" ) @@ -39,7 +38,7 @@ func TestMainProgram(t *testing.T) { if realMain() == 0 { t.Fatal("should fail and it succeeded") } - dir, err := ioutil.TempDir("", "backup") + dir, err := os.MkdirTemp("", "backup") if err != nil { t.Fatal(err) } diff --git a/cmd/root_test.go b/cmd/root_test.go index cb675e74e..9f4e96e5f 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -5,7 +5,6 @@ package cmd_test import ( "fmt" "io" - "io/ioutil" "os" "reflect" "strings" @@ -34,7 +33,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o done := make(chan struct{}) var readErr error go func() { - output, readErr = ioutil.ReadAll(out) + output, readErr = io.ReadAll(out) close(done) }() err = cmd.Execute() @@ -143,7 +142,7 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command { os.Setenv("PILOSA_POSTGRES_BIND", "") // make command and set args - rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard) + rc := cmd.NewRootCommand(strings.NewReader(""), io.Discard, io.Discard) rc.SetArgs(ct.args) err = cfgFile.Close() diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 6a338645c..340af3c97 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" gohttp "net/http" "net/url" "os" @@ -55,7 +54,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { return err } } - roaringData, err := ioutil.ReadAll(tr) + roaringData, err := io.ReadAll(tr) if err != nil { return err } @@ -91,7 +90,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { } - byteData, err := ioutil.ReadAll(tr) + byteData, err := io.ReadAll(tr) vprint.PanicOn(err) readerFunc := func() (io.Reader, error) { return bytes.NewReader(byteData), nil @@ -107,7 +106,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { if err != nil { return err } - byteData, err := ioutil.ReadAll(tr) + byteData, err := io.ReadAll(tr) vprint.PanicOn(err) readerFunc := func() (io.Reader, error) { return bytes.NewReader(byteData), nil diff --git a/ctl/auth_token.go b/ctl/auth_token.go index fb644d442..cd18b74fc 100644 --- a/ctl/auth_token.go +++ b/ctl/auth_token.go @@ -169,7 +169,7 @@ func (cmd *AuthTokenCommand) Run(ctx context.Context) (err error) { } // Prompt the user to visit verification_uri and enter code. - fmt.Printf(formatPromptBox(dar.VerificationURI, dar.UserCode)) + fmt.Print(formatPromptBox(dar.VerificationURI, dar.UserCode)) // Request a token until success or error response, slowing down if requested. interval := dar.Interval diff --git a/ctl/backup.go b/ctl/backup.go index 22876892b..c796ccf52 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "os" "path/filepath" "time" @@ -194,7 +193,7 @@ func (cmd *BackupCommand) backupSchema(ctx context.Context, schema *pilosa.Schem return fmt.Errorf("marshaling schema: %w", err) } - if err := ioutil.WriteFile(filepath.Join(cmd.OutputDir, "schema"), buf, 0600); err != nil { + if err := os.WriteFile(filepath.Join(cmd.OutputDir, "schema"), buf, 0600); err != nil { return fmt.Errorf("writing schema: %w", err) } diff --git a/ctl/import_test.go b/ctl/import_test.go index dae069293..c4e116ba8 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -18,13 +17,13 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/server" "github.com/featurebasedb/featurebase/v3/test" "github.com/featurebasedb/featurebase/v3/testhook" + "github.com/golang-jwt/jwt" ) func TestImportCommand_Validation(t *testing.T) { @@ -305,7 +304,7 @@ func TestImportCommand_KeyReplication(t *testing.T) { // Read body and unmarshal response. exp := `{"results":[1000]}` + "\n" - if body, err := ioutil.ReadAll(resp.Body); err != nil { + if body, err := io.ReadAll(resp.Body); err != nil { return fmt.Errorf("reading: %s", err) } else if !reflect.DeepEqual(body, []byte(exp)) { return fmt.Errorf("expected: %s, but got: %s", exp, body) diff --git a/ctl/server.go b/ctl/server.go index 39c4922cf..3204375b0 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -57,9 +57,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.Int64Var(&srv.Config.Etcd.HeartbeatTTL, "etcd.heartbeat-ttl", srv.Config.Etcd.HeartbeatTTL, "Timeout used to determine cluster status") flags.StringVar(&srv.Config.Etcd.Cluster, "etcd.static-cluster", srv.Config.Etcd.Cluster, "EXPERIMENTAL static featurebase cluster name1=apurl1,name2=apurl2") - flags.MarkHidden("etcd.static-cluster") + _ = flags.MarkHidden("etcd.static-cluster") flags.StringVar(&srv.Config.Etcd.EtcdHosts, "etcd.etcd-hosts", srv.Config.Etcd.EtcdHosts, "EXPERIMENTAL etcd server host:port comma separated list") - flags.MarkHidden("etcd.etcd-hosts") // TODO (twg) expose when ready for public consumption + _ = flags.MarkHidden("etcd.etcd-hosts") // TODO (twg) expose when ready for public consumption // External postgres database for ExternalLookup flags.StringVar(&srv.Config.LookupDBDSN, "lookup-db-dsn", "", "external (postgres) database DSN to use for ExternalLookup calls") diff --git a/etcd/embed.go b/etcd/embed.go index e15cd8078..e681f8090 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -371,7 +371,7 @@ func (e *Etcd) parseOptions() (*embed.Config, error) { if e.options.InitCluster != "" { // Checks if FB is running the single-node free version or the multi-node // enterprise version. Sentry.io is enabled on single-node. - if AllowCluster() == false { + if !AllowCluster() { // %% begin sonarcloud ignore %% monitor.InitErrorMonitor(e.version) diff --git a/etcd/external.go b/etcd/external.go index f31d7bfbc..b362533fb 100644 --- a/etcd/external.go +++ b/etcd/external.go @@ -49,9 +49,7 @@ func NewExternalEtcd(p *Etcd, o Options) (*ExternalEtcd, error) { // populate parent.sortedNodes hosts := strings.Split(o.EtcdHosts, ",") this.etcdHosts = make([]string, len(hosts)) - for i := range hosts { - this.etcdHosts[i] = hosts[i] - } + copy(this.etcdHosts, hosts) return this, nil // %% end sonarcloud ignore %% diff --git a/etcd/leasedkv_test.go b/etcd/leasedkv_test.go index cfdc481ad..aa10da142 100644 --- a/etcd/leasedkv_test.go +++ b/etcd/leasedkv_test.go @@ -12,6 +12,7 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" @@ -41,11 +42,20 @@ func TestClusterKv(t *testing.T) { t.Fatalf("trying to cause election: %v", err) } ctx := context.TODO() - c.nodes[0].SetState(ctx, disco.NodeStateStarting) - c.nodes[1].SetState(ctx, disco.NodeStateStarting) + err = c.nodes[0].SetState(ctx, disco.NodeStateStarting) + assert.NoError(t, err) + + err = c.nodes[1].SetState(ctx, disco.NodeStateStarting) + assert.NoError(t, err) + c.MustAwaitClusterState(disco.ClusterStateStarting) - c.nodes[0].SetState(ctx, disco.NodeStateStarted) - c.nodes[1].SetState(ctx, disco.NodeStateStarted) + + err = c.nodes[0].SetState(ctx, disco.NodeStateStarted) + assert.NoError(t, err) + + err = c.nodes[1].SetState(ctx, disco.NodeStateStarted) + assert.NoError(t, err) + // Two of three nodes are up, one is down, we have 2 replicas, so // we should be able to handle reads but not writes, so we're in // a Degraded state. diff --git a/executor_internal_test.go b/executor_internal_test.go index fd7a2a6d8..3cfd9b7c7 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -13,6 +13,7 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/testhook" + "github.com/stretchr/testify/assert" ) func TestExecutor_TranslateRowsOnBool(t *testing.T) { @@ -592,6 +593,7 @@ func TestExecutor_DeleteRows(t *testing.T) { } changed, err = DeleteRows(ctx, row, idx, shard) + assert.NoError(t, err) if changed { t.Fatalf("expected delete to not clear bit but it did") } diff --git a/executor_test.go b/executor_test.go index 7275204db..fe830a488 100644 --- a/executor_test.go +++ b/executor_test.go @@ -12,10 +12,10 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math" "math/rand" _ "net/http/pprof" + "os" "reflect" "sort" "strconv" @@ -24,8 +24,6 @@ import ( "time" "github.com/davecgh/go-spew/spew" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/boltdb" "github.com/featurebasedb/featurebase/v3/ctl" @@ -36,6 +34,8 @@ import ( "github.com/featurebasedb/featurebase/v3/test" "github.com/featurebasedb/featurebase/v3/testhook" . "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" ) @@ -6894,7 +6894,7 @@ func TestExecutor_Execute_NoIndex(t *testing.T) { } func TestExecutor_Execute_CountDistinct(t *testing.T) { - data, err := ioutil.ReadFile("testdata/schema.json") + data, err := os.ReadFile("testdata/schema.json") if err != nil { t.Fatal(err) } @@ -7127,7 +7127,7 @@ func TestExecutor_BareDistinct(t *testing.T) { } func TestExecutor_Execute_TopNDistinct(t *testing.T) { - data, err := ioutil.ReadFile("testdata/schema.json") + data, err := os.ReadFile("testdata/schema.json") if err != nil { t.Fatal(err) } @@ -7202,7 +7202,7 @@ func Test_Executor_Execute_UnionRows(t *testing.T) { } func TestTimelessClearRegression(t *testing.T) { - data, err := ioutil.ReadFile("testdata/timeRegressionSchema.json") + data, err := os.ReadFile("testdata/timeRegressionSchema.json") if err != nil { t.Fatal(err) } diff --git a/field_test.go b/field_test.go index 2e1192d0f..7018c71b9 100644 --- a/field_test.go +++ b/field_test.go @@ -9,12 +9,12 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/featurebasedb/featurebase/v3/test" "github.com/featurebasedb/featurebase/v3/testhook" + "github.com/google/go-cmp/cmp" "github.com/pkg/errors" ) @@ -302,7 +302,7 @@ func TestFieldInfoMarshal(t *testing.T) { t.Fatalf("unexpected error marshalling index info, %v", err) } expected := []byte(`{"name":"timestamp","createdAt":1649270079233541000,"options":{"type":"timestamp","epoch":"1970-01-01T00:00:00Z","bitDepth":0,"min":-4294967296,"max":4294967296,"timeUnit":"s"}}`) - if bytes.Compare(a, expected) != 0 { + if !bytes.Equal(a, expected) { t.Fatalf("expected %s, got %s", expected, a) } } diff --git a/fragment.go b/fragment.go index 318efb86c..be650ce61 100644 --- a/fragment.go +++ b/fragment.go @@ -11,7 +11,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "math" "math/bits" "os" @@ -23,7 +22,6 @@ import ( "time" "github.com/cespare/xxhash" - "github.com/gogo/protobuf/proto" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/logger" pnet "github.com/featurebasedb/featurebase/v3/net" @@ -35,6 +33,7 @@ import ( "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/featurebasedb/featurebase/v3/vprint" + "github.com/gogo/protobuf/proto" "github.com/pkg/errors" ) @@ -250,7 +249,7 @@ func (f *fragment) openCache() error { // Read cache data from disk. path := f.cachePath() - buf, err := ioutil.ReadFile(path) + buf, err := os.ReadFile(path) if os.IsNotExist(err) { return nil } else if err != nil { @@ -2612,7 +2611,7 @@ func (f *fragment) flushCache() error { return errors.Wrap(err, "mkdir") } // Write to disk. - if err := ioutil.WriteFile(f.cachePath(), buf, 0600); err != nil { + if err := os.WriteFile(f.cachePath(), buf, 0600); err != nil { return errors.Wrap(err, "writing") } @@ -2675,7 +2674,7 @@ func (f *fragment) writeCacheToArchive(tw *tar.Writer) error { defer f.mu.Unlock() // Read cache into buffer. - buf, err := ioutil.ReadFile(f.cachePath()) + buf, err := os.ReadFile(f.cachePath()) if os.IsNotExist(err) { return nil } else if err != nil { @@ -2744,9 +2743,9 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { // this is reading from inside a tarball, so definitely no need // to close it here. - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { - return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)") + return errors.Wrap(err, "fillFragmentFromArchive io.ReadAll(r)") } if len(data) == 0 { return nil @@ -2771,10 +2770,10 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { func (f *fragment) readCacheFromArchive(r io.Reader) error { // Slurp data from reader and write to disk. - buf, err := ioutil.ReadAll(r) + buf, err := io.ReadAll(r) if err != nil { return errors.Wrap(err, "reading") - } else if err := ioutil.WriteFile(f.cachePath(), buf, 0600); err != nil { + } else if err := os.WriteFile(f.cachePath(), buf, 0600); err != nil { return errors.Wrap(err, "writing") } @@ -3652,7 +3651,7 @@ func (f *fragment) sortBsiData(tx Tx, filter *Row, bitDepth uint64, sort_desc bo return nil, err } pos := consider.Difference(row) - row, err = f.row(tx, 0) + _, err = f.row(tx, 0) if err != nil { return nil, err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 70534d992..512fe9da4 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math/rand" "os" "reflect" @@ -3083,7 +3082,7 @@ func BenchmarkFileWrite(b *testing.B) { // We're deleting these files as we go because // otherwise the benchmark could fill up // $TMPDIR before it finishes running. - f, err := ioutil.TempFile(*TempDir, "") + f, err := os.CreateTemp(*TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } diff --git a/hash/blake3_test.go b/hash/blake3_test.go index f3e8e4976..8952cb52c 100644 --- a/hash/blake3_test.go +++ b/hash/blake3_test.go @@ -5,7 +5,6 @@ package hash import ( "encoding/hex" "fmt" - "io/ioutil" "os" "path" "testing" @@ -55,19 +54,19 @@ func TestHashOfDir(t *testing.T) { } bmessage := []byte("hello B\n") - if err := ioutil.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil { + if err := os.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil { t.Fatal(err) } cmessage := []byte("hello C\n") - if err := ioutil.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil { + if err := os.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil { t.Fatal(err) } hsh := HashOfDir(dir) c2message := []byte("hello C2\n") - if err := ioutil.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil { + if err := os.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil { t.Fatal(err) } diff --git a/http_handler.go b/http_handler.go index f46c5cbc7..cdd186338 100644 --- a/http_handler.go +++ b/http_handler.go @@ -27,9 +27,6 @@ import ( "sync" "time" - "github.com/felixge/fgprof" - "github.com/gorilla/handlers" - "github.com/gorilla/mux" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/authz" "github.com/featurebasedb/featurebase/v3/disco" @@ -40,6 +37,9 @@ import ( "github.com/featurebasedb/featurebase/v3/rbf" "github.com/featurebasedb/featurebase/v3/storage" "github.com/featurebasedb/featurebase/v3/tracing" + "github.com/felixge/fgprof" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" dto "github.com/prometheus/client_model/go" @@ -703,8 +703,6 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http return } - ctx := r.Context() - // check if IP is in allowed networks, if yes give it admin permissions allowedNetwork, ctx := h.chkAllowedNetworks(r) if allowedNetwork { @@ -721,9 +719,6 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http uinfo, err := h.auth.Authenticate(access, refresh) - ctx = context.WithValue(ctx, authn.ContextValueAccessToken, "Bearer "+access) - ctx = context.WithValue(ctx, authn.ContextValueRefreshToken, refresh) - if err != nil { http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusForbidden) return @@ -1512,7 +1507,7 @@ type postIndexRequest struct { Options IndexOptions `json:"options"` } -//_postIndexRequest is necessary to avoid recursion while decoding. +// _postIndexRequest is necessary to avoid recursion while decoding. type _postIndexRequest postIndexRequest // Custom Unmarshal JSON to validate request body when creating a new index. @@ -3266,7 +3261,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request ctx := r.Context() // Read entire body. - span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body") body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() @@ -3339,7 +3334,7 @@ func (h *Handler) handlePostShardImportRoaring(w http.ResponseWriter, r *http.Re ctx := r.Context() // Read entire body. - span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body") body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() @@ -3408,7 +3403,7 @@ func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { ctx := r.Context() // Read entire body. - span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body") body, err := readBody(r) span.LogKV("bodySize", len(body)) span.Finish() diff --git a/http_handler_internal_test.go b/http_handler_internal_test.go index c6a7192aa..eb8748dea 100644 --- a/http_handler_internal_test.go +++ b/http_handler_internal_test.go @@ -8,7 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "net/url" @@ -18,8 +18,8 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt" "github.com/featurebasedb/featurebase/v3/authn" + "github.com/golang-jwt/jwt" "golang.org/x/oauth2" "github.com/featurebasedb/featurebase/v3/authz" @@ -187,7 +187,7 @@ func TestFieldOptionValidation(t *testing.T) { func readResponse(w *httptest.ResponseRecorder) ([]byte, error) { res := w.Result() defer res.Body.Close() - return ioutil.ReadAll(res.Body) + return io.ReadAll(res.Body) } // common variables used for testing auth @@ -668,7 +668,7 @@ func TestChkAuthN(t *testing.T) { expiredToken = "Bearer " + expiredToken testingHandler := func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("good")) + _, _ = w.Write([]byte("good")) } cases := []struct { @@ -704,7 +704,7 @@ func TestChkAuthN(t *testing.T) { r.Header.Add("Authorization", test.token) test.handler(w, r) resp := w.Result() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) defer resp.Body.Close() if err != nil { t.Fatal(err) diff --git a/http_handler_test.go b/http_handler_test.go index b57b41d88..eb7d9f4b6 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -6,7 +6,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" gohttp "net/http" @@ -22,6 +22,7 @@ import ( "github.com/featurebasedb/featurebase/v3/encoding/proto" "github.com/featurebasedb/featurebase/v3/server" "github.com/featurebasedb/featurebase/v3/test" + "github.com/stretchr/testify/assert" ) func TestHandlerOptions(t *testing.T) { @@ -195,7 +196,8 @@ func TestUpdateFieldTTL(t *testing.T) { if resp.StatusCode == 400 || resp.StatusCode == 404 { // unmarshal error message to check against expErr var respBody map[string]interface{} - json.NewDecoder(resp.Body).Decode(&respBody) + err := json.NewDecoder(resp.Body).Decode(&respBody) + assert.NoError(t, err) errMsg := respBody["error"].(map[string]interface{})["message"].(string) if !strings.Contains(errMsg, test.expErr) { @@ -300,7 +302,8 @@ func TestUpdateFieldNoStandardView(t *testing.T) { if resp.StatusCode == 400 { // unmarshal error message to check against expErr var respBody map[string]interface{} - json.NewDecoder(resp.Body).Decode(&respBody) + err := json.NewDecoder(resp.Body).Decode(&respBody) + assert.NoError(t, err) errMsg := respBody["error"].(map[string]interface{})["message"].(string) if !strings.Contains(errMsg, test.expErr) { @@ -740,7 +743,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` tmpDir := t.TempDir() permissionsPath := path.Join(tmpDir, "test-permissions.yaml") - err := ioutil.WriteFile(permissionsPath, []byte(permissions1), 0600) + err := os.WriteFile(permissionsPath, []byte(permissions1), 0600) if err != nil { t.Fatalf("failed to write permissions file: %v", err) } @@ -902,7 +905,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"` } if ipTest.StatusCode == 200 { - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading resp body :%v", err) } diff --git a/http_translator.go b/http_translator.go index f1b733392..051055053 100644 --- a/http_translator.go +++ b/http_translator.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "reflect" "sync" @@ -109,7 +108,7 @@ func (r *HTTPTranslateEntryReader) Open() error { r.body.Close() return ErrNotImplemented } else if resp.StatusCode != http.StatusOK { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) r.body.Close() return fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, r.URL, bytes.TrimSpace(body)) } diff --git a/idk/api/source_test.go b/idk/api/source_test.go index 5cd73525f..c8108240d 100644 --- a/idk/api/source_test.go +++ b/idk/api/source_test.go @@ -2,7 +2,6 @@ package api import ( "io" - "io/ioutil" "net/http" "strings" "testing" @@ -119,13 +118,13 @@ func TestIngest(t *testing.T) { return } defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck + defer io.Copy(io.Discard, resp.Body) //nolint: errcheck if !assert.Equal(t, 200, resp.StatusCode) { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) t.Logf("request error: %s", body) return } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { return } @@ -190,13 +189,13 @@ func TestIngest(t *testing.T) { return } defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck + defer io.Copy(io.Discard, resp.Body) //nolint: errcheck if !assert.Equal(t, 200, resp.StatusCode) { - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) t.Logf("request error: %s", body) return } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if !assert.NoError(t, err) { return } diff --git a/idk/cmd/molecula-consumer-github/main.go b/idk/cmd/molecula-consumer-github/main.go index a10ae8266..2af592adc 100644 --- a/idk/cmd/molecula-consumer-github/main.go +++ b/idk/cmd/molecula-consumer-github/main.go @@ -7,7 +7,6 @@ import ( "expvar" "fmt" "io" - "io/ioutil" "log" "net/http" _ "net/http/pprof" @@ -15,9 +14,9 @@ import ( "path/filepath" "time" - "github.com/jaffee/commandeer/pflag" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/jaffee/commandeer/pflag" "github.com/pkg/errors" ) @@ -384,9 +383,9 @@ func (m *Main) openURLReader(t time.Time) (io.ReadCloser, error) { } // If cache enabled, write to file first and then return. - if buf, err := ioutil.ReadAll(resp.Body); err != nil { + if buf, err := io.ReadAll(resp.Body); err != nil { return nil, err - } else if err := ioutil.WriteFile(cachePath+".tmp", buf, 0666); err != nil { + } else if err := os.WriteFile(cachePath+".tmp", buf, 0666); err != nil { return nil, err } else if err := os.Rename(cachePath+".tmp", cachePath); err != nil { return nil, err diff --git a/idk/common/configMap.go b/idk/common/configMap.go index ad62826ca..3d7d138e2 100644 --- a/idk/common/configMap.go +++ b/idk/common/configMap.go @@ -3,7 +3,7 @@ package common import ( "encoding/json" "fmt" - "io/ioutil" + "os" "strings" "sync/atomic" @@ -47,7 +47,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { var err error configMap := &confluent.ConfigMap{} if m.KafkaConfiguration != "" { - file, er := ioutil.ReadFile(m.KafkaConfiguration) + file, er := os.ReadFile(m.KafkaConfiguration) if er != nil { return nil, er } diff --git a/idk/csv/csv_test.go b/idk/csv/csv_test.go index 619c9bc3f..b0f6f361d 100644 --- a/idk/csv/csv_test.go +++ b/idk/csv/csv_test.go @@ -3,7 +3,6 @@ package csv import ( "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -257,7 +256,7 @@ func tim(t *testing.T, tstr string) time.Time { } func writeTempFile(t *testing.T, data string) string { - f, err := ioutil.TempFile("", "") + f, err := os.CreateTemp("", "") if err != nil { t.Fatalf("getting temp file: %v", err) } @@ -321,7 +320,7 @@ func TestStreamFileNames(t *testing.T) { } func testDirTree(tree string) (string, error) { - tmp, err := ioutil.TempDir("", "") + tmp, err := os.MkdirTemp("", "") if err != nil { return "", err } diff --git a/idk/datagen/custom.go b/idk/datagen/custom.go index 90723c8bb..d81a422a0 100644 --- a/idk/datagen/custom.go +++ b/idk/datagen/custom.go @@ -5,7 +5,6 @@ import ( "encoding/csv" "fmt" "io" - "io/ioutil" "log" "math/rand" "os" @@ -31,7 +30,7 @@ type Custom struct { // NewCustom returns a new instance of Custom. func NewCustom(cfg SourceGeneratorConfig) Sourcer { conf := &CustomConfig{} - if bytes, err := ioutil.ReadFile(cfg.CustomConfig); err != nil { + if bytes, err := os.ReadFile(cfg.CustomConfig); err != nil { return &Custom{err: errors.Wrap(err, "reading custom config file")} } else if err = yaml.Unmarshal(bytes, conf); err != nil { return &Custom{err: errors.Wrap(err, "unmarshaling custom config file")} diff --git a/idk/datagen/custom_test.go b/idk/datagen/custom_test.go index ec289ef99..9a07fe29a 100644 --- a/idk/datagen/custom_test.go +++ b/idk/datagen/custom_test.go @@ -1,7 +1,6 @@ package datagen import ( - "io/ioutil" "math/rand" "os" "strings" @@ -148,7 +147,7 @@ func TestGetIDKFields(t *testing.T) { } func TestNewCustomUnmarshalling(t *testing.T) { - tmp, err := ioutil.TempFile("", "") + tmp, err := os.CreateTemp("", "") if err != nil { t.Fatalf("creating temp file: %v", err) } diff --git a/idk/idallocator.go b/idk/idallocator.go index 0660471aa..1ce11b53d 100644 --- a/idk/idallocator.go +++ b/idk/idallocator.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math/bits" "net/http" "net/url" @@ -377,7 +376,7 @@ func (ps *pilosaIDManager) reserve(ctx context.Context, reserveReq pilosacore.ID err = errors.Wrap(cerr, "closing ID reservation request body") } }() - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading ID reservation request body") } @@ -442,7 +441,7 @@ func (ps *pilosaIDManager) commit(ctx context.Context, commitRequest pilosacore. err = errors.Wrap(cerr, "closing ID reservation request body") } }() - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "reading ID reservation request body") } diff --git a/idk/idktest/idktest.go b/idk/idktest/idktest.go index 8ba55f0e6..48473820b 100644 --- a/idk/idktest/idktest.go +++ b/idk/idktest/idktest.go @@ -3,7 +3,7 @@ package idktest import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "strings" @@ -62,7 +62,7 @@ func DoExtractQuery(pql, index string) (ExtractResponse, error) { } defer resp.Body.Close() - s, err := ioutil.ReadAll(resp.Body) + s, err := io.ReadAll(resp.Body) if err != nil { return eResp, errors.Errorf("reading response: %v", err) } diff --git a/idk/ingest.go b/idk/ingest.go index d915476d6..7722bdcab 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -23,7 +23,6 @@ import ( "syscall" "time" - "github.com/felixge/fgprof" pilosacore "github.com/featurebasedb/featurebase/v3" pilosagrpc "github.com/featurebasedb/featurebase/v3/api/client" pilosaclient "github.com/featurebasedb/featurebase/v3/client" @@ -32,6 +31,7 @@ import ( "github.com/featurebasedb/featurebase/v3/prometheus" proto "github.com/featurebasedb/featurebase/v3/proto" "github.com/featurebasedb/featurebase/v3/stats" + "github.com/felixge/fgprof" "github.com/pkg/errors" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -1500,8 +1500,6 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor case DeleteSentinel: if hasMutex { //need to clear the mutex rec.Clears[valIdx] = nil //? maybe - } else { //TODO(twg) set fields not supported - } default: rec.Values[valIdx], err = idkField.PilosafyVal(rawRec[i]) diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 267f858b1..0b2d17ecb 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -15,11 +14,11 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt" "github.com/featurebasedb/featurebase/v3/authn" pilosaclient "github.com/featurebasedb/featurebase/v3/client" "github.com/featurebasedb/featurebase/v3/idk/idktest" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/golang-jwt/jwt" "github.com/pkg/errors" ) @@ -436,7 +435,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) { if err != nil { t.Errorf("request error: %v", err) } - contents, err := ioutil.ReadAll(response.Body) + contents, err := io.ReadAll(response.Body) defer response.Body.Close() if err != nil { t.Errorf("read error: %v", err) diff --git a/idk/internal/s3.go b/idk/internal/s3.go index fb2172021..ed6a3ea05 100644 --- a/idk/internal/s3.go +++ b/idk/internal/s3.go @@ -2,7 +2,6 @@ package internal import ( "bytes" - "io/ioutil" "net/url" "os" "strings" @@ -57,7 +56,7 @@ func ReadFileOrURL(name string, s3client s3iface.S3API) ([]byte, error) { } content = buf.Bytes() } else { - content, err = ioutil.ReadFile(name) + content, err = os.ReadFile(name) if err != nil { if os.IsNotExist(err) { return nil, FileOrURLNotFound @@ -91,7 +90,7 @@ func WriteFileOrURL(name string, contents []byte, s3client s3iface.S3API) error return errors.Wrapf(err, "putting S3 object %v", name) } } else { - err = ioutil.WriteFile(name, contents, 0644) + err = os.WriteFile(name, contents, 0644) if err != nil { return errors.Wrapf(err, "reading file %v", name) } diff --git a/idk/kafka/cmd_test.go b/idk/kafka/cmd_test.go index 2ea1879bc..c4d445552 100644 --- a/idk/kafka/cmd_test.go +++ b/idk/kafka/cmd_test.go @@ -5,7 +5,7 @@ package kafka import ( "fmt" - "io/ioutil" + "io" "math/rand" "net/http" "os" @@ -17,11 +17,11 @@ import ( "time" confluent "github.com/confluentinc/confluent-kafka-go/kafka" - liavro "github.com/linkedin/goavro/v2" pilosaclient "github.com/featurebasedb/featurebase/v3/client" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/idk/kafka/csrc" "github.com/featurebasedb/featurebase/v3/logger" + liavro "github.com/linkedin/goavro/v2" ) var pilosaHost string @@ -673,7 +673,7 @@ func tDoHTTPPost(t *testing.T, url, contentType, body string) string { t.Fatalf("making POST request: %v", err) } - bod, err := ioutil.ReadAll(resp.Body) + bod, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading POST response bdoy: %v", err) } diff --git a/idk/kafka/csrc/csrc.go b/idk/kafka/csrc/csrc.go index e5bff755e..85ae1b85a 100644 --- a/idk/kafka/csrc/csrc.go +++ b/idk/kafka/csrc/csrc.go @@ -4,7 +4,7 @@ import ( "crypto/tls" "encoding/json" "fmt" - "io/ioutil" + "io" "log" "net/http" "strings" @@ -114,7 +114,7 @@ func unmarshalRespErr(resp *http.Response, err error, into interface{}) error { return errors.Wrap(err, "making http request") } if resp.StatusCode != 200 { - bod, err := ioutil.ReadAll(resp.Body) + bod, err := io.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "reading body") } diff --git a/idk/kafka/put.go b/idk/kafka/put.go index 9fd940833..1f6c943e9 100644 --- a/idk/kafka/put.go +++ b/idk/kafka/put.go @@ -3,16 +3,15 @@ package kafka import ( "context" "encoding/binary" - "io/ioutil" "os" "time" confluent "github.com/confluentinc/confluent-kafka-go/kafka" - liavro "github.com/linkedin/goavro/v2" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/idk/common" "github.com/featurebasedb/featurebase/v3/idk/kafka/csrc" "github.com/featurebasedb/featurebase/v3/logger" + liavro "github.com/linkedin/goavro/v2" "github.com/pkg/errors" ) @@ -173,7 +172,7 @@ func (p *PutCmd) getSchema() (string, error) { if p.Schema != "" { return p.Schema, nil } - bytes, err := ioutil.ReadFile(p.SchemaFile) + bytes, err := os.ReadFile(p.SchemaFile) if err != nil { return "", errors.Wrap(err, "reading schema file") } diff --git a/idk/kafka/source.go b/idk/kafka/source.go index 6423fb60a..b78ffd428 100644 --- a/idk/kafka/source.go +++ b/idk/kafka/source.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -21,11 +20,11 @@ import ( "time" confluent "github.com/confluentinc/confluent-kafka-go/kafka" - "github.com/go-avro/avro" pilosaclient "github.com/featurebasedb/featurebase/v3/client" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/idk/common" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/go-avro/avro" "github.com/pkg/errors" ) @@ -253,11 +252,6 @@ func (r *Record) Data() []interface{} { return r.data } -// Assuming committed msgs are in order -func calOffsetDiff(section, committed []confluent.TopicPartition) []confluent.TopicPartition { - return section[len(committed):] -} - func (s *Source) CommitMessages(recs []confluent.TopicPartition) ([]confluent.TopicPartition, error) { return s.client.CommitOffsets(recs) } @@ -1022,7 +1016,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { } defer schemaUrlResponse.Body.Close() if schemaUrlResponse.StatusCode >= 300 { - bod, err := ioutil.ReadAll(schemaUrlResponse.Body) + bod, err := io.ReadAll(schemaUrlResponse.Body) if err != nil { return nil, errors.Wrapf(err, "Failed to get schema, code: %d, no body", schemaUrlResponse.StatusCode) } @@ -1056,7 +1050,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { s.Log.Infof("Problem getting subject/version info for schema: %v", err) } else { if subVerResponse.StatusCode >= 300 { - bod, err := ioutil.ReadAll(subVerResponse.Body) + bod, err := io.ReadAll(subVerResponse.Body) s.Log.Infof("Problem getting subject/version info for schema, response: %s. Err reading body: %v", bod, err) } defer subVerResponse.Body.Close() @@ -1066,7 +1060,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) { Version int `json:"version"` } - if bod, err := ioutil.ReadAll(subVerResponse.Body); err != nil { + if bod, err := io.ReadAll(subVerResponse.Body); err != nil { s.Log.Infof("decoding subj/version %s body: %v", schemaSubVerUrl, err) } else if err := json.Unmarshal(bod, &tempSchemaStruct); err != nil { s.Log.Infof("decoding schema subject & version from registry: %v", err) diff --git a/idk/kafka/source_test.go b/idk/kafka/source_test.go index 6cca9926b..3f796a259 100644 --- a/idk/kafka/source_test.go +++ b/idk/kafka/source_test.go @@ -9,7 +9,6 @@ import ( "encoding/binary" "encoding/json" "fmt" - "io/ioutil" "math/big" "math/rand" "os" @@ -20,13 +19,13 @@ import ( "github.com/confluentinc/confluent-kafka-go/kafka" confluent "github.com/confluentinc/confluent-kafka-go/kafka" - "github.com/glycerine/vprint" - "github.com/go-avro/avro" - liavro "github.com/linkedin/goavro/v2" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/idk/common" "github.com/featurebasedb/featurebase/v3/idk/kafka/csrc" "github.com/featurebasedb/featurebase/v3/logger" + "github.com/glycerine/vprint" + "github.com/go-avro/avro" + liavro "github.com/linkedin/goavro/v2" ) func configureSourceTestFlags(source *Source) { @@ -81,7 +80,7 @@ func TestAvroToPDKSchema(t *testing.T) { } // check that we've covered all the test schemas - files, err := ioutil.ReadDir("./testdata/schemas") + files, err := os.ReadDir("./testdata/schemas") if err != nil { t.Fatalf("reading directory: %v", err) } @@ -251,7 +250,7 @@ func decodeTestSchema(t *testing.T, filename string) avro.Schema { } func readTestSchema(t *testing.T, filename string) string { - bytes, err := ioutil.ReadFile("./testdata/schemas/" + filename) + bytes, err := os.ReadFile("./testdata/schemas/" + filename) if err != nil { t.Fatalf("reading schema file: %v", err) } diff --git a/idk/kafka_sasl/source.go b/idk/kafka_sasl/source.go index 61a3a757d..32cfa0b8a 100644 --- a/idk/kafka_sasl/source.go +++ b/idk/kafka_sasl/source.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "io" - "io/ioutil" + "os" "sort" "strconv" "sync" @@ -185,11 +185,6 @@ func (r *Record) Commit(ctx context.Context) error { return nil } -// Assuming committed msgs are in order -func calOffsetDiff(section, committed []confluent.TopicPartition) []confluent.TopicPartition { - return section[len(committed):] -} - func (r *Record) Data() []interface{} { return r.data } @@ -200,7 +195,7 @@ func (s *Source) Open() error { return errors.New("needs header specification file") } - headerData, err := ioutil.ReadFile(s.Header) + headerData, err := os.ReadFile(s.Header) if err != nil { return errors.Wrap(err, "reading header file") } diff --git a/idk/kafka_static/source.go b/idk/kafka_static/source.go index b545a2769..6e3b02200 100644 --- a/idk/kafka_static/source.go +++ b/idk/kafka_static/source.go @@ -5,8 +5,8 @@ import ( "context" "encoding/json" "io" - "io/ioutil" "net/url" + "os" "strconv" "strings" "time" @@ -262,7 +262,7 @@ func (s *Source) readFileOrURL(name string) ([]byte, error) { s.Log.Printf("read %d bytes from %s\n", bytesRead, name) content = buf.Bytes() } else { - content, err = ioutil.ReadFile(name) + content, err = os.ReadFile(name) if err != nil { return nil, errors.Wrapf(err, "reading file %v", name) } diff --git a/idk/kafka_static/source_test.go b/idk/kafka_static/source_test.go index d840f86e7..e1fc37cb7 100644 --- a/idk/kafka_static/source_test.go +++ b/idk/kafka_static/source_test.go @@ -6,9 +6,9 @@ package kafka_static import ( "context" "fmt" - "io/ioutil" "math/rand" "net" + "os" "reflect" "testing" "time" @@ -68,7 +68,7 @@ func TestKafkaStaticSourceLocal(t *testing.T) { src := NewSource() configureSourceTestFlags(src) { - headerData, err := ioutil.ReadFile(test.header) + headerData, err := os.ReadFile(test.header) if err != nil { t.Fatalf("reading header file: %v", err) } diff --git a/idk/kafkagen/cmd.go b/idk/kafkagen/cmd.go index 205ddc3aa..2c6e29ca1 100644 --- a/idk/kafkagen/cmd.go +++ b/idk/kafkagen/cmd.go @@ -3,15 +3,14 @@ package kafkagen import ( "encoding/binary" "fmt" - "io/ioutil" "log" "os" confluent "github.com/confluentinc/confluent-kafka-go/kafka" - liavro "github.com/linkedin/goavro/v2" "github.com/featurebasedb/featurebase/v3/idk" "github.com/featurebasedb/featurebase/v3/idk/common" "github.com/featurebasedb/featurebase/v3/idk/kafka/csrc" + liavro "github.com/linkedin/goavro/v2" "github.com/pkg/errors" ) @@ -125,7 +124,7 @@ func (m *Main) putRecordKafka(p *confluent.Producer, schemaID int, schema *liavr } func readSchema(filename string) (string, error) { - bytes, err := ioutil.ReadFile(filename) + bytes, err := os.ReadFile(filename) if err != nil { return "", err } diff --git a/idk/kinesis/logger_test.go b/idk/kinesis/logger_test.go index 19a49e79a..43586451e 100644 --- a/idk/kinesis/logger_test.go +++ b/idk/kinesis/logger_test.go @@ -102,7 +102,7 @@ func TestNewSinkErrorQueueSuccess(t *testing.T) { queue, err := NewSinkErrorQueue(mockSQS, "dummy-123", "a-b-c") - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, "a-b-c", queue.sinkId) assert.Equal(t, "dummy-123", queue.name) assert.Equal(t, "https://unit-test.queue.dummy-123.url", queue.url) @@ -149,8 +149,7 @@ func TestSinkErrorQueueFromSuccess(t *testing.T) { StreamName: streamName, } - var queue *SinkErrorQueue - queue = SinkErrorQueueFrom(mockSQS, source) + queue := SinkErrorQueueFrom(mockSQS, source) assert.Equal(t, validUuid, queue.sinkId) assert.Equal(t, "my-queue-01234", queue.name) @@ -166,8 +165,7 @@ func TestSinkErrorQueueFromFailMissingQueueName(t *testing.T) { streamName := fmt.Sprintf("sink-%s", validUuid) // Structure mimics how cloud ECS instance names its Kinesis stream. source := &Source{StreamName: streamName} - var queue *SinkErrorQueue - queue = SinkErrorQueueFrom(mockSQS, source) + queue := SinkErrorQueueFrom(mockSQS, source) assert.Equal(t, "", queue.sinkId) assert.Equal(t, "", queue.name) @@ -194,8 +192,7 @@ func TestSinkErrorQueueFromFailMalformedSinkId(t *testing.T) { StreamName: streamName, } - var queue *SinkErrorQueue - queue = SinkErrorQueueFrom(mockSQS, source) + queue := SinkErrorQueueFrom(mockSQS, source) assert.Equal(t, malformedUuid, queue.sinkId) // Keeps invalid sink ID around for downstream logging purposes. assert.Equal(t, "valid-queue-90123", queue.name) @@ -220,8 +217,7 @@ func TestSinkErrorQueueFromFailEmptyStreamNameDoesNotCausePanic(t *testing.T) { StreamName: "", } - var queue *SinkErrorQueue - queue = SinkErrorQueueFrom(mockSQS, source) + queue := SinkErrorQueueFrom(mockSQS, source) assert.Equal(t, "", queue.sinkId) // Keeps invalid sink ID around for downstream logging purposes. assert.Equal(t, "valid-queue-88888888888", queue.name) @@ -249,8 +245,7 @@ func TestSinkErrorQueueFromFailSQSGetQueueUrlErrored(t *testing.T) { StreamName: streamName, } - var queue *SinkErrorQueue - queue = SinkErrorQueueFrom(mockSQS, source) + queue := SinkErrorQueueFrom(mockSQS, source) assert.Equal(t, validUuid, queue.sinkId) // Keeps valid sink ID around for downstream logging purposes. assert.Equal(t, "queue-that-we-stole-962463", queue.name) @@ -276,7 +271,7 @@ func TestSinkErrorQueuePushWhenQueueNotAvailableDoesNotThrowErrorAndNoOps(t *tes // Logger is nil. err := emptySeq.Push(RecoverableErrorType, "werqw zxcw7228323974", nil) - assert.Nil(t, err) + assert.NoError(t, err) // Logger is not nil; check that a warning is issued using the Logger instance. logger := NewMapStashLogger() @@ -313,7 +308,7 @@ func TestSinkErrorQueuePushSuccess(t *testing.T) { errMsg := "079 cxmn, 198sfakjl" err := seq.Push(PanicErrorType, errMsg, nil) - assert.Nil(t, err) + assert.NoError(t, err) // Verify queue URL. assert.Equal(t, seq.url, actualQueueUrl) diff --git a/idk/kinesis/reader_test.go b/idk/kinesis/reader_test.go index ac84442c0..4bd5a3f03 100644 --- a/idk/kinesis/reader_test.go +++ b/idk/kinesis/reader_test.go @@ -158,19 +158,19 @@ func TestStreamReaderStartFetchCommitWithoutOffsets(t *testing.T) { } err := reader.Start() - assert.Nil(t, err) + assert.NoError(t, err) var records []ShardRecord for i := 0; i < testShardCount; i++ { rec, err := reader.FetchMessage(context.Background()) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, rec.SequenceNumber, aws.String("1")) records = append(records, rec) } assert.Empty(t, reader.offsets.Shards) err = reader.CommitMessages(context.Background(), records...) - assert.Nil(t, err) + assert.NoError(t, err) assert.Len(t, reader.offsets.Shards, 2) for i := 0; i < testShardCount; i++ { @@ -253,12 +253,12 @@ func TestStreamReaderStartFetchCommitFromExistingOffsets(t *testing.T) { } err := reader.Start() - assert.Nil(t, err) + assert.NoError(t, err) var records []ShardRecord for i := 0; i < testShardCount; i++ { rec, err := reader.FetchMessage(context.Background()) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, rec.SequenceNumber, aws.String("1")) records = append(records, rec) } @@ -270,7 +270,7 @@ func TestStreamReaderStartFetchCommitFromExistingOffsets(t *testing.T) { } err = reader.CommitMessages(context.Background(), records...) - assert.Nil(t, err) + assert.NoError(t, err) assert.Len(t, reader.offsets.Shards, 2) for i := 0; i < testShardCount; i++ { @@ -389,17 +389,17 @@ func TestStreamOrderlyReadsAfterResharding(t *testing.T) { ).Return(getRecordsOutput, nil).Once() err := reader.Start() - assert.Nil(t, err) + assert.NoError(t, err) for i := 1; i <= recordsBeforeClose; i++ { rec, err := reader.FetchMessage(context.Background()) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, strconv.Itoa(i), *rec.SequenceNumber) assert.Equal(t, rec.ShardID, shardName(0)) } rec, err := reader.FetchMessage(context.Background()) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, "1", *rec.SequenceNumber) assert.Equal(t, rec.ShardID, shardName(1)) diff --git a/idk/tls.go b/idk/tls.go index a91c8c079..63ed4ed69 100644 --- a/idk/tls.go +++ b/idk/tls.go @@ -3,7 +3,6 @@ package idk import ( "crypto/tls" "crypto/x509" - "io/ioutil" "log" "os" "os/signal" @@ -148,7 +147,7 @@ func getCertPool(capath string) (*x509.CertPool, error) { ) caCertData = []byte(capath) } else { - caCertData, err = ioutil.ReadFile(capath) + caCertData, err = os.ReadFile(capath) if err != nil { return nil, errors.Wrap(err, "loading tls ca key") } diff --git a/ingest/codec.go b/ingest/codec.go index 17af834cd..cd16c1a1e 100644 --- a/ingest/codec.go +++ b/ingest/codec.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math" "sort" "strconv" @@ -748,7 +747,7 @@ func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err // Parse reads a request, but does not sort the results at all or divide // them into shards. func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/ingest_test.go b/ingest_test.go index 044ab0cda..85241ffdd 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "encoding/json" - "io/ioutil" "os" "path/filepath" "strconv" @@ -234,7 +233,7 @@ func testQueries(t *testing.T, ctx context.Context, cmd *test.Command, index str // testOneIngestTestcase runs a set of actions, then cleans up after itself func testOneIngestTestcase(t *testing.T, ctx context.Context, cmd *test.Command, tcpath string) { - data, err := ioutil.ReadFile(tcpath) + data, err := os.ReadFile(tcpath) if err != nil { t.Fatalf("reading %q: %v", tcpath, err) } diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index 21f87cf76..8aebe5a26 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -130,6 +130,8 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient, } func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) { + //TODO lint - uses the fs.FileInfo.Mode to filter out directories later + // this does not exist in the os.DirEntry elements returned by os.ReadDir dirEntries, err := ioutil.ReadDir(dirPath) if err != nil { return nil, err diff --git a/internal_client.go b/internal_client.go index 43a918512..c0ad6f322 100644 --- a/internal_client.go +++ b/internal_client.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math" "math/rand" "net/http" @@ -20,13 +19,13 @@ import ( "strings" "time" - "github.com/hashicorp/go-retryablehttp" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/ingest" "github.com/featurebasedb/featurebase/v3/logger" pnet "github.com/featurebasedb/featurebase/v3/net" "github.com/featurebasedb/featurebase/v3/tracing" + "github.com/hashicorp/go-retryablehttp" "github.com/pkg/errors" "golang.org/x/oauth2" ) @@ -421,7 +420,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] return nil, errors.Wrap(err, "executing request") } defer resp.Body.Close() - buf, err = ioutil.ReadAll(resp.Body) + buf, err = io.ReadAll(resp.Body) if resp.StatusCode != 200 { if err != nil { return nil, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status) @@ -439,7 +438,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] } return nil, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg) } - // this is the err from ioutil.ReadAll, but in the case where resp.StatusCode + // this is the err from io.ReadAll, but in the case where resp.StatusCode // was 2xx, so we don't have a bad status. if err != nil { return nil, errors.Wrapf(err, "error reading response body") @@ -728,7 +727,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str defer resp.Body.Close() // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -816,7 +815,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *disco.Node, index defer resp.Body.Close() // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "reading") } @@ -1339,7 +1338,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi // Decode response object. var rsp BlockDataResponse - if body, err := ioutil.ReadAll(resp.Body); err != nil { + if body, err := io.ReadAll(resp.Body); err != nil { return nil, nil, errors.Wrap(err, "reading") } else if err := c.serializer.Unmarshal(body, &rsp); err != nil { return nil, nil, errors.Wrap(err, "unmarshalling") @@ -1372,7 +1371,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b return errors.Wrap(err, "executing request") } defer resp.Body.Close() - _, err = io.Copy(ioutil.Discard, resp.Body) + _, err = io.Copy(io.Discard, resp.Body) return errors.Wrap(err, "draining SendMessage response body") } @@ -1421,7 +1420,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i defer resp.Body.Close() // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1473,7 +1472,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in defer resp.Body.Close() // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1505,7 +1504,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]P defer resp.Body.Close() // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading") } @@ -1552,7 +1551,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i }() // Read the response body. - result, err := ioutil.ReadAll(resp.Body) + result, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading response") } @@ -1601,7 +1600,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i }() // Read the response body. - result, err := ioutil.ReadAll(resp.Body) + result, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading response") } @@ -1651,7 +1650,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, }() // Read the response body. - result, err := ioutil.ReadAll(resp.Body) + result, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading response") } @@ -1704,7 +1703,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, }() // Read the response body. - result, err := ioutil.ReadAll(resp.Body) + result, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading response") } @@ -1749,7 +1748,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, }() // Read the response body. - result, err := ioutil.ReadAll(resp.Body) + result, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "reading response") } @@ -1781,7 +1780,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*Transact return nil, errors.Wrap(err, "executing request") } defer func() { - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() }() trnsMap := make(map[string]*Transaction) @@ -1820,7 +1819,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou return nil, errors.Wrap(err, "executing request") } defer func() { - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() }() tr := &TransactionResponse{} @@ -1855,7 +1854,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*Tra return nil, errors.Wrap(err, "executing request") } defer func() { - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() }() tr := &TransactionResponse{} @@ -1892,7 +1891,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*Transa return nil, errors.Wrap(err, "executing request") } defer func() { - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() }() tr := &TransactionResponse{} @@ -1994,7 +1993,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return resp, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status) } diff --git a/logger/filewriter_test.go b/logger/filewriter_test.go index a731536a5..d69b67ea6 100644 --- a/logger/filewriter_test.go +++ b/logger/filewriter_test.go @@ -26,7 +26,6 @@ package logger import ( - "io/ioutil" "os" "testing" @@ -35,14 +34,13 @@ import ( // TestReopenAppend -- make sure we always append to an existing file // -// 1. Create a sample file using normal means -// 2. Open a ioreopen.File -// write line 1 -// 3. call Reopen -// write line 2 -// 4. close file -// 5. read file, make sure it contains line0,line1,line2 -// +// 1. Create a sample file using normal means +// 2. Open a ioreopen.File +// write line 1 +// 3. call Reopen +// write line 2 +// 4. close file +// 5. read file, make sure it contains line0,line1,line2 func TestReopenAppend(t *testing.T) { forig, err := testhook.TempFile(t, "logger-reopen") if err != nil { @@ -83,7 +81,7 @@ func TestReopenAppend(t *testing.T) { t.Errorf("Got closing error for %s: %s", fname, err) } - out, err := ioutil.ReadFile(fname) + out, err := os.ReadFile(fname) if err != nil { t.Fatalf("Unable read in final file %s: %s", fname, err) } @@ -152,7 +150,7 @@ func TestChangeInode(t *testing.T) { t.Errorf("Got closing error for %s: %s", fname, err) } - out, err := ioutil.ReadFile(fname) + out, err := os.ReadFile(fname) if err != nil { t.Fatalf("Unable read in final file %s: %s", fname, err) } diff --git a/logger/logger.go b/logger/logger.go index 0fbbf0d2b..6ef8cc1f8 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -6,7 +6,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" "sync" "time" @@ -258,5 +257,5 @@ func (b *bufferLogger) Panicf(format string, v ...interface{}) { func (b *bufferLogger) ReadAll() ([]byte, error) { b.mu.Lock() defer b.mu.Unlock() - return ioutil.ReadAll(b.buf) + return io.ReadAll(b.buf) } diff --git a/pb/pb.go b/pb/pb.go index 0776df321..3b688a1c9 100644 --- a/pb/pb.go +++ b/pb/pb.go @@ -4,7 +4,6 @@ package pb import ( "io" - "io/ioutil" "github.com/gogo/protobuf/proto" ) @@ -52,7 +51,7 @@ func NewDecoder(r io.Reader) *Decoder { // Decode reads all bytes from the reader and unmarshals them into pb. func (dec *Decoder) Decode(pb proto.Message) error { - buf, err := ioutil.ReadAll(dec.r) + buf, err := io.ReadAll(dec.r) if err != nil { return err } diff --git a/pg/protocol.go b/pg/protocol.go index db250ed8b..fffd01760 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -11,7 +11,6 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "net" "regexp" "strings" @@ -1087,7 +1086,7 @@ func (s *Server) handleShutdown(conn net.Conn, w message.Writer, encoder *messag go func() { defer wg.Done() - io.Copy(ioutil.Discard, conn) //nolint:errcheck + io.Copy(io.Discard, conn) //nolint:errcheck }() // Attempt to send the shutdown notification. diff --git a/pql/parser.go b/pql/parser.go index 2dcf6720b..f805329ae 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -5,7 +5,6 @@ package pql import ( "fmt" "io" - "io/ioutil" "strconv" "strings" "unicode/utf8" @@ -40,7 +39,7 @@ func ParseString(s string) (*Query, error) { // Parse parses the next node in the query. func (p *parser) Parse() (*Query, error) { - buf, err := ioutil.ReadAll(p.r) + buf, err := io.ReadAll(p.r) if err != nil { return nil, errors.Wrap(err, "reading buffer to parse") } diff --git a/pql/parser_test.go b/pql/parser_test.go index 9c5e2ec55..670e9adff 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -218,7 +218,7 @@ func TestParser_Parse(t *testing.T) { ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } - q, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`) + _, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`) if err == nil { t.Fatal("no error parsing invalid date") } else if !strings.Contains(err.Error(), "not a valid timestamp") { diff --git a/rbf.go b/rbf.go index 97809b1b4..5c042d676 100644 --- a/rbf.go +++ b/rbf.go @@ -68,7 +68,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) { // rbfDBRegistrar also allows opening the same path twice to // result in sharing the same open database handle, and // thus the same transactional guarantees. -// type rbfDBRegistrar struct { mu sync.Mutex mp map[*RbfDBWrapper]bool @@ -224,11 +223,6 @@ func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (c // which is expensive in practice and only really useful occasionally. const sortedParanoia = false -type countResults struct { - changeCount int - err error -} - func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { if len(a) == 0 { return 0, nil diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 1a5914c2c..ad2b536b6 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -15,6 +15,7 @@ import ( "github.com/featurebasedb/featurebase/v3/rbf" "github.com/featurebasedb/featurebase/v3/roaring" + "github.com/stretchr/testify/assert" ) func TestCursor_FirstNext(t *testing.T) { @@ -593,7 +594,8 @@ func TestCursor_BitmapBitN(t *testing.T) { if c1 != c2 { t.Fatalf("expected count %d, got %d", c2, c1) } - tx.Commit() + err = tx.Commit() + assert.NoError(t, err) } func TestCursor_RLEConversion(t *testing.T) { @@ -1060,7 +1062,7 @@ func TestCursor_RemoveCells(t *testing.T) { //f, err := os.OpenFile("before.dot", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 066) } -//These aren't test i'm just using to generate graphs to look at structure +// These aren't test i'm just using to generate graphs to look at structure func TestCursor_PlayContainer(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) diff --git a/rbf/db.go b/rbf/db.go index 93e719d5e..ed374deb3 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -344,7 +344,7 @@ func (db *DB) checkpoint() (err error) { if IsBitmapHeader(page) { pgno = readPageNo(page) if i+1 < db.walPageN { - if page, err = db.readWALPageAt(i + 1); err != nil { + if _, err = db.readWALPageAt(i + 1); err != nil { return err } } else { diff --git a/rbf/db_test.go b/rbf/db_test.go index 459e0b519..0b9243265 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -15,9 +15,9 @@ import ( _ "net/http/pprof" - "github.com/felixge/fgprof" "github.com/featurebasedb/featurebase/v3/rbf" rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg" + "github.com/felixge/fgprof" "golang.org/x/sync/errgroup" ) @@ -579,7 +579,7 @@ func BenchmarkDbCheckpoint(b *testing.B) { benchmarkOneCheckpoint(b, randInts) } b.StopTimer() - done() + _ = done() } // better diagnosis of deadlocks/hung situations versus just really slow "Quick" tests. diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 1e840e1ae..0a160f412 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -16,6 +16,7 @@ import ( "github.com/featurebasedb/featurebase/v3/rbf" "github.com/featurebasedb/featurebase/v3/roaring" + "github.com/stretchr/testify/assert" ) func TestTx_CommitRollback(t *testing.T) { @@ -1150,37 +1151,37 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { for pgno, info := range infos { switch info := info.(type) { case *rbf.MetaPageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "meta") - pf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "meta") + _, _ = pf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) case *rbf.RootRecordPageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "rootrec") - pf("next=%d\n", info.Next) + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "rootrec") + _, _ = pf("next=%d\n", info.Next) case *rbf.LeafPageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "leaf") - pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "leaf") + _, _ = pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BranchPageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "branch") - pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "branch") + _, _ = pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BitmapPageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "bitmap") - pf("-\n") + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "bitmap") + _, _ = pf("-\n") case *rbf.FreePageInfo: - pf("%-8d ", pgno) - pf("%-10s ", "free") - pf("-\n") + _, _ = pf("%-8d ", pgno) + _, _ = pf("%-10s ", "free") + _, _ = pf("-\n") default: - t.Fatal(fmt.Sprintf("unexpected page info type %T", info)) + t.Fatalf("unexpected page info type %T", info) } } @@ -1212,7 +1213,7 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { ifError(db.Check()) tx := MustBegin(t, db, true) - tx.DeleteBitmapsWithPrefix(prefix) + assert.Nil(t, tx.DeleteBitmapsWithPrefix(prefix)) ifError(tx.Commit()) ifError(db.Check()) checkInfos(pBuf) diff --git a/roaring/fuzzer.go b/roaring/fuzzer.go index 38f7bd8d9..7dc754dc9 100644 --- a/roaring/fuzzer.go +++ b/roaring/fuzzer.go @@ -8,7 +8,6 @@ package roaring import ( "encoding/binary" "fmt" - "io/ioutil" "reflect" ) @@ -293,7 +292,7 @@ func bytesToUint64s(data []byte) []uint64 { // make sure filename is not already in the corpus. func addSliceToCorpus(slice []uint64, filename, path string) { data := uint64sToBytes(slice) - err := ioutil.WriteFile(path+"/"+filename, data, 0750) + err := os.WriteFile(path+"/"+filename, data, 0750) if err != nil { fmt.Printf("could not write to file: %v\n", err) } diff --git a/roaring/roaring.go b/roaring/roaring.go index 3465f93f8..5603199c3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1364,31 +1364,49 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // the bitmap at a specific key, ^ symbol represents the bitmaps current container iteration position, // and the - symbol represents a container that is at the current iteration position, but has been marked as "handled". // -// ---------------------------- | ---------------------------- | ---------------------------- +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| -// ^ | _ | -// ---------------------------- | ---------------------------- | ---------------------------- +// +// ^ | _ | +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 2 |_______X________X______X___| | |_______X_______________X___| | |_______X_______________X___| -// ^ | ^ | -// ---------------------------- | ---------------------------- | ---------------------------- +// +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| -// ^ | ^ | -// ---------------------------- | ---------------------------- | ---------------------------- +// +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| -// ^ | _ | +// +// ^ | _ | +// // ------------------------------------------------------------------------------------------------------------------------ -// ---------------------------- | ---------------------------- | ---------------------------- +// +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| -// _ | ^ | _ -// ---------------------------- | ---------------------------- | ---------------------------- +// +// _ | ^ | _ +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 2 |_______X_______________X___| | |_______X_______________X___| | |_______X_______________X___| -// _ | ^ | ^ -// ---------------------------- | ---------------------------- | ---------------------------- +// +// _ | ^ | ^ +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| -// _ | | -// ---------------------------- | ---------------------------- | ---------------------------- +// +// _ | | +// ---------------------------- | ---------------------------- | ---------------------------- +// // Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| -// _ +// +// _ func (b *Bitmap) unionInPlace(others ...*Bitmap) { const staticSize = 20 var ( @@ -1823,10 +1841,7 @@ type containerIteratorRoaringIteratorWrapper struct { func (c *containerIteratorRoaringIteratorWrapper) Next() bool { c.nextKey, c.nextCont = c.r.NextContainer() - if c.nextCont == nil { - return false - } - return true + return c.nextCont != nil } func (c *containerIteratorRoaringIteratorWrapper) Value() (uint64, *Container) { @@ -6744,7 +6759,7 @@ func xorCompare(x *xorstm) (r1 Interval16, hasData bool) { return r1, hasData } -//stm is state machine used to "xor" iterate over runs. +// stm is state machine used to "xor" iterate over runs. type xorstm struct { vaValid, vbValid bool va, vb Interval16 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 3d08ee833..95757bfb3 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -6,8 +6,8 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" "math/rand" + "os" "reflect" "runtime" "strings" @@ -2022,7 +2022,7 @@ func TestXorArrayRun(t *testing.T) { } -//special case that didn't fit the xorrunrun table testing below. +// special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { a := NewContainerRun([]Interval16{{Start: 4, Last: 10}}) b := NewContainerRun([]Interval16{{Start: 5, Last: 10}}) @@ -3779,7 +3779,7 @@ func TestContainerCombinations(t *testing.T) { } } -//func getFunc(func(a, b *container) *container, m, n *container) *container { +// func getFunc(func(a, b *container) *container, m, n *container) *container { func runContainerFunc(f interface{}, c ...*Container) *Container { switch f := f.(type) { case func(*Container) *Container: @@ -3822,7 +3822,7 @@ func TestUnmarshalRoaringWithNoErrors(t *testing.T) { t.Fatalf("hex decode %s", err) } } else { - testContainer, _ = ioutil.ReadFile(testCase.roaringFileName) + testContainer, _ = os.ReadFile(testCase.roaringFileName) } bm := NewBitmap() err = bm.UnmarshalBinary(testContainer) @@ -3890,24 +3890,25 @@ func newTestBitmapContainer() *Container { /* // This function exercises an arcane edge case in dead code. // It doesn't need to be run right now. -func TestEquals(t *testing.T) { - bma := NewBitmap() - bmr := NewBitmap() - for i := uint64(0); i < 30; i++ { - bma.Add(i) - bmr.Add(i) + + func TestEquals(t *testing.T) { + bma := NewBitmap() + bmr := NewBitmap() + for i := uint64(0); i < 30; i++ { + bma.Add(i) + bmr.Add(i) + } + bmr.Optimize() + bmi := bma.Intersect(bmr) + err := bitmapsEqual(bmi, bma) + if err != nil { + t.Fatalf("expected intersection to equal array") + } + err = bitmapsEqual(bmi, bmr) + if err != nil { + t.Fatalf("expected intersection to equal run") + } } - bmr.Optimize() - bmi := bma.Intersect(bmr) - err := bitmapsEqual(bmi, bma) - if err != nil { - t.Fatalf("expected intersection to equal array") - } - err = bitmapsEqual(bmi, bmr) - if err != nil { - t.Fatalf("expected intersection to equal run") - } -} */ func TestShiftArray(t *testing.T) { tests := []struct { @@ -4564,7 +4565,6 @@ func TestRoaringIteratorSkip(t *testing.T) { // large an run container, which was causing problems when // we write to the transactional backends. Verify that // unionRunRunInPlace() converts to bitmap if its too large. -// func TestContainer_unionRunRunInPlace_TwoBigRunArrays(t *testing.T) { a := NewContainerRun(nil) diff --git a/server/grpc.go b/server/grpc.go index 15f59c173..a475a1bdd 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/improbable-eng/grpc-web/go/grpcweb" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/authz" @@ -24,6 +23,7 @@ import ( vdsm_pb "github.com/featurebasedb/featurebase/v3/proto/vdsm" "github.com/featurebasedb/featurebase/v3/sql" "github.com/featurebasedb/featurebase/v3/stats" + "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -1674,7 +1674,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { // LogQuery logs requests func LogQuery(ctx context.Context, method string, req interface{}, logger logger.Logger) { - uinfo, ok := ctx.Value("userinfo").(*authn.UserInfo) + uinfo, _ := ctx.Value("userinfo").(*authn.UserInfo) md, _ := metadata.FromIncomingContext(ctx) p, ok := peer.FromContext(ctx) ip := "" diff --git a/server/grpc_test.go b/server/grpc_test.go index 082be2e80..eab8f0998 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -16,7 +16,6 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/authn" "github.com/featurebasedb/featurebase/v3/authz" @@ -27,7 +26,9 @@ import ( "github.com/featurebasedb/featurebase/v3/sql" "github.com/featurebasedb/featurebase/v3/test" "github.com/featurebasedb/featurebase/v3/vprint" + "github.com/golang-jwt/jwt" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -1894,7 +1895,8 @@ func writeTestFile(t *testing.T, filename, content string) string { if err != nil { t.Fatal(err) } - io.WriteString(f, content) + _, err = io.WriteString(f, content) + assert.NoError(t, err) if err := f.Close(); err != nil { t.Fatal(err) } @@ -1919,11 +1921,7 @@ func Test_ChainUnaryInterceptor(t *testing.T) { interceptors0 := []grpc.UnaryServerInterceptor{} interceptors1 := []grpc.UnaryServerInterceptor{salt} interceptors2 := []grpc.UnaryServerInterceptor{salt, pepper} - // interceptors5 := []grpc.UnaryServerInterceptor{interceptor, interceptor, interceptor, interceptor, interceptor} - type args struct { - interceptors []grpc.UnaryServerInterceptor - } tests := []struct { name string interceptors []grpc.UnaryServerInterceptor @@ -2001,13 +1999,15 @@ func Test_ChainStreamInterceptor(t *testing.T) { salt := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { md := fromIncomingContext(ss.Context()) md.Append("ingredient", "with salt") - ss.SetHeader(md) + err := ss.SetHeader(md) + assert.NoError(t, err) return handler(srv, ss) } pepper := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { md := fromIncomingContext(ss.Context()) md.Append("ingredient", "and pepper") - ss.SetHeader(md) + err := ss.SetHeader(md) + assert.NoError(t, err) return handler(srv, ss) } @@ -2015,9 +2015,6 @@ func Test_ChainStreamInterceptor(t *testing.T) { interceptors1 := []grpc.StreamServerInterceptor{salt} interceptors2 := []grpc.StreamServerInterceptor{salt, pepper} - type args struct { - interceptors []grpc.StreamServerInterceptor - } tests := []struct { name string interceptors []grpc.StreamServerInterceptor @@ -2042,7 +2039,8 @@ func Test_ChainStreamInterceptor(t *testing.T) { } t.Run(tt.name, func(t *testing.T) { chained := server.ChainStreamInterceptors(tt.interceptors...) - chained(srv, ss, info, handler) + err := chained(srv, ss, info, handler) + assert.NoError(t, err) if !reflect.DeepEqual(result, tt.want) { t.Errorf("ChainStreamInterceptor() = %v, want %v", result, tt.want) } diff --git a/server/handler_test.go b/server/handler_test.go index 24ef4d82a..373f08000 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math" gohttp "net/http" "net/http/httptest" @@ -37,7 +36,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) if w.Code != gohttp.StatusNoContent { - bod, err := ioutil.ReadAll(w.Result().Body) + bod, err := io.ReadAll(w.Result().Body) if err != nil { t.Errorf("reading body: %v", err) } @@ -113,7 +112,7 @@ func TestHandler_Endpoints(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`))) if w.Code != gohttp.StatusNoContent { - bod, err := ioutil.ReadAll(w.Result().Body) + bod, err := io.ReadAll(w.Result().Body) if err != nil { t.Errorf("reading body: %v", err) } @@ -1414,94 +1413,6 @@ func TestClusterTranslator(t *testing.T) { } } -// func TestQueryHistory(t *testing.T) { -// cluster := test.MustRunCluster(t, 3, -// []server.CommandOption{ -// server.OptCommandServerOptions( -// pilosa.OptServerNodeID("1"), -// )}, -// []server.CommandOption{ -// server.OptCommandServerOptions( -// pilosa.OptServerNodeID("0"), -// )}, -// []server.CommandOption{ -// server.OptCommandServerOptions( -// pilosa.OptServerNodeID("2"), -// )}, -// ) -// defer cluster.Close() - -// cmd := cluster.GetNode(0) -// h := cmd.Handler.(*pilosa.Handler).Handler - -// w := httptest.NewRecorder() - -// test.Do(t, "POST", cmd.URL()+"/index/i0", "") -// test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "") - -// gh := server.NewGRPCHandler(cmd.API) -// stream := &MockServerTransportStream{} -// ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream) -// _, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{ -// Sql: `select * from i0`, -// }) - -// if err != nil { -// t.Fatalf("QuerySQLUnary failed: %v", err) -// } - -// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)") -// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)") -// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)") - -// h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil)) -// if w.Code != gohttp.StatusOK { -// t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) -// } - -// ret := make([]pilosa.PastQueryStatus, 4) -// b, err := ioutil.ReadAll(w.Body) -// if err != nil { -// t.Fatalf("reading: %v", err) -// } -// err = json.Unmarshal(b, &ret) -// if err != nil { -// t.Fatalf("unmarshalling: %v", err) -// } - -// // verify result length -// if len(ret) != 4 { -// // each set query executes on both nodes once -// // topn query gets added to history on node0 once, node1 twice -// t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret) -// } - -// // verify sort order -// if !sort.SliceIsSorted(ret, func(i, j int) bool { -// // must match the sort in api.PastQueries -// return ret[i].Start.After(ret[j].Start) -// }) { -// t.Fatalf("response list not sorted correctly") -// } - -// // verify some response values -// if ret[0].Index != "i0" { -// t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index) -// } -// if ret[0].Node != cluster.GetNode(0).Server.NodeID() { -// t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID()) -// } -// if ret[3].PQL != "Extract(All(),Rows(f0))" { -// t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL) -// } -// if ret[3].SQL != "select * from i0" { -// t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL) -// } -// if ret[0].PQL != "TopN(f0)" { -// t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL) -// } -// } - func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { dec := json.NewDecoder(r) err := dec.Decode(&ret) diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go index d52b4e21a..9ae63e353 100644 --- a/server/pg_internal_test.go +++ b/server/pg_internal_test.go @@ -7,6 +7,7 @@ import ( pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/pg" + "github.com/stretchr/testify/assert" ) // pg_internal_test.go tests unexported methods from server/pg.go @@ -35,7 +36,9 @@ func (t *TestQueryResultWriter) Tag(tag string) { func TestPgWriteDistinctTimestamp(t *testing.T) { w := TestQueryResultWriter{} expected := pilosa.DistinctTimestamp{Name: "test", Values: []string{"date1", "date2", "date3"}} - pgWriteDistinctTimestamp(&w, expected) + err := pgWriteDistinctTimestamp(&w, expected) + assert.NoError(t, err) + if w.Header[0].Name != expected.Name { t.Fatalf("Header Name is wrong. got %v, want %v", w.Header[0], expected.Name) } diff --git a/server/server.go b/server/server.go index b8b9ca81e..5bd40c238 100644 --- a/server/server.go +++ b/server/server.go @@ -13,7 +13,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "log" "math/rand" "net" @@ -189,7 +188,7 @@ func (m *Command) doSetupResourceLimits() error { } // We don't have corresponding options for non-Linux right now, but probably should. if runtime.GOOS == "linux" { - result, err := ioutil.ReadFile("/proc/sys/vm/max_map_count") + result, err := os.ReadFile("/proc/sys/vm/max_map_count") if err != nil { m.logger.Infof("Tried unsuccessfully to check system mmap limit: %w", err) } else { diff --git a/server/server_test.go b/server/server_test.go index e06955884..61997719c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -8,10 +8,10 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "math/rand" "net" nethttp "net/http" + "os" "reflect" "sort" "strings" @@ -698,7 +698,7 @@ func TestMain_ImportTimestamp(t *testing.T) { } // Ensure the correct views were created. dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName) - files, err := ioutil.ReadDir(dir) + files, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } @@ -754,7 +754,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { // Ensure the correct views were created. dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName) - files, err := ioutil.ReadDir(dir) + files, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } diff --git a/server/tlsconfig.go b/server/tlsconfig.go index ab9f3de55..d0b3900fb 100644 --- a/server/tlsconfig.go +++ b/server/tlsconfig.go @@ -38,7 +38,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "os" "os/signal" "sync" @@ -136,7 +135,7 @@ func GetTLSConfig(tlsConfig *TLSConfig, logger logger.Logger) (TLSConfig *tls.Co } if hasCA { - b, err := ioutil.ReadFile(tlsConfig.CACertPath) + b, err := os.ReadFile(tlsConfig.CACertPath) if err != nil { return nil, errors.Wrap(err, "loading tls ca key") } diff --git a/test/pilosa.go b/test/pilosa.go index 48a6cebd6..d8515faa3 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -6,7 +6,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" gohttp "net/http" "os" "reflect" @@ -21,7 +21,7 @@ import ( "github.com/featurebasedb/featurebase/v3/testhook" ) -//////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////// // Command represents a test wrapper for server.Command. type Command struct { *server.Command @@ -52,7 +52,7 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command { }, opts...) m := &Command{commandOptions: opts} - m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) + m.Command = server.NewCommand(bytes.NewReader(nil), io.Discard, io.Discard, opts...) // pick etcd ports using a socket rather than a real port err = GetPortsGenConfigs(tb, []*Command{m}) if err != nil { @@ -114,7 +114,7 @@ func (m *Command) Reopen() error { // Create new main with the same config. config := m.Command.Config - m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...) + m.Command = server.NewCommand(bytes.NewReader(nil), io.Discard, io.Discard, m.commandOptions...) m.Command.Config = config // Run new program. @@ -240,7 +240,7 @@ func (m *Command) QueryProtobuf(indexName string, query string) (*pilosa.QueryRe } defer resp.Body.Close() - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -291,7 +291,7 @@ func Do(t testing.TB, method, urlStr string, body string) *httpResponse { } defer resp.Body.Close() - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) } diff --git a/testhook/hook.go b/testhook/hook.go index 4d12cfa33..6ca2aacb8 100644 --- a/testhook/hook.go +++ b/testhook/hook.go @@ -4,7 +4,6 @@ package testhook import ( "fmt" - "io/ioutil" "os" "sync" "testing" @@ -71,7 +70,7 @@ func RunTestsWithHooks(m *testing.M) { // TempDir creates a temp directory that will be automatically deleted when // this test completes, using go1.14's [TB].Cleanup() if available. func TempDir(tb testing.TB, pattern string) (path string, err error) { - path, err = ioutil.TempDir("", pattern) + path, err = os.MkdirTemp("", pattern) if err == nil { Cleanup(tb, func() { os.RemoveAll(path) @@ -83,7 +82,7 @@ func TempDir(tb testing.TB, pattern string) (path string, err error) { // TempFile creates a temp file that will be automatically deleted when // this test completes, using go1.14's [TB].Cleanup() if available. func TempFile(tb testing.TB, pattern string) (file *os.File, err error) { - file, err = ioutil.TempFile("", pattern) + file, err = os.CreateTemp("", pattern) if err == nil { path := file.Name() Cleanup(tb, func() { @@ -99,7 +98,7 @@ func TempFile(tb testing.TB, pattern string) (file *os.File, err error) { // path instead of the default Go TMPDIR. Only some tests use this, which is // possibly an error... func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err error) { - path, err = ioutil.TempDir(dir, pattern) + path, err = os.MkdirTemp(dir, pattern) if err == nil { Cleanup(tb, func() { os.RemoveAll(path) @@ -113,7 +112,7 @@ func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err e // path instead of the default Go TMPDIR. Only some tests use this, which is // possibly an error... func TempFileInDir(tb testing.TB, dir string, pattern string) (file *os.File, err error) { - file, err = ioutil.TempFile(dir, pattern) + file, err = os.CreateTemp(dir, pattern) if err == nil { path := file.Name() Cleanup(tb, func() { diff --git a/translate.go b/translate.go index c40ffb332..560557d0c 100644 --- a/translate.go +++ b/translate.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "sort" "sync" @@ -35,13 +34,14 @@ var ( // TranslateStore is the storage for translation string-to-uint64 values. // For BoltDB implementation an empty string will be converted into the sentinel byte slice: -// var emptyKey = []byte{ -// 0x00, 0x00, 0x00, -// 0x4d, 0x54, 0x4d, 0x54, // MTMT -// 0x00, -// 0xc2, 0xa0, // NO-BREAK SPACE -// 0x00, -// } +// +// var emptyKey = []byte{ +// 0x00, 0x00, 0x00, +// 0x4d, 0x54, 0x4d, 0x54, // MTMT +// 0x00, +// 0xc2, 0xa0, // NO-BREAK SPACE +// 0x00, +// } type TranslateStore interface { // TODO: refactor this interface; readonly should be part of the type and replication should be an impl detail io.Closer @@ -585,7 +585,7 @@ func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error) { s.mu.Lock() defer s.mu.Unlock() var bytes []byte - bytes, err = ioutil.ReadAll(r) + bytes, err = io.ReadAll(r) count = int64(len(bytes)) if err != nil { return count, err diff --git a/txfactory.go b/txfactory.go index 5be1d2d82..4d4879a8a 100644 --- a/txfactory.go +++ b/txfactory.go @@ -29,7 +29,6 @@ const ( // with -2 when the Tx completes. // // Should be false for production. -// const DetectMemAccessPastTx = false var sep = string(os.PathSeparator) @@ -43,11 +42,11 @@ var sep = string(os.PathSeparator) // The most common use of Qcx is to call GetTx() to obtain a Tx locally, // once the index/shard pair is known: // -// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) { -// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...}) -// defer finisher(&err0) -// ... -// } +// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) { +// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...}) +// defer finisher(&err0) +// ... +// } // // Qcx reuses read-only Tx on the same index/shard pair. See // the Qcx.GetTx() for further discussion. The caveat is of @@ -82,7 +81,6 @@ var sep = string(os.PathSeparator) // This is then committed at the final, top-level, Qcx.Finish() call. // // See also the Qcx.GetTx() example and the TxGroup description below. -// type Qcx struct { Grp *TxGroup Txf *TxFactory @@ -203,12 +201,12 @@ var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset // // Note we are tracking the returned err0 error value of someFunc(). An option instead is to say // -// defer finisher(nil) +// defer finisher(nil) // // This means always Commit writes, ignoring if there were errors. This style // is expected to be rare compared to the typical // -// defer finisher(&err0) +// defer finisher(&err0) // // invocation, where err0 is your return from the enclosing function error. // If the Tx is local and not a part of a group, then the finisher @@ -223,7 +221,6 @@ var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset // locally by another _, err := f() call. For this reason, it can // be clearer (and much safer) to rename the enclosing functions 'err' to 'err0', // to make it clear we are referring to the first and final error. -// func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { if qcx.workers != nil { qcx.workers.Block() @@ -665,14 +662,6 @@ func dirExists(name string) bool { return false } -func fileSize(name string) (int64, error) { - fi, err := os.Stat(name) - if err != nil { - return -1, err - } - return fi.Size(), nil -} - var _ = anyGlobalDBWrappersStillOpen // happy linter func anyGlobalDBWrappersStillOpen() bool {