scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: updating code to meet linting requirements (#2171)

Browse source 
* removes unused filesize function

* removes ioutil usage

* updates ioutil.ReadAll to io.ReadAll

* updates ioutil.TempFile to os.CreateTemp

* updates ioutil.TempDir to os.MkdirTemp

* updates ioutil.ReadAll to os.ReadAll

* update ioutil.WriteFile to os.WriteFile

* updates ioutil.Discard to io.Discard

* updates ioutil.ReadDir to os.ReadDir where applicable

* removes unused code in idk

* creates type to use for context value keys

* replaces assert.Nil with assert.NoError for error checks
This commit is contained in:
CLoZengineer 2022-09-29 12:34:29 -04:00 • committed by GitHub
parent 51249cda78
commit f9ddb5d5c1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
85 changed files with 363 additions and 529 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
api.go
View file

@ -12,7 +12,6 @@ import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"math"
"net/url"
"os"
@ -27,7 +26,6 @@ import (
"github.com/featurebasedb/featurebase/v3/ingest"
"github.com/featurebasedb/featurebase/v3/rbf"
//"github.com/featurebasedb/featurebase/v3/pg"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/stats"
@ -803,7 +801,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) (_ []byte
return nil, errors.Wrap(err, "validating api method")
}
reqBytes, err := ioutil.ReadAll(body)
reqBytes, err := io.ReadAll(body)
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read body error"))
}
@ -1016,7 +1014,7 @@ func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error {
}
// Read entire body.
body, err := ioutil.ReadAll(reqBody)
body, err := io.ReadAll(reqBody)
if err != nil {
return errors.Wrap(err, "reading body")
}
@ -2398,7 +2396,7 @@ func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []u
// ErrTranslatingKeyNotFound error will be swallowed here, so the empty response will be returned.
func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error) {
var req TranslateKeysRequest
buf, err := ioutil.ReadAll(r)
buf, err := io.ReadAll(r)
if err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read translate keys request error"))
} else if err := api.Serializer.Unmarshal(buf, &req); err != nil {
@ -2435,7 +2433,7 @@ func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err e
// TranslateIDs handles a TranslateIDRequest.
func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error) {
var req TranslateIDsRequest
if buf, err := ioutil.ReadAll(r); err != nil {
if buf, err := io.ReadAll(r); err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "read translate ids request error"))
} else if err := api.Serializer.Unmarshal(buf, &req); err != nil {
return nil, NewBadRequestError(errors.Wrap(err, "unmarshal translate ids request error"))
@ -2884,14 +2882,15 @@ func (api *API) MutexCheckNode(ctx context.Context, qcx *Qcx, indexName string,
// MutexCheck checks a named field for mutex violations, returning a
// map of record IDs to values for records that have multiple values in the
// field. The return will be one of:
// details true:
// map[uint64][]uint64 // unkeyed index, unkeyed field
// map[uint64][]string // unkeyed index, keyed field
// map[string][]uint64 // keyed index, unkeyed field
// map[string][]string // keyed index, keyed field
// details false:
// []uint64 // unkeyed index
// []string // keyed index
//
// details true:
// map[uint64][]uint64 // unkeyed index, unkeyed field
// map[uint64][]string // unkeyed index, keyed field
// map[string][]uint64 // keyed index, unkeyed field
// map[string][]string // keyed index, keyed field
// details false:
// []uint64 // unkeyed index
// []string // keyed index
func (api *API) MutexCheck(ctx context.Context, qcx *Qcx, indexName string, fieldName string, details bool, limit int) (result interface{}, err error) {
if err = api.validate(apiMutexCheck); err != nil {
return nil, errors.Wrap(err, "validating api method")

13
authn/authenticate.go
View file

@ -25,6 +25,9 @@ import (
"golang.org/x/oauth2"
)
// AuthContextKey is a unique type to prevent collisions when using context.WithValue()
type AuthContextKey string
const (
// AccessCookieName is the name of the cookie that holds the access token.
AccessCookieName = "molecula-chip"
@ -36,10 +39,10 @@ const (
RefreshHeaderName = "X-Molecula-Refresh-Token"
// ContextValueAccessToken is the key used to set AccessTokens in a ctx.
ContextValueAccessToken = "Access"
ContextValueAccessToken = AuthContextKey("Access")
// ContextValueRefreshToken is the key used to set RefreshTokens in a ctx.
ContextValueRefreshToken = "Refresh"
ContextValueRefreshToken = AuthContextKey("Refresh")
)
// cachedGroups is used to hold groups and when they were last cached
@ -171,7 +174,7 @@ func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
// it is caller's responsibility to inform the user that the access token has been refreshed
func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
// clean up the cache every 30 minutes or so
if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
if time.Since(a.lastCacheClean) >= 30*time.Minute {
a.cleanCache()
}
@ -237,7 +240,7 @@ func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
func (a *Auth) cleanCache() {
for access, tkn := range a.groupsCache {
// if it's been more than 24 hours since the groups were cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
if time.Since(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.groupsCache, access)
}
@ -300,7 +303,7 @@ func (a *Auth) getGroups(token string) ([]Group, error) {
var groups Groups
gc, ok := a.groupsCache[token]
if ok && (time.Now().Sub(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
if ok && (time.Since(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
return gc.groups, nil
}

7
authn/authenticate_internal_test.go
View file

@ -16,8 +16,8 @@ import (
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/golang-jwt/jwt"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
@ -67,7 +67,7 @@ func TestSetGRPCMetadata(t *testing.T) {
"otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}},
} {
t.Run(name, func(t *testing.T) {
ogCookies, _ := md["cookie"]
ogCookies := md["cookie"]
ctx := grpc.NewContextWithServerTransportStream(
metadata.NewIncomingContext(context.TODO(),
md,
@ -280,8 +280,7 @@ func TestAuthenticate(t *testing.T) {
a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
}
if test.refresh {
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
t.Fatalf("unexpected error: %v", err)
}

3
authz/authorization.go
View file

@ -6,7 +6,6 @@ package authz
import (
"fmt"
"io"
"io/ioutil"
"github.com/featurebasedb/featurebase/v3/authn"
@ -43,7 +42,7 @@ func (p Permission) Satisfies(b Permission) bool {
}
func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
permsData, err := ioutil.ReadAll(permsFile)
permsData, err := io.ReadAll(permsFile)
if err != nil {
return fmt.Errorf("reading permissions failed with error: %s", err)

15
client/batch_test.go
View file

@ -14,6 +14,7 @@ import (
featurebase "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/stretchr/testify/assert"
"github.com/pkg/errors"
)
@ -1662,8 +1663,7 @@ func mutexClearRegression(t *testing.T, c *test.Cluster, client *Client) {
t.Fatalf("getting batch: %v", err)
}
col := uint64(0)
row := uint64(1)
var col, row uint64
for i := uint64(0); i <= 21; i++ {
col = (i%2+1)*featurebase.ShardWidth + i%5
row = i % 3
@ -1720,8 +1720,7 @@ func mutexNilClearID(t *testing.T, c *test.Cluster, client *Client) {
t.Fatalf("getting batch: %v", err)
}
col := uint64(0)
row := uint64(1)
var col, row uint64
// populate mutex with some data
for i := uint64(0); i < 11; i++ {
col = (i%2+1)*featurebase.ShardWidth + i%5
@ -1746,14 +1745,17 @@ func mutexNilClearID(t *testing.T, c *test.Cluster, client *Client) {
}
items := resp.Result().Row().Columns
// delete item 0
b.Add(
err = b.Add(
Row{
ID: items[0],
Values: []interface{}{nil},
Clears: map[int]interface{}{0: nil},
},
)
b.Import()
assert.NoError(t, err)
err = b.Import()
assert.NoError(t, err)
items = items[1:]
// confirm record removed
resp, err = client.Query(idx.RawQuery("Row(mut=0)"))
@ -1820,6 +1822,7 @@ func mutexNilClearKey(t *testing.T, c *test.Cluster, client *Client) {
t.Fatalf("importing: %v", err)
}
resp, err := client.Query(idx.RawQuery(`Row(mut="a")`))
assert.NoError(t, err)
errorIfNotEqual(t, resp.Result().Row().Keys, []string{"0", "2"})
r.ID = "2"

4
client/client.go
View file

@ -20,7 +20,6 @@ import (
"sync"
"time"
"github.com/golang/protobuf/proto" //nolint:staticcheck
pilosa "github.com/featurebasedb/featurebase/v3"
fbproto "github.com/featurebasedb/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import
"github.com/featurebasedb/featurebase/v3/logger"
@ -29,6 +28,7 @@ import (
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/stats"
"github.com/golang/protobuf/proto" //nolint:staticcheck
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -40,8 +40,6 @@ const PQLVersion = "1.0"
// DefaultShardWidth is used if an index doesn't have it defined.
const DefaultShardWidth = pilosa.ShardWidth
const maxHosts = 10
// Client is the HTTP client for Pilosa server.
type Client struct {
cluster *Cluster

6
client/client_it_test.go
View file

@ -5,7 +5,7 @@ package client
import (
"bytes"
"fmt"
"io/ioutil"
"io"
"testing"
"time"
@ -671,7 +671,7 @@ func TestClientAgainstCluster(t *testing.T) {
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := ioutil.ReadAll(r)
b, err := io.ReadAll(r)
require.NoError(t, err)
target := "1,1\n1,10\n2,1048577\n"
@ -696,7 +696,7 @@ func TestClientAgainstCluster(t *testing.T) {
r, err := cli.ExportField(testFieldExport)
require.NoErrorf(t, err, "ExportField")
b, err := ioutil.ReadAll(r)
b, err := io.ReadAll(r)
require.NoError(t, err)
target := "1,one\n1,ten\n2,big-number\n"

4
client/ingest_api_batch_test.go
View file

@ -7,6 +7,7 @@ import (
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/stretchr/testify/assert"
)
func TestIngestAPIBatchAdd(t *testing.T) {
@ -148,7 +149,7 @@ func TestIngestAPIBatch(t *testing.T) {
}
defer cli.Close()
cli.IngestSchema(map[string]interface{}{
_, err = cli.IngestSchema(map[string]interface{}{
"index-name": "test-1",
"index-action": "create",
"primary-key-type": "uint",
@ -197,6 +198,7 @@ func TestIngestAPIBatch(t *testing.T) {
},
},
})
assert.NoError(t, err)
schema, err := cli.Schema()
if err != nil {

4
client/logimport_test.go
View file

@ -5,7 +5,6 @@ package client
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
@ -93,7 +92,8 @@ func TestEncodeDecode(t *testing.T) {
})
}
buf, err := ioutil.TempFile("", "")
buf, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("getting temp file: %v", err)
}

40
cluster_internal_test.go
View file

@ -14,47 +14,9 @@ import (
"github.com/featurebasedb/featurebase/v3/disco"
pnet "github.com/featurebasedb/featurebase/v3/net"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/testhook"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
_ "github.com/featurebasedb/featurebase/v3/vprint"
)
// newHolderWithTempPath returns a new instance of Holder.
func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-holder-")
if err != nil {
panic(err)
}
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = backend
h := NewHolder(path, cfg)
PanicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
})
return h
}
// newIndexWithTempPath returns a new instance of Index.
func newIndexWithTempPath(tb testing.TB, name string) *Index {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
if err != nil {
panic(err)
}
cfg := DefaultHolderConfig()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := NewHolder(path, cfg)
PanicOn(h.Open())
index, err := h.CreateIndex(name, IndexOptions{})
testhook.Cleanup(tb, func() {
h.Close()
})
if err != nil {
panic(err)
}
return index
}
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := cluster{

3
cmd/badloader/badloader.go
View file

@ -10,7 +10,6 @@ import (
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
pilosa "github.com/featurebasedb/featurebase/v3"
@ -92,7 +91,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
}
}
roaringData, err := ioutil.ReadAll(tarReader)
roaringData, err := io.ReadAll(tarReader)
if err != nil {
return err
}

4
cmd/pilosa-bench/main.go
View file

@ -7,7 +7,7 @@ import (
"expvar"
"flag"
"fmt"
"io/ioutil"
"io"
"log"
"math/rand"
"net/http"
@ -72,7 +72,7 @@ func run(ctx context.Context, args []string) (err error) {
// Clear time prefix on log.
log.SetFlags(0)
if !*verbose {
log.SetOutput(ioutil.Discard)
log.SetOutput(io.Discard)
}
// Setup PRNG to have consistent values for the same set of data.

9
cmd/roaring-migrate/main.go
View file

@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
@ -146,7 +145,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
t := strings.Split(pathX, "/")
index := t[1]
src := dataDir + pathX
content, err := ioutil.ReadFile(src)
content, err := os.ReadFile(src)
if err != nil {
return err
}
@ -198,7 +197,7 @@ func Extract(filename string) (index, field, view string, shard uint64) {
return parts[1], parts[2], parts[4], shard
}
//just a way to collect all the open dbs
// just a way to collect all the open dbs
type rbfFile struct {
working *rbf.DB
last string
@ -294,7 +293,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
return err
}
err = ioutil.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644)
err = os.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644)
if err != nil {
return err
}
@ -342,7 +341,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
if verbose {
glogger.Infof("processing: %v", dataDir+filename)
}
content, err := ioutil.ReadFile(dataDir + filename)
content, err := os.ReadFile(dataDir + filename)
if err != nil {
return err
}

3
cmd/roaring-migrate/main_test.go
View file

@ -1,7 +1,6 @@
package main
import (
"io/ioutil"
"os"
"testing"
)
@ -39,7 +38,7 @@ func TestMainProgram(t *testing.T) {
if realMain() == 0 {
t.Fatal("should fail and it succeeded")
}
dir, err := ioutil.TempDir("", "backup")
dir, err := os.MkdirTemp("", "backup")
if err != nil {
t.Fatal(err)
}

5
cmd/root_test.go
View file

@ -5,7 +5,6 @@ package cmd_test
import (
"fmt"
"io"
"io/ioutil"
"os"
"reflect"
"strings"
@ -34,7 +33,7 @@ func tExec(t *testing.T, cmd *cobra.Command, out io.Reader, w io.WriteCloser) (o
done := make(chan struct{})
var readErr error
go func() {
output, readErr = ioutil.ReadAll(out)
output, readErr = io.ReadAll(out)
close(done)
}()
err = cmd.Execute()
@ -143,7 +142,7 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command {
os.Setenv("PILOSA_POSTGRES_BIND", "")
// make command and set args
rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard)
rc := cmd.NewRootCommand(strings.NewReader(""), io.Discard, io.Discard)
rc.SetArgs(ct.args)
err = cfgFile.Close()

7
cmd/slurp/slurp.go
View file

@ -10,7 +10,6 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"net/url"
"os"
@ -55,7 +54,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
return err
}
}
roaringData, err := ioutil.ReadAll(tr)
roaringData, err := io.ReadAll(tr)
if err != nil {
return err
}
@ -91,7 +90,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
}
byteData, err := ioutil.ReadAll(tr)
byteData, err := io.ReadAll(tr)
vprint.PanicOn(err)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil
@ -107,7 +106,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
if err != nil {
return err
}
byteData, err := ioutil.ReadAll(tr)
byteData, err := io.ReadAll(tr)
vprint.PanicOn(err)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil

2
ctl/auth_token.go
View file

@ -169,7 +169,7 @@ func (cmd *AuthTokenCommand) Run(ctx context.Context) (err error) {
}
// Prompt the user to visit verification_uri and enter code.
fmt.Printf(formatPromptBox(dar.VerificationURI, dar.UserCode))
fmt.Print(formatPromptBox(dar.VerificationURI, dar.UserCode))
// Request a token until success or error response, slowing down if requested.
interval := dar.Interval

3
ctl/backup.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
@ -194,7 +193,7 @@ func (cmd *BackupCommand) backupSchema(ctx context.Context, schema *pilosa.Schem
return fmt.Errorf("marshaling schema: %w", err)
}
if err := ioutil.WriteFile(filepath.Join(cmd.OutputDir, "schema"), buf, 0600); err != nil {
if err := os.WriteFile(filepath.Join(cmd.OutputDir, "schema"), buf, 0600); err != nil {
return fmt.Errorf("writing schema: %w", err)
}

5
ctl/import_test.go
View file

@ -9,7 +9,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
@ -18,13 +17,13 @@ import (
"testing"
"time"
"github.com/golang-jwt/jwt"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/testhook"
"github.com/golang-jwt/jwt"
)
func TestImportCommand_Validation(t *testing.T) {
@ -305,7 +304,7 @@ func TestImportCommand_KeyReplication(t *testing.T) {
// Read body and unmarshal response.
exp := `{"results":[1000]}` + "\n"
if body, err := ioutil.ReadAll(resp.Body); err != nil {
if body, err := io.ReadAll(resp.Body); err != nil {
return fmt.Errorf("reading: %s", err)
} else if !reflect.DeepEqual(body, []byte(exp)) {
return fmt.Errorf("expected: %s, but got: %s", exp, body)

4
ctl/server.go
View file

@ -57,9 +57,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.Int64Var(&srv.Config.Etcd.HeartbeatTTL, "etcd.heartbeat-ttl", srv.Config.Etcd.HeartbeatTTL, "Timeout used to determine cluster status")
flags.StringVar(&srv.Config.Etcd.Cluster, "etcd.static-cluster", srv.Config.Etcd.Cluster, "EXPERIMENTAL static featurebase cluster name1=apurl1,name2=apurl2")
flags.MarkHidden("etcd.static-cluster")
_ = flags.MarkHidden("etcd.static-cluster")
flags.StringVar(&srv.Config.Etcd.EtcdHosts, "etcd.etcd-hosts", srv.Config.Etcd.EtcdHosts, "EXPERIMENTAL etcd server host:port comma separated list")
flags.MarkHidden("etcd.etcd-hosts") // TODO (twg) expose when ready for public consumption
_ = flags.MarkHidden("etcd.etcd-hosts") // TODO (twg) expose when ready for public consumption
// External postgres database for ExternalLookup
flags.StringVar(&srv.Config.LookupDBDSN, "lookup-db-dsn", "", "external (postgres) database DSN to use for ExternalLookup calls")

2
etcd/embed.go
View file

@ -371,7 +371,7 @@ func (e *Etcd) parseOptions() (*embed.Config, error) {
if e.options.InitCluster != "" {
// Checks if FB is running the single-node free version or the multi-node
// enterprise version. Sentry.io is enabled on single-node.
if AllowCluster() == false {
if !AllowCluster() {
// %% begin sonarcloud ignore %%
monitor.InitErrorMonitor(e.version)

4
etcd/external.go
View file

@ -49,9 +49,7 @@ func NewExternalEtcd(p *Etcd, o Options) (*ExternalEtcd, error) {
// populate parent.sortedNodes
hosts := strings.Split(o.EtcdHosts, ",")
this.etcdHosts = make([]string, len(hosts))
for i := range hosts {
this.etcdHosts[i] = hosts[i]
}
copy(this.etcdHosts, hosts)
return this, nil
// %% end sonarcloud ignore %%

18
etcd/leasedkv_test.go
View file

@ -12,6 +12,7 @@ import (
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/testhook"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/server/v3/embed"
"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"
@ -41,11 +42,20 @@ func TestClusterKv(t *testing.T) {
t.Fatalf("trying to cause election: %v", err)
}
ctx := context.TODO()
c.nodes[0].SetState(ctx, disco.NodeStateStarting)
c.nodes[1].SetState(ctx, disco.NodeStateStarting)
err = c.nodes[0].SetState(ctx, disco.NodeStateStarting)
assert.NoError(t, err)
err = c.nodes[1].SetState(ctx, disco.NodeStateStarting)
assert.NoError(t, err)
c.MustAwaitClusterState(disco.ClusterStateStarting)
c.nodes[0].SetState(ctx, disco.NodeStateStarted)
c.nodes[1].SetState(ctx, disco.NodeStateStarted)
err = c.nodes[0].SetState(ctx, disco.NodeStateStarted)
assert.NoError(t, err)
err = c.nodes[1].SetState(ctx, disco.NodeStateStarted)
assert.NoError(t, err)
// Two of three nodes are up, one is down, we have 2 replicas, so
// we should be able to handle reads but not writes, so we're in
// a Degraded state.

2
executor_internal_test.go
View file

@ -13,6 +13,7 @@ import (
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/testhook"
"github.com/stretchr/testify/assert"
)
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
@ -592,6 +593,7 @@ func TestExecutor_DeleteRows(t *testing.T) {
}
changed, err = DeleteRows(ctx, row, idx, shard)
assert.NoError(t, err)
if changed {
t.Fatalf("expected delete to not clear bit but it did")
}

12
executor_test.go
View file

@ -12,10 +12,10 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
_ "net/http/pprof"
"os"
"reflect"
"sort"
"strconv"
@ -24,8 +24,6 @@ import (
"time"
"github.com/davecgh/go-spew/spew"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/boltdb"
"github.com/featurebasedb/featurebase/v3/ctl"
@ -36,6 +34,8 @@ import (
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/testhook"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/pkg/errors"
)
@ -6894,7 +6894,7 @@ func TestExecutor_Execute_NoIndex(t *testing.T) {
}
func TestExecutor_Execute_CountDistinct(t *testing.T) {
data, err := ioutil.ReadFile("testdata/schema.json")
data, err := os.ReadFile("testdata/schema.json")
if err != nil {
t.Fatal(err)
}
@ -7127,7 +7127,7 @@ func TestExecutor_BareDistinct(t *testing.T) {
}
func TestExecutor_Execute_TopNDistinct(t *testing.T) {
data, err := ioutil.ReadFile("testdata/schema.json")
data, err := os.ReadFile("testdata/schema.json")
if err != nil {
t.Fatal(err)
}
@ -7202,7 +7202,7 @@ func Test_Executor_Execute_UnionRows(t *testing.T) {
}
func TestTimelessClearRegression(t *testing.T) {
data, err := ioutil.ReadFile("testdata/timeRegressionSchema.json")
data, err := os.ReadFile("testdata/timeRegressionSchema.json")
if err != nil {
t.Fatal(err)
}

4
field_test.go
View file

@ -9,12 +9,12 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/testhook"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
)
@ -302,7 +302,7 @@ func TestFieldInfoMarshal(t *testing.T) {
t.Fatalf("unexpected error marshalling index info, %v", err)
}
expected := []byte(`{"name":"timestamp","createdAt":1649270079233541000,"options":{"type":"timestamp","epoch":"1970-01-01T00:00:00Z","bitDepth":0,"min":-4294967296,"max":4294967296,"timeUnit":"s"}}`)
if bytes.Compare(a, expected) != 0 {
if !bytes.Equal(a, expected) {
t.Fatalf("expected %s, got %s", expected, a)
}
}

19
fragment.go
View file

@ -11,7 +11,6 @@ import (
"fmt"
"hash"
"io"
"io/ioutil"
"math"
"math/bits"
"os"
@ -23,7 +22,6 @@ import (
"time"
"github.com/cespare/xxhash"
"github.com/gogo/protobuf/proto"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
pnet "github.com/featurebasedb/featurebase/v3/net"
@ -35,6 +33,7 @@ import (
"github.com/featurebasedb/featurebase/v3/testhook"
"github.com/featurebasedb/featurebase/v3/tracing"
"github.com/featurebasedb/featurebase/v3/vprint"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
)
@ -250,7 +249,7 @@ func (f *fragment) openCache() error {
// Read cache data from disk.
path := f.cachePath()
buf, err := ioutil.ReadFile(path)
buf, err := os.ReadFile(path)
if os.IsNotExist(err) {
return nil
} else if err != nil {
@ -2612,7 +2611,7 @@ func (f *fragment) flushCache() error {
return errors.Wrap(err, "mkdir")
}
// Write to disk.
if err := ioutil.WriteFile(f.cachePath(), buf, 0600); err != nil {
if err := os.WriteFile(f.cachePath(), buf, 0600); err != nil {
return errors.Wrap(err, "writing")
}
@ -2675,7 +2674,7 @@ func (f *fragment) writeCacheToArchive(tw *tar.Writer) error {
defer f.mu.Unlock()
// Read cache into buffer.
buf, err := ioutil.ReadFile(f.cachePath())
buf, err := os.ReadFile(f.cachePath())
if os.IsNotExist(err) {
return nil
} else if err != nil {
@ -2744,9 +2743,9 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
// this is reading from inside a tarball, so definitely no need
// to close it here.
data, err := ioutil.ReadAll(r)
data, err := io.ReadAll(r)
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)")
return errors.Wrap(err, "fillFragmentFromArchive io.ReadAll(r)")
}
if len(data) == 0 {
return nil
@ -2771,10 +2770,10 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
func (f *fragment) readCacheFromArchive(r io.Reader) error {
// Slurp data from reader and write to disk.
buf, err := ioutil.ReadAll(r)
buf, err := io.ReadAll(r)
if err != nil {
return errors.Wrap(err, "reading")
} else if err := ioutil.WriteFile(f.cachePath(), buf, 0600); err != nil {
} else if err := os.WriteFile(f.cachePath(), buf, 0600); err != nil {
return errors.Wrap(err, "writing")
}
@ -3652,7 +3651,7 @@ func (f *fragment) sortBsiData(tx Tx, filter *Row, bitDepth uint64, sort_desc bo
return nil, err
}
pos := consider.Difference(row)
row, err = f.row(tx, 0)
_, err = f.row(tx, 0)
if err != nil {
return nil, err
}

3
fragment_internal_test.go
View file

@ -8,7 +8,6 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"reflect"
@ -3083,7 +3082,7 @@ func BenchmarkFileWrite(b *testing.B) {
// We're deleting these files as we go because
// otherwise the benchmark could fill up
// $TMPDIR before it finishes running.
f, err := ioutil.TempFile(*TempDir, "")
f, err := os.CreateTemp(*TempDir, "")
if err != nil {
b.Fatalf("getting temp file: %v", err)
}

7
hash/blake3_test.go
View file

@ -5,7 +5,6 @@ package hash
import (
"encoding/hex"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
@ -55,19 +54,19 @@ func TestHashOfDir(t *testing.T) {
}
bmessage := []byte("hello B\n")
if err := ioutil.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil {
if err := os.WriteFile(path.Join(b, "b_content"), bmessage, 0644); err != nil {
t.Fatal(err)
}
cmessage := []byte("hello C\n")
if err := ioutil.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil {
if err := os.WriteFile(path.Join(c, "c_content"), cmessage, 0644); err != nil {
t.Fatal(err)
}
hsh := HashOfDir(dir)
c2message := []byte("hello C2\n")
if err := ioutil.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil {
if err := os.WriteFile(path.Join(c, "c_content"), c2message, 0644); err != nil {
t.Fatal(err)
}

19
http_handler.go
View file

@ -27,9 +27,6 @@ import (
"sync"
"time"
"github.com/felixge/fgprof"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/authz"
"github.com/featurebasedb/featurebase/v3/disco"
@ -40,6 +37,9 @@ import (
"github.com/featurebasedb/featurebase/v3/rbf"
"github.com/featurebasedb/featurebase/v3/storage"
"github.com/featurebasedb/featurebase/v3/tracing"
"github.com/felixge/fgprof"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
@ -703,8 +703,6 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http
return
}
ctx := r.Context()
// check if IP is in allowed networks, if yes give it admin permissions
allowedNetwork, ctx := h.chkAllowedNetworks(r)
if allowedNetwork {
@ -721,9 +719,6 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http
uinfo, err := h.auth.Authenticate(access, refresh)
ctx = context.WithValue(ctx, authn.ContextValueAccessToken, "Bearer "+access)
ctx = context.WithValue(ctx, authn.ContextValueRefreshToken, refresh)
if err != nil {
http.Error(w, errors.Wrap(err, "authenticating").Error(), http.StatusForbidden)
return
@ -1512,7 +1507,7 @@ type postIndexRequest struct {
Options IndexOptions `json:"options"`
}
//_postIndexRequest is necessary to avoid recursion while decoding.
// _postIndexRequest is necessary to avoid recursion while decoding.
type _postIndexRequest postIndexRequest
// Custom Unmarshal JSON to validate request body when creating a new index.
@ -3266,7 +3261,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
ctx := r.Context()
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body")
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
@ -3339,7 +3334,7 @@ func (h *Handler) handlePostShardImportRoaring(w http.ResponseWriter, r *http.Re
ctx := r.Context()
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body")
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
@ -3408,7 +3403,7 @@ func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
span, _ := tracing.StartSpanFromContext(ctx, "io.ReadAll-Body")
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()

10
http_handler_internal_test.go
View file

@ -8,7 +8,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/http/httptest"
"net/url"
@ -18,8 +18,8 @@ import (
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/golang-jwt/jwt"
"golang.org/x/oauth2"
"github.com/featurebasedb/featurebase/v3/authz"
@ -187,7 +187,7 @@ func TestFieldOptionValidation(t *testing.T) {
func readResponse(w *httptest.ResponseRecorder) ([]byte, error) {
res := w.Result()
defer res.Body.Close()
return ioutil.ReadAll(res.Body)
return io.ReadAll(res.Body)
}
// common variables used for testing auth
@ -668,7 +668,7 @@ func TestChkAuthN(t *testing.T) {
expiredToken = "Bearer " + expiredToken
testingHandler := func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("good"))
_, _ = w.Write([]byte("good"))
}
cases := []struct {
@ -704,7 +704,7 @@ func TestChkAuthN(t *testing.T) {
r.Header.Add("Authorization", test.token)
test.handler(w, r)
resp := w.Result()
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
t.Fatal(err)

13
http_handler_test.go
View file

@ -6,7 +6,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net"
"net/http"
gohttp "net/http"
@ -22,6 +22,7 @@ import (
"github.com/featurebasedb/featurebase/v3/encoding/proto"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/stretchr/testify/assert"
)
func TestHandlerOptions(t *testing.T) {
@ -195,7 +196,8 @@ func TestUpdateFieldTTL(t *testing.T) {
if resp.StatusCode == 400 || resp.StatusCode == 404 {
// unmarshal error message to check against expErr
var respBody map[string]interface{}
json.NewDecoder(resp.Body).Decode(&respBody)
err := json.NewDecoder(resp.Body).Decode(&respBody)
assert.NoError(t, err)
errMsg := respBody["error"].(map[string]interface{})["message"].(string)
if !strings.Contains(errMsg, test.expErr) {
@ -300,7 +302,8 @@ func TestUpdateFieldNoStandardView(t *testing.T) {
if resp.StatusCode == 400 {
// unmarshal error message to check against expErr
var respBody map[string]interface{}
json.NewDecoder(resp.Body).Decode(&respBody)
err := json.NewDecoder(resp.Body).Decode(&respBody)
assert.NoError(t, err)
errMsg := respBody["error"].(map[string]interface{})["message"].(string)
if !strings.Contains(errMsg, test.expErr) {
@ -740,7 +743,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
tmpDir := t.TempDir()
permissionsPath := path.Join(tmpDir, "test-permissions.yaml")
err := ioutil.WriteFile(permissionsPath, []byte(permissions1), 0600)
err := os.WriteFile(permissionsPath, []byte(permissions1), 0600)
if err != nil {
t.Fatalf("failed to write permissions file: %v", err)
}
@ -902,7 +905,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
}
if ipTest.StatusCode == 200 {
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("reading resp body :%v", err)
}

3
http_translator.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"reflect"
"sync"
@ -109,7 +108,7 @@ func (r *HTTPTranslateEntryReader) Open() error {
r.body.Close()
return ErrNotImplemented
} else if resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
r.body.Close()
return fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, r.URL, bytes.TrimSpace(body))
}

13
idk/api/source_test.go
View file

@ -2,7 +2,6 @@ package api
import (
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
@ -119,13 +118,13 @@ func TestIngest(t *testing.T) {
return
}
defer resp.Body.Close()
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
defer io.Copy(io.Discard, resp.Body) //nolint: errcheck
if !assert.Equal(t, 200, resp.StatusCode) {
body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
t.Logf("request error: %s", body)
return
}
data, err := ioutil.ReadAll(resp.Body)
data, err := io.ReadAll(resp.Body)
if !assert.NoError(t, err) {
return
}
@ -190,13 +189,13 @@ func TestIngest(t *testing.T) {
return
}
defer resp.Body.Close()
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
defer io.Copy(io.Discard, resp.Body) //nolint: errcheck
if !assert.Equal(t, 200, resp.StatusCode) {
body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
t.Logf("request error: %s", body)
return
}
data, err := ioutil.ReadAll(resp.Body)
data, err := io.ReadAll(resp.Body)
if !assert.NoError(t, err) {
return
}

7
idk/cmd/molecula-consumer-github/main.go
View file

@ -7,7 +7,6 @@ import (
"expvar"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
_ "net/http/pprof"
@ -15,9 +14,9 @@ import (
"path/filepath"
"time"
"github.com/jaffee/commandeer/pflag"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/jaffee/commandeer/pflag"
"github.com/pkg/errors"
)
@ -384,9 +383,9 @@ func (m *Main) openURLReader(t time.Time) (io.ReadCloser, error) {
}
// If cache enabled, write to file first and then return.
if buf, err := ioutil.ReadAll(resp.Body); err != nil {
if buf, err := io.ReadAll(resp.Body); err != nil {
return nil, err
} else if err := ioutil.WriteFile(cachePath+".tmp", buf, 0666); err != nil {
} else if err := os.WriteFile(cachePath+".tmp", buf, 0666); err != nil {
return nil, err
} else if err := os.Rename(cachePath+".tmp", cachePath); err != nil {
return nil, err

4
idk/common/configMap.go
View file

@ -3,7 +3,7 @@ package common
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"sync/atomic"
@ -47,7 +47,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) {
var err error
configMap := &confluent.ConfigMap{}
if m.KafkaConfiguration != "" {
file, er := ioutil.ReadFile(m.KafkaConfiguration)
file, er := os.ReadFile(m.KafkaConfiguration)
if er != nil {
return nil, er
}

5
idk/csv/csv_test.go
View file

@ -3,7 +3,6 @@ package csv
import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
@ -257,7 +256,7 @@ func tim(t *testing.T, tstr string) time.Time {
}
func writeTempFile(t *testing.T, data string) string {
f, err := ioutil.TempFile("", "")
f, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("getting temp file: %v", err)
}
@ -321,7 +320,7 @@ func TestStreamFileNames(t *testing.T) {
}
func testDirTree(tree string) (string, error) {
tmp, err := ioutil.TempDir("", "")
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}

3
idk/datagen/custom.go
View file

@ -5,7 +5,6 @@ import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
@ -31,7 +30,7 @@ type Custom struct {
// NewCustom returns a new instance of Custom.
func NewCustom(cfg SourceGeneratorConfig) Sourcer {
conf := &CustomConfig{}
if bytes, err := ioutil.ReadFile(cfg.CustomConfig); err != nil {
if bytes, err := os.ReadFile(cfg.CustomConfig); err != nil {
return &Custom{err: errors.Wrap(err, "reading custom config file")}
} else if err = yaml.Unmarshal(bytes, conf); err != nil {
return &Custom{err: errors.Wrap(err, "unmarshaling custom config file")}

3
idk/datagen/custom_test.go
View file

@ -1,7 +1,6 @@
package datagen
import (
"io/ioutil"
"math/rand"
"os"
"strings"
@ -148,7 +147,7 @@ func TestGetIDKFields(t *testing.T) {
}
func TestNewCustomUnmarshalling(t *testing.T) {
tmp, err := ioutil.TempFile("", "")
tmp, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("creating temp file: %v", err)
}

5
idk/idallocator.go
View file

@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/bits"
"net/http"
"net/url"
@ -377,7 +376,7 @@ func (ps *pilosaIDManager) reserve(ctx context.Context, reserveReq pilosacore.ID
err = errors.Wrap(cerr, "closing ID reservation request body")
}
}()
body, err = ioutil.ReadAll(resp.Body)
body, err = io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading ID reservation request body")
}
@ -442,7 +441,7 @@ func (ps *pilosaIDManager) commit(ctx context.Context, commitRequest pilosacore.
err = errors.Wrap(cerr, "closing ID reservation request body")
}
}()
body, err = ioutil.ReadAll(resp.Body)
body, err = io.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "reading ID reservation request body")
}

4
idk/idktest/idktest.go
View file

@ -3,7 +3,7 @@ package idktest
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"os"
"strings"
@ -62,7 +62,7 @@ func DoExtractQuery(pql, index string) (ExtractResponse, error) {
}
defer resp.Body.Close()
s, err := ioutil.ReadAll(resp.Body)
s, err := io.ReadAll(resp.Body)
if err != nil {
return eResp, errors.Errorf("reading response: %v", err)
}

4
idk/ingest.go
View file

@ -23,7 +23,6 @@ import (
"syscall"
"time"
"github.com/felixge/fgprof"
pilosacore "github.com/featurebasedb/featurebase/v3"
pilosagrpc "github.com/featurebasedb/featurebase/v3/api/client"
pilosaclient "github.com/featurebasedb/featurebase/v3/client"
@ -32,6 +31,7 @@ import (
"github.com/featurebasedb/featurebase/v3/prometheus"
proto "github.com/featurebasedb/featurebase/v3/proto"
"github.com/featurebasedb/featurebase/v3/stats"
"github.com/felixge/fgprof"
"github.com/pkg/errors"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -1500,8 +1500,6 @@ func (m *Main) batchFromSchema(schema []Field) ([]Recordizer, pilosaclient.Recor
case DeleteSentinel:
if hasMutex { //need to clear the mutex
rec.Clears[valIdx] = nil //? maybe
} else { //TODO(twg) set fields not supported
}
default:
rec.Values[valIdx], err = idkField.PilosafyVal(rawRec[i])

5
idk/ingest_test.go
View file

@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
@ -15,11 +14,11 @@ import (
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/featurebasedb/featurebase/v3/authn"
pilosaclient "github.com/featurebasedb/featurebase/v3/client"
"github.com/featurebasedb/featurebase/v3/idk/idktest"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/golang-jwt/jwt"
"github.com/pkg/errors"
)
@ -436,7 +435,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) {
if err != nil {
t.Errorf("request error: %v", err)
}
contents, err := ioutil.ReadAll(response.Body)
contents, err := io.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
t.Errorf("read error: %v", err)

5
idk/internal/s3.go
View file

@ -2,7 +2,6 @@ package internal
import (
"bytes"
"io/ioutil"
"net/url"
"os"
"strings"
@ -57,7 +56,7 @@ func ReadFileOrURL(name string, s3client s3iface.S3API) ([]byte, error) {
}
content = buf.Bytes()
} else {
content, err = ioutil.ReadFile(name)
content, err = os.ReadFile(name)
if err != nil {
if os.IsNotExist(err) {
return nil, FileOrURLNotFound
@ -91,7 +90,7 @@ func WriteFileOrURL(name string, contents []byte, s3client s3iface.S3API) error
return errors.Wrapf(err, "putting S3 object %v", name)
}
} else {
err = ioutil.WriteFile(name, contents, 0644)
err = os.WriteFile(name, contents, 0644)
if err != nil {
return errors.Wrapf(err, "reading file %v", name)
}

6
idk/kafka/cmd_test.go
View file

@ -5,7 +5,7 @@ package kafka
import (
"fmt"
"io/ioutil"
"io"
"math/rand"
"net/http"
"os"
@ -17,11 +17,11 @@ import (
"time"
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
liavro "github.com/linkedin/goavro/v2"
pilosaclient "github.com/featurebasedb/featurebase/v3/client"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
"github.com/featurebasedb/featurebase/v3/logger"
liavro "github.com/linkedin/goavro/v2"
)
var pilosaHost string
@ -673,7 +673,7 @@ func tDoHTTPPost(t *testing.T, url, contentType, body string) string {
t.Fatalf("making POST request: %v", err)
}
bod, err := ioutil.ReadAll(resp.Body)
bod, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("reading POST response bdoy: %v", err)
}

4
idk/kafka/csrc/csrc.go
View file

@ -4,7 +4,7 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"strings"
@ -114,7 +114,7 @@ func unmarshalRespErr(resp *http.Response, err error, into interface{}) error {
return errors.Wrap(err, "making http request")
}
if resp.StatusCode != 200 {
bod, err := ioutil.ReadAll(resp.Body)
bod, err := io.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "reading body")
}

5
idk/kafka/put.go
View file

@ -3,16 +3,15 @@ package kafka
import (
"context"
"encoding/binary"
"io/ioutil"
"os"
"time"
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
liavro "github.com/linkedin/goavro/v2"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/idk/common"
"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
"github.com/featurebasedb/featurebase/v3/logger"
liavro "github.com/linkedin/goavro/v2"
"github.com/pkg/errors"
)
@ -173,7 +172,7 @@ func (p *PutCmd) getSchema() (string, error) {
if p.Schema != "" {
return p.Schema, nil
}
bytes, err := ioutil.ReadFile(p.SchemaFile)
bytes, err := os.ReadFile(p.SchemaFile)
if err != nil {
return "", errors.Wrap(err, "reading schema file")
}

14
idk/kafka/source.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
@ -21,11 +20,11 @@ import (
"time"
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/go-avro/avro"
pilosaclient "github.com/featurebasedb/featurebase/v3/client"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/idk/common"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/go-avro/avro"
"github.com/pkg/errors"
)
@ -253,11 +252,6 @@ func (r *Record) Data() []interface{} {
return r.data
}
// Assuming committed msgs are in order
func calOffsetDiff(section, committed []confluent.TopicPartition) []confluent.TopicPartition {
return section[len(committed):]
}
func (s *Source) CommitMessages(recs []confluent.TopicPartition) ([]confluent.TopicPartition, error) {
return s.client.CommitOffsets(recs)
}
@ -1022,7 +1016,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
}
defer schemaUrlResponse.Body.Close()
if schemaUrlResponse.StatusCode >= 300 {
bod, err := ioutil.ReadAll(schemaUrlResponse.Body)
bod, err := io.ReadAll(schemaUrlResponse.Body)
if err != nil {
return nil, errors.Wrapf(err, "Failed to get schema, code: %d, no body", schemaUrlResponse.StatusCode)
}
@ -1056,7 +1050,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
s.Log.Infof("Problem getting subject/version info for schema: %v", err)
} else {
if subVerResponse.StatusCode >= 300 {
bod, err := ioutil.ReadAll(subVerResponse.Body)
bod, err := io.ReadAll(subVerResponse.Body)
s.Log.Infof("Problem getting subject/version info for schema, response: %s. Err reading body: %v", bod, err)
}
defer subVerResponse.Body.Close()
@ -1066,7 +1060,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
Version int `json:"version"`
}
if bod, err := ioutil.ReadAll(subVerResponse.Body); err != nil {
if bod, err := io.ReadAll(subVerResponse.Body); err != nil {
s.Log.Infof("decoding subj/version %s body: %v", schemaSubVerUrl, err)
} else if err := json.Unmarshal(bod, &tempSchemaStruct); err != nil {
s.Log.Infof("decoding schema subject & version from registry: %v", err)

11
idk/kafka/source_test.go
View file

@ -9,7 +9,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"math/rand"
"os"
@ -20,13 +19,13 @@ import (
"github.com/confluentinc/confluent-kafka-go/kafka"
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/glycerine/vprint"
"github.com/go-avro/avro"
liavro "github.com/linkedin/goavro/v2"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/idk/common"
"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/glycerine/vprint"
"github.com/go-avro/avro"
liavro "github.com/linkedin/goavro/v2"
)
func configureSourceTestFlags(source *Source) {
@ -81,7 +80,7 @@ func TestAvroToPDKSchema(t *testing.T) {
}
// check that we've covered all the test schemas
files, err := ioutil.ReadDir("./testdata/schemas")
files, err := os.ReadDir("./testdata/schemas")
if err != nil {
t.Fatalf("reading directory: %v", err)
}
@ -251,7 +250,7 @@ func decodeTestSchema(t *testing.T, filename string) avro.Schema {
}
func readTestSchema(t *testing.T, filename string) string {
bytes, err := ioutil.ReadFile("./testdata/schemas/" + filename)
bytes, err := os.ReadFile("./testdata/schemas/" + filename)
if err != nil {
t.Fatalf("reading schema file: %v", err)
}

9
idk/kafka_sasl/source.go
View file

@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"os"
"sort"
"strconv"
"sync"
@ -185,11 +185,6 @@ func (r *Record) Commit(ctx context.Context) error {
return nil
}
// Assuming committed msgs are in order
func calOffsetDiff(section, committed []confluent.TopicPartition) []confluent.TopicPartition {
return section[len(committed):]
}
func (r *Record) Data() []interface{} {
return r.data
}
@ -200,7 +195,7 @@ func (s *Source) Open() error {
return errors.New("needs header specification file")
}
headerData, err := ioutil.ReadFile(s.Header)
headerData, err := os.ReadFile(s.Header)
if err != nil {
return errors.Wrap(err, "reading header file")
}

4
idk/kafka_static/source.go
View file

@ -5,8 +5,8 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/url"
"os"
"strconv"
"strings"
"time"
@ -262,7 +262,7 @@ func (s *Source) readFileOrURL(name string) ([]byte, error) {
s.Log.Printf("read %d bytes from %s\n", bytesRead, name)
content = buf.Bytes()
} else {
content, err = ioutil.ReadFile(name)
content, err = os.ReadFile(name)
if err != nil {
return nil, errors.Wrapf(err, "reading file %v", name)
}

4
idk/kafka_static/source_test.go
View file

@ -6,9 +6,9 @@ package kafka_static
import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net"
"os"
"reflect"
"testing"
"time"
@ -68,7 +68,7 @@ func TestKafkaStaticSourceLocal(t *testing.T) {
src := NewSource()
configureSourceTestFlags(src)
{
headerData, err := ioutil.ReadFile(test.header)
headerData, err := os.ReadFile(test.header)
if err != nil {
t.Fatalf("reading header file: %v", err)
}

5
idk/kafkagen/cmd.go
View file

@ -3,15 +3,14 @@ package kafkagen
import (
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
liavro "github.com/linkedin/goavro/v2"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/idk/common"
"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
liavro "github.com/linkedin/goavro/v2"
"github.com/pkg/errors"
)
@ -125,7 +124,7 @@ func (m *Main) putRecordKafka(p *confluent.Producer, schemaID int, schema *liavr
}
func readSchema(filename string) (string, error) {
bytes, err := ioutil.ReadFile(filename)
bytes, err := os.ReadFile(filename)
if err != nil {
return "", err
}

21
idk/kinesis/logger_test.go
View file

@ -102,7 +102,7 @@ func TestNewSinkErrorQueueSuccess(t *testing.T) {
queue, err := NewSinkErrorQueue(mockSQS, "dummy-123", "a-b-c")
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, "a-b-c", queue.sinkId)
assert.Equal(t, "dummy-123", queue.name)
assert.Equal(t, "https://unit-test.queue.dummy-123.url", queue.url)
@ -149,8 +149,7 @@ func TestSinkErrorQueueFromSuccess(t *testing.T) {
StreamName: streamName,
}
var queue *SinkErrorQueue
queue = SinkErrorQueueFrom(mockSQS, source)
queue := SinkErrorQueueFrom(mockSQS, source)
assert.Equal(t, validUuid, queue.sinkId)
assert.Equal(t, "my-queue-01234", queue.name)
@ -166,8 +165,7 @@ func TestSinkErrorQueueFromFailMissingQueueName(t *testing.T) {
streamName := fmt.Sprintf("sink-%s", validUuid) // Structure mimics how cloud ECS instance names its Kinesis stream.
source := &Source{StreamName: streamName}
var queue *SinkErrorQueue
queue = SinkErrorQueueFrom(mockSQS, source)
queue := SinkErrorQueueFrom(mockSQS, source)
assert.Equal(t, "", queue.sinkId)
assert.Equal(t, "", queue.name)
@ -194,8 +192,7 @@ func TestSinkErrorQueueFromFailMalformedSinkId(t *testing.T) {
StreamName: streamName,
}
var queue *SinkErrorQueue
queue = SinkErrorQueueFrom(mockSQS, source)
queue := SinkErrorQueueFrom(mockSQS, source)
assert.Equal(t, malformedUuid, queue.sinkId) // Keeps invalid sink ID around for downstream logging purposes.
assert.Equal(t, "valid-queue-90123", queue.name)
@ -220,8 +217,7 @@ func TestSinkErrorQueueFromFailEmptyStreamNameDoesNotCausePanic(t *testing.T) {
StreamName: "",
}
var queue *SinkErrorQueue
queue = SinkErrorQueueFrom(mockSQS, source)
queue := SinkErrorQueueFrom(mockSQS, source)
assert.Equal(t, "", queue.sinkId) // Keeps invalid sink ID around for downstream logging purposes.
assert.Equal(t, "valid-queue-88888888888", queue.name)
@ -249,8 +245,7 @@ func TestSinkErrorQueueFromFailSQSGetQueueUrlErrored(t *testing.T) {
StreamName: streamName,
}
var queue *SinkErrorQueue
queue = SinkErrorQueueFrom(mockSQS, source)
queue := SinkErrorQueueFrom(mockSQS, source)
assert.Equal(t, validUuid, queue.sinkId) // Keeps valid sink ID around for downstream logging purposes.
assert.Equal(t, "queue-that-we-stole-962463", queue.name)
@ -276,7 +271,7 @@ func TestSinkErrorQueuePushWhenQueueNotAvailableDoesNotThrowErrorAndNoOps(t *tes
// Logger is nil.
err := emptySeq.Push(RecoverableErrorType, "werqw zxcw7228323974", nil)
assert.Nil(t, err)
assert.NoError(t, err)
// Logger is not nil; check that a warning is issued using the Logger instance.
logger := NewMapStashLogger()
@ -313,7 +308,7 @@ func TestSinkErrorQueuePushSuccess(t *testing.T) {
errMsg := "079 cxmn, 198sfakjl"
err := seq.Push(PanicErrorType, errMsg, nil)
assert.Nil(t, err)
assert.NoError(t, err)
// Verify queue URL.
assert.Equal(t, seq.url, actualQueueUrl)

18
idk/kinesis/reader_test.go
View file

@ -158,19 +158,19 @@ func TestStreamReaderStartFetchCommitWithoutOffsets(t *testing.T) {
}
err := reader.Start()
assert.Nil(t, err)
assert.NoError(t, err)
var records []ShardRecord
for i := 0; i < testShardCount; i++ {
rec, err := reader.FetchMessage(context.Background())
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, rec.SequenceNumber, aws.String("1"))
records = append(records, rec)
}
assert.Empty(t, reader.offsets.Shards)
err = reader.CommitMessages(context.Background(), records...)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Len(t, reader.offsets.Shards, 2)
for i := 0; i < testShardCount; i++ {
@ -253,12 +253,12 @@ func TestStreamReaderStartFetchCommitFromExistingOffsets(t *testing.T) {
}
err := reader.Start()
assert.Nil(t, err)
assert.NoError(t, err)
var records []ShardRecord
for i := 0; i < testShardCount; i++ {
rec, err := reader.FetchMessage(context.Background())
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, rec.SequenceNumber, aws.String("1"))
records = append(records, rec)
}
@ -270,7 +270,7 @@ func TestStreamReaderStartFetchCommitFromExistingOffsets(t *testing.T) {
}
err = reader.CommitMessages(context.Background(), records...)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Len(t, reader.offsets.Shards, 2)
for i := 0; i < testShardCount; i++ {
@ -389,17 +389,17 @@ func TestStreamOrderlyReadsAfterResharding(t *testing.T) {
).Return(getRecordsOutput, nil).Once()
err := reader.Start()
assert.Nil(t, err)
assert.NoError(t, err)
for i := 1; i <= recordsBeforeClose; i++ {
rec, err := reader.FetchMessage(context.Background())
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, strconv.Itoa(i), *rec.SequenceNumber)
assert.Equal(t, rec.ShardID, shardName(0))
}
rec, err := reader.FetchMessage(context.Background())
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, "1", *rec.SequenceNumber)
assert.Equal(t, rec.ShardID, shardName(1))

3
idk/tls.go
View file

@ -3,7 +3,6 @@ package idk
import (
"crypto/tls"
"crypto/x509"
"io/ioutil"
"log"
"os"
"os/signal"
@ -148,7 +147,7 @@ func getCertPool(capath string) (*x509.CertPool, error) {
)
caCertData = []byte(capath)
} else {
caCertData, err = ioutil.ReadFile(capath)
caCertData, err = os.ReadFile(capath)
if err != nil {
return nil, errors.Wrap(err, "loading tls ca key")
}

3
ingest/codec.go
View file

@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"sort"
"strconv"
@ -748,7 +747,7 @@ func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err
// Parse reads a request, but does not sort the results at all or divide
// them into shards.
func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) {
data, err := ioutil.ReadAll(r)
data, err := io.ReadAll(r)
if err != nil {
return nil, err
}

3
ingest_test.go
View file

@ -6,7 +6,6 @@ import (
"bytes"
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strconv"
@ -234,7 +233,7 @@ func testQueries(t *testing.T, ctx context.Context, cmd *test.Command, index str
// testOneIngestTestcase runs a set of actions, then cleans up after itself
func testOneIngestTestcase(t *testing.T, ctx context.Context, cmd *test.Command, tcpath string) {
data, err := ioutil.ReadFile(tcpath)
data, err := os.ReadFile(tcpath)
if err != nil {
t.Fatalf("reading %q: %v", tcpath, err)
}

2
internal/clustertests/pause_node_test.go
View file

@ -130,6 +130,8 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient,
}
func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) {
//TODO lint - uses the fs.FileInfo.Mode to filter out directories later
// this does not exist in the os.DirEntry elements returned by os.ReadDir
dirEntries, err := ioutil.ReadDir(dirPath)
if err != nil {
return nil, err

41
internal_client.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
"net/http"
@ -20,13 +19,13 @@ import (
"strings"
"time"
"github.com/hashicorp/go-retryablehttp"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/ingest"
"github.com/featurebasedb/featurebase/v3/logger"
pnet "github.com/featurebasedb/featurebase/v3/net"
"github.com/featurebasedb/featurebase/v3/tracing"
"github.com/hashicorp/go-retryablehttp"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
@ -421,7 +420,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []
return nil, errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
buf, err = ioutil.ReadAll(resp.Body)
buf, err = io.ReadAll(resp.Body)
if resp.StatusCode != 200 {
if err != nil {
return nil, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status)
@ -439,7 +438,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []
}
return nil, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg)
}
// this is the err from ioutil.ReadAll, but in the case where resp.StatusCode
// this is the err from io.ReadAll, but in the case where resp.StatusCode
// was 2xx, so we don't have a bad status.
if err != nil {
return nil, errors.Wrapf(err, "error reading response body")
@ -728,7 +727,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -816,7 +815,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *disco.Node, index
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "reading")
}
@ -1339,7 +1338,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi
// Decode response object.
var rsp BlockDataResponse
if body, err := ioutil.ReadAll(resp.Body); err != nil {
if body, err := io.ReadAll(resp.Body); err != nil {
return nil, nil, errors.Wrap(err, "reading")
} else if err := c.serializer.Unmarshal(body, &rsp); err != nil {
return nil, nil, errors.Wrap(err, "unmarshalling")
@ -1372,7 +1371,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []b
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
_, err = io.Copy(ioutil.Discard, resp.Body)
_, err = io.Copy(io.Discard, resp.Body)
return errors.Wrap(err, "draining SendMessage response body")
}
@ -1421,7 +1420,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1473,7 +1472,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1505,7 +1504,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]P
defer resp.Body.Close()
// Read body and unmarshal response.
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading")
}
@ -1552,7 +1551,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i
}()
// Read the response body.
result, err := ioutil.ReadAll(resp.Body)
result, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading response")
}
@ -1601,7 +1600,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i
}()
// Read the response body.
result, err := ioutil.ReadAll(resp.Body)
result, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading response")
}
@ -1651,7 +1650,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI,
}()
// Read the response body.
result, err := ioutil.ReadAll(resp.Body)
result, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading response")
}
@ -1704,7 +1703,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI,
}()
// Read the response body.
result, err := ioutil.ReadAll(resp.Body)
result, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading response")
}
@ -1749,7 +1748,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI,
}()
// Read the response body.
result, err := ioutil.ReadAll(resp.Body)
result, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "reading response")
}
@ -1781,7 +1780,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*Transact
return nil, errors.Wrap(err, "executing request")
}
defer func() {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
trnsMap := make(map[string]*Transaction)
@ -1820,7 +1819,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou
return nil, errors.Wrap(err, "executing request")
}
defer func() {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
tr := &TransactionResponse{}
@ -1855,7 +1854,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*Tra
return nil, errors.Wrap(err, "executing request")
}
defer func() {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
tr := &TransactionResponse{}
@ -1892,7 +1891,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*Transa
return nil, errors.Wrap(err, "executing request")
}
defer func() {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
tr := &TransactionResponse{}
@ -1994,7 +1993,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
buf, err := io.ReadAll(resp.Body)
if err != nil {
return resp, errors.Wrapf(err, "bad status '%s' and err reading body", resp.Status)
}

20
logger/filewriter_test.go
View file

@ -26,7 +26,6 @@
package logger
import (
"io/ioutil"
"os"
"testing"
@ -35,14 +34,13 @@ import (
// TestReopenAppend -- make sure we always append to an existing file
//
// 1. Create a sample file using normal means
// 2. Open a ioreopen.File
// write line 1
// 3. call Reopen
// write line 2
// 4. close file
// 5. read file, make sure it contains line0,line1,line2
//
// 1. Create a sample file using normal means
// 2. Open a ioreopen.File
// write line 1
// 3. call Reopen
// write line 2
// 4. close file
// 5. read file, make sure it contains line0,line1,line2
func TestReopenAppend(t *testing.T) {
forig, err := testhook.TempFile(t, "logger-reopen")
if err != nil {
@ -83,7 +81,7 @@ func TestReopenAppend(t *testing.T) {
t.Errorf("Got closing error for %s: %s", fname, err)
}
out, err := ioutil.ReadFile(fname)
out, err := os.ReadFile(fname)
if err != nil {
t.Fatalf("Unable read in final file %s: %s", fname, err)
}
@ -152,7 +150,7 @@ func TestChangeInode(t *testing.T) {
t.Errorf("Got closing error for %s: %s", fname, err)
}
out, err := ioutil.ReadFile(fname)
out, err := os.ReadFile(fname)
if err != nil {
t.Fatalf("Unable read in final file %s: %s", fname, err)
}

3
logger/logger.go
View file

@ -6,7 +6,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"sync"
"time"
@ -258,5 +257,5 @@ func (b *bufferLogger) Panicf(format string, v ...interface{}) {
func (b *bufferLogger) ReadAll() ([]byte, error) {
b.mu.Lock()
defer b.mu.Unlock()
return ioutil.ReadAll(b.buf)
return io.ReadAll(b.buf)
}

3
pb/pb.go
View file

@ -4,7 +4,6 @@ package pb
import (
"io"
"io/ioutil"
"github.com/gogo/protobuf/proto"
)
@ -52,7 +51,7 @@ func NewDecoder(r io.Reader) *Decoder {
// Decode reads all bytes from the reader and unmarshals them into pb.
func (dec *Decoder) Decode(pb proto.Message) error {
buf, err := ioutil.ReadAll(dec.r)
buf, err := io.ReadAll(dec.r)
if err != nil {
return err
}

3
pg/protocol.go
View file

@ -11,7 +11,6 @@ import (
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net"
"regexp"
"strings"
@ -1087,7 +1086,7 @@ func (s *Server) handleShutdown(conn net.Conn, w message.Writer, encoder *messag
go func() {
defer wg.Done()
io.Copy(ioutil.Discard, conn) //nolint:errcheck
io.Copy(io.Discard, conn) //nolint:errcheck
}()
// Attempt to send the shutdown notification.

3
pql/parser.go
View file

@ -5,7 +5,6 @@ package pql
import (
"fmt"
"io"
"io/ioutil"
"strconv"
"strings"
"unicode/utf8"
@ -40,7 +39,7 @@ func ParseString(s string) (*Query, error) {
// Parse parses the next node in the query.
func (p *parser) Parse() (*Query, error) {
buf, err := ioutil.ReadAll(p.r)
buf, err := io.ReadAll(p.r)
if err != nil {
return nil, errors.Wrap(err, "reading buffer to parse")
}

2
pql/parser_test.go
View file

@ -218,7 +218,7 @@ func TestParser_Parse(t *testing.T) {
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
q, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`)
_, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`)
if err == nil {
t.Fatal("no error parsing invalid date")
} else if !strings.Contains(err.Error(), "not a valid timestamp") {

6
rbf.go
View file

@ -68,7 +68,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) {
// rbfDBRegistrar also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
//
type rbfDBRegistrar struct {
mu sync.Mutex
mp map[*RbfDBWrapper]bool
@ -224,11 +223,6 @@ func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (c
// which is expensive in practice and only really useful occasionally.
const sortedParanoia = false
type countResults struct {
changeCount int
err error
}
func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) {
if len(a) == 0 {
return 0, nil

6
rbf/cursor_test.go
View file

@ -15,6 +15,7 @@ import (
"github.com/featurebasedb/featurebase/v3/rbf"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/stretchr/testify/assert"
)
func TestCursor_FirstNext(t *testing.T) {
@ -593,7 +594,8 @@ func TestCursor_BitmapBitN(t *testing.T) {
if c1 != c2 {
t.Fatalf("expected count %d, got %d", c2, c1)
}
tx.Commit()
err = tx.Commit()
assert.NoError(t, err)
}
func TestCursor_RLEConversion(t *testing.T) {
@ -1060,7 +1062,7 @@ func TestCursor_RemoveCells(t *testing.T) {
//f, err := os.OpenFile("before.dot", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 066)
}
//These aren't test i'm just using to generate graphs to look at structure
// These aren't test i'm just using to generate graphs to look at structure
func TestCursor_PlayContainer(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)

2
rbf/db.go
View file

@ -344,7 +344,7 @@ func (db *DB) checkpoint() (err error) {
if IsBitmapHeader(page) {
pgno = readPageNo(page)
if i+1 < db.walPageN {
if page, err = db.readWALPageAt(i + 1); err != nil {
if _, err = db.readWALPageAt(i + 1); err != nil {
return err
}
} else {

4
rbf/db_test.go
View file

@ -15,9 +15,9 @@ import (
_ "net/http/pprof"
"github.com/felixge/fgprof"
"github.com/featurebasedb/featurebase/v3/rbf"
rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg"
"github.com/felixge/fgprof"
"golang.org/x/sync/errgroup"
)
@ -579,7 +579,7 @@ func BenchmarkDbCheckpoint(b *testing.B) {
benchmarkOneCheckpoint(b, randInts)
}
b.StopTimer()
done()
_ = done()
}
// better diagnosis of deadlocks/hung situations versus just really slow "Quick" tests.

41
rbf/tx_test.go
View file

@ -16,6 +16,7 @@ import (
"github.com/featurebasedb/featurebase/v3/rbf"
"github.com/featurebasedb/featurebase/v3/roaring"
"github.com/stretchr/testify/assert"
)
func TestTx_CommitRollback(t *testing.T) {
@ -1150,37 +1151,37 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) {
for pgno, info := range infos {
switch info := info.(type) {
case *rbf.MetaPageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "meta")
pf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo)
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "meta")
_, _ = pf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo)
case *rbf.RootRecordPageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "rootrec")
pf("next=%d\n", info.Next)
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "rootrec")
_, _ = pf("next=%d\n", info.Next)
case *rbf.LeafPageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "leaf")
pf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "leaf")
_, _ = pf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
case *rbf.BranchPageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "branch")
pf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "branch")
_, _ = pf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
case *rbf.BitmapPageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "bitmap")
pf("-\n")
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "bitmap")
_, _ = pf("-\n")
case *rbf.FreePageInfo:
pf("%-8d ", pgno)
pf("%-10s ", "free")
pf("-\n")
_, _ = pf("%-8d ", pgno)
_, _ = pf("%-10s ", "free")
_, _ = pf("-\n")
default:
t.Fatal(fmt.Sprintf("unexpected page info type %T", info))
t.Fatalf("unexpected page info type %T", info)
}
}
@ -1212,7 +1213,7 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) {
ifError(db.Check())
tx := MustBegin(t, db, true)
tx.DeleteBitmapsWithPrefix(prefix)
assert.Nil(t, tx.DeleteBitmapsWithPrefix(prefix))
ifError(tx.Commit())
ifError(db.Check())
checkInfos(pBuf)

3
roaring/fuzzer.go
View file

@ -8,7 +8,6 @@ package roaring
import (
"encoding/binary"
"fmt"
"io/ioutil"
"reflect"
)
@ -293,7 +292,7 @@ func bytesToUint64s(data []byte) []uint64 {
// make sure filename is not already in the corpus.
func addSliceToCorpus(slice []uint64, filename, path string) {
data := uint64sToBytes(slice)
err := ioutil.WriteFile(path+"/"+filename, data, 0750)
err := os.WriteFile(path+"/"+filename, data, 0750)
if err != nil {
fmt.Printf("could not write to file: %v\n", err)
}

57
roaring/roaring.go
View file

@ -1364,31 +1364,49 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
// the bitmap at a specific key, ^ symbol represents the bitmaps current container iteration position,
// and the - symbol represents a container that is at the current iteration position, but has been marked as "handled".
//
// ---------------------------- | ---------------------------- | ----------------------------
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________|
// ^ | _ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// ^ | _ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 2 |_______X________X______X___| | |_______X_______________X___| | |_______X_______________X___|
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________|
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// ^ | ^ |
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________|
// ^ | _ |
//
// ^ | _ |
//
// ------------------------------------------------------------------------------------------------------------------------
// ---------------------------- | ---------------------------- | ----------------------------
//
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________|
// _ | ^ | _
// ---------------------------- | ---------------------------- | ----------------------------
//
// _ | ^ | _
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 2 |_______X_______________X___| | |_______X_______________X___| | |_______X_______________X___|
// _ | ^ | ^
// ---------------------------- | ---------------------------- | ----------------------------
//
// _ | ^ | ^
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________|
// _ | |
// ---------------------------- | ---------------------------- | ----------------------------
//
// _ | |
// ---------------------------- | ---------------------------- | ----------------------------
//
// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________|
// _
//
// _
func (b *Bitmap) unionInPlace(others ...*Bitmap) {
const staticSize = 20
var (
@ -1823,10 +1841,7 @@ type containerIteratorRoaringIteratorWrapper struct {
func (c *containerIteratorRoaringIteratorWrapper) Next() bool {
c.nextKey, c.nextCont = c.r.NextContainer()
if c.nextCont == nil {
return false
}
return true
return c.nextCont != nil
}
func (c *containerIteratorRoaringIteratorWrapper) Value() (uint64, *Container) {
@ -6744,7 +6759,7 @@ func xorCompare(x *xorstm) (r1 Interval16, hasData bool) {
return r1, hasData
}
//stm is state machine used to "xor" iterate over runs.
// stm is state machine used to "xor" iterate over runs.
type xorstm struct {
vaValid, vbValid bool
va, vb Interval16

44
roaring/roaring_internal_test.go
View file

@ -6,8 +6,8 @@ import (
"bytes"
"encoding/hex"
"fmt"
"io/ioutil"
"math/rand"
"os"
"reflect"
"runtime"
"strings"
@ -2022,7 +2022,7 @@ func TestXorArrayRun(t *testing.T) {
}
//special case that didn't fit the xorrunrun table testing below.
// special case that didn't fit the xorrunrun table testing below.
func TestXorRunRun1(t *testing.T) {
a := NewContainerRun([]Interval16{{Start: 4, Last: 10}})
b := NewContainerRun([]Interval16{{Start: 5, Last: 10}})
@ -3779,7 +3779,7 @@ func TestContainerCombinations(t *testing.T) {
}
}
//func getFunc(func(a, b *container) *container, m, n *container) *container {
// func getFunc(func(a, b *container) *container, m, n *container) *container {
func runContainerFunc(f interface{}, c ...*Container) *Container {
switch f := f.(type) {
case func(*Container) *Container:
@ -3822,7 +3822,7 @@ func TestUnmarshalRoaringWithNoErrors(t *testing.T) {
t.Fatalf("hex decode %s", err)
}
} else {
testContainer, _ = ioutil.ReadFile(testCase.roaringFileName)
testContainer, _ = os.ReadFile(testCase.roaringFileName)
}
bm := NewBitmap()
err = bm.UnmarshalBinary(testContainer)
@ -3890,24 +3890,25 @@ func newTestBitmapContainer() *Container {
/*
// This function exercises an arcane edge case in dead code.
// It doesn't need to be run right now.
func TestEquals(t *testing.T) {
bma := NewBitmap()
bmr := NewBitmap()
for i := uint64(0); i < 30; i++ {
bma.Add(i)
bmr.Add(i)
func TestEquals(t *testing.T) {
bma := NewBitmap()
bmr := NewBitmap()
for i := uint64(0); i < 30; i++ {
bma.Add(i)
bmr.Add(i)
}
bmr.Optimize()
bmi := bma.Intersect(bmr)
err := bitmapsEqual(bmi, bma)
if err != nil {
t.Fatalf("expected intersection to equal array")
}
err = bitmapsEqual(bmi, bmr)
if err != nil {
t.Fatalf("expected intersection to equal run")
}
}
bmr.Optimize()
bmi := bma.Intersect(bmr)
err := bitmapsEqual(bmi, bma)
if err != nil {
t.Fatalf("expected intersection to equal array")
}
err = bitmapsEqual(bmi, bmr)
if err != nil {
t.Fatalf("expected intersection to equal run")
}
}
*/
func TestShiftArray(t *testing.T) {
tests := []struct {
@ -4564,7 +4565,6 @@ func TestRoaringIteratorSkip(t *testing.T) {
// large an run container, which was causing problems when
// we write to the transactional backends. Verify that
// unionRunRunInPlace() converts to bitmap if its too large.
//
func TestContainer_unionRunRunInPlace_TwoBigRunArrays(t *testing.T) {
a := NewContainerRun(nil)

4
server/grpc.go
View file

@ -13,7 +13,6 @@ import (
"sync"
"time"
"github.com/improbable-eng/grpc-web/go/grpcweb"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/authz"
@ -24,6 +23,7 @@ import (
vdsm_pb "github.com/featurebasedb/featurebase/v3/proto/vdsm"
"github.com/featurebasedb/featurebase/v3/sql"
"github.com/featurebasedb/featurebase/v3/stats"
"github.com/improbable-eng/grpc-web/go/grpcweb"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -1674,7 +1674,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
// LogQuery logs requests
func LogQuery(ctx context.Context, method string, req interface{}, logger logger.Logger) {
uinfo, ok := ctx.Value("userinfo").(*authn.UserInfo)
uinfo, _ := ctx.Value("userinfo").(*authn.UserInfo)
md, _ := metadata.FromIncomingContext(ctx)
p, ok := peer.FromContext(ctx)
ip := ""

22
server/grpc_test.go
View file

@ -16,7 +16,6 @@ import (
"testing"
"time"
"github.com/golang-jwt/jwt"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/authn"
"github.com/featurebasedb/featurebase/v3/authz"
@ -27,7 +26,9 @@ import (
"github.com/featurebasedb/featurebase/v3/sql"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/vprint"
"github.com/golang-jwt/jwt"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
@ -1894,7 +1895,8 @@ func writeTestFile(t *testing.T, filename, content string) string {
if err != nil {
t.Fatal(err)
}
io.WriteString(f, content)
_, err = io.WriteString(f, content)
assert.NoError(t, err)
if err := f.Close(); err != nil {
t.Fatal(err)
}
@ -1919,11 +1921,7 @@ func Test_ChainUnaryInterceptor(t *testing.T) {
interceptors0 := []grpc.UnaryServerInterceptor{}
interceptors1 := []grpc.UnaryServerInterceptor{salt}
interceptors2 := []grpc.UnaryServerInterceptor{salt, pepper}
// interceptors5 := []grpc.UnaryServerInterceptor{interceptor, interceptor, interceptor, interceptor, interceptor}
type args struct {
interceptors []grpc.UnaryServerInterceptor
}
tests := []struct {
name string
interceptors []grpc.UnaryServerInterceptor
@ -2001,13 +1999,15 @@ func Test_ChainStreamInterceptor(t *testing.T) {
salt := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md := fromIncomingContext(ss.Context())
md.Append("ingredient", "with salt")
ss.SetHeader(md)
err := ss.SetHeader(md)
assert.NoError(t, err)
return handler(srv, ss)
}
pepper := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md := fromIncomingContext(ss.Context())
md.Append("ingredient", "and pepper")
ss.SetHeader(md)
err := ss.SetHeader(md)
assert.NoError(t, err)
return handler(srv, ss)
}
@ -2015,9 +2015,6 @@ func Test_ChainStreamInterceptor(t *testing.T) {
interceptors1 := []grpc.StreamServerInterceptor{salt}
interceptors2 := []grpc.StreamServerInterceptor{salt, pepper}
type args struct {
interceptors []grpc.StreamServerInterceptor
}
tests := []struct {
name string
interceptors []grpc.StreamServerInterceptor
@ -2042,7 +2039,8 @@ func Test_ChainStreamInterceptor(t *testing.T) {
}
t.Run(tt.name, func(t *testing.T) {
chained := server.ChainStreamInterceptors(tt.interceptors...)
chained(srv, ss, info, handler)
err := chained(srv, ss, info, handler)
assert.NoError(t, err)
if !reflect.DeepEqual(result, tt.want) {
t.Errorf("ChainStreamInterceptor() = %v, want %v", result, tt.want)
}

93
server/handler_test.go
View file

@ -9,7 +9,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
gohttp "net/http"
"net/http/httptest"
@ -37,7 +36,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`)))
if w.Code != gohttp.StatusNoContent {
bod, err := ioutil.ReadAll(w.Result().Body)
bod, err := io.ReadAll(w.Result().Body)
if err != nil {
t.Errorf("reading body: %v", err)
}
@ -113,7 +112,7 @@ func TestHandler_Endpoints(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/schema", strings.NewReader(`{"indexes":[{"name":"blah","options":{"keys":false,"trackExistence":true},"fields":[{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}`)))
if w.Code != gohttp.StatusNoContent {
bod, err := ioutil.ReadAll(w.Result().Body)
bod, err := io.ReadAll(w.Result().Body)
if err != nil {
t.Errorf("reading body: %v", err)
}
@ -1414,94 +1413,6 @@ func TestClusterTranslator(t *testing.T) {
}
}
// func TestQueryHistory(t *testing.T) {
// cluster := test.MustRunCluster(t, 3,
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("1"),
// )},
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("0"),
// )},
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("2"),
// )},
// )
// defer cluster.Close()
// cmd := cluster.GetNode(0)
// h := cmd.Handler.(*pilosa.Handler).Handler
// w := httptest.NewRecorder()
// test.Do(t, "POST", cmd.URL()+"/index/i0", "")
// test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "")
// gh := server.NewGRPCHandler(cmd.API)
// stream := &MockServerTransportStream{}
// ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream)
// _, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{
// Sql: `select * from i0`,
// })
// if err != nil {
// t.Fatalf("QuerySQLUnary failed: %v", err)
// }
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)")
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)")
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)")
// h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil))
// if w.Code != gohttp.StatusOK {
// t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
// }
// ret := make([]pilosa.PastQueryStatus, 4)
// b, err := ioutil.ReadAll(w.Body)
// if err != nil {
// t.Fatalf("reading: %v", err)
// }
// err = json.Unmarshal(b, &ret)
// if err != nil {
// t.Fatalf("unmarshalling: %v", err)
// }
// // verify result length
// if len(ret) != 4 {
// // each set query executes on both nodes once
// // topn query gets added to history on node0 once, node1 twice
// t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret)
// }
// // verify sort order
// if !sort.SliceIsSorted(ret, func(i, j int) bool {
// // must match the sort in api.PastQueries
// return ret[i].Start.After(ret[j].Start)
// }) {
// t.Fatalf("response list not sorted correctly")
// }
// // verify some response values
// if ret[0].Index != "i0" {
// t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index)
// }
// if ret[0].Node != cluster.GetNode(0).Server.NodeID() {
// t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID())
// }
// if ret[3].PQL != "Extract(All(),Rows(f0))" {
// t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL)
// }
// if ret[3].SQL != "select * from i0" {
// t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL)
// }
// if ret[0].PQL != "TopN(f0)" {
// t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL)
// }
// }
func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
dec := json.NewDecoder(r)
err := dec.Decode(&ret)

5
server/pg_internal_test.go
View file

@ -7,6 +7,7 @@ import (
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/pg"
"github.com/stretchr/testify/assert"
)
// pg_internal_test.go tests unexported methods from server/pg.go
@ -35,7 +36,9 @@ func (t *TestQueryResultWriter) Tag(tag string) {
func TestPgWriteDistinctTimestamp(t *testing.T) {
w := TestQueryResultWriter{}
expected := pilosa.DistinctTimestamp{Name: "test", Values: []string{"date1", "date2", "date3"}}
pgWriteDistinctTimestamp(&w, expected)
err := pgWriteDistinctTimestamp(&w, expected)
assert.NoError(t, err)
if w.Header[0].Name != expected.Name {
t.Fatalf("Header Name is wrong. got %v, want %v", w.Header[0], expected.Name)
}

3
server/server.go
View file

@ -13,7 +13,6 @@ import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net"
@ -189,7 +188,7 @@ func (m *Command) doSetupResourceLimits() error {
}
// We don't have corresponding options for non-Linux right now, but probably should.
if runtime.GOOS == "linux" {
result, err := ioutil.ReadFile("/proc/sys/vm/max_map_count")
result, err := os.ReadFile("/proc/sys/vm/max_map_count")
if err != nil {
m.logger.Infof("Tried unsuccessfully to check system mmap limit: %w", err)
} else {

6
server/server_test.go
View file

@ -8,10 +8,10 @@ import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"math/rand"
"net"
nethttp "net/http"
"os"
"reflect"
"sort"
"strings"
@ -698,7 +698,7 @@ func TestMain_ImportTimestamp(t *testing.T) {
}
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName)
files, err := ioutil.ReadDir(dir)
files, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
@ -754,7 +754,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
// Ensure the correct views were created.
dir := fmt.Sprintf("%s/%s/%s/%s/%s/views", m.Config.DataDir, pilosa.IndexesDir, indexName, pilosa.FieldsDir, fieldName)
files, err := ioutil.ReadDir(dir)
files, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}

3
server/tlsconfig.go
View file

@ -38,7 +38,6 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"os"
"os/signal"
"sync"
@ -136,7 +135,7 @@ func GetTLSConfig(tlsConfig *TLSConfig, logger logger.Logger) (TLSConfig *tls.Co
}
if hasCA {
b, err := ioutil.ReadFile(tlsConfig.CACertPath)
b, err := os.ReadFile(tlsConfig.CACertPath)
if err != nil {
return nil, errors.Wrap(err, "loading tls ca key")
}

12
test/pilosa.go
View file

@ -6,7 +6,7 @@ import (
"bytes"
"context"
"fmt"
"io/ioutil"
"io"
gohttp "net/http"
"os"
"reflect"
@ -21,7 +21,7 @@ import (
"github.com/featurebasedb/featurebase/v3/testhook"
)
////////////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////////
// Command represents a test wrapper for server.Command.
type Command struct {
*server.Command
@ -52,7 +52,7 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
}, opts...)
m := &Command{commandOptions: opts}
m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
m.Command = server.NewCommand(bytes.NewReader(nil), io.Discard, io.Discard, opts...)
// pick etcd ports using a socket rather than a real port
err = GetPortsGenConfigs(tb, []*Command{m})
if err != nil {
@ -114,7 +114,7 @@ func (m *Command) Reopen() error {
// Create new main with the same config.
config := m.Command.Config
m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...)
m.Command = server.NewCommand(bytes.NewReader(nil), io.Discard, io.Discard, m.commandOptions...)
m.Command.Config = config
// Run new program.
@ -240,7 +240,7 @@ func (m *Command) QueryProtobuf(indexName string, query string) (*pilosa.QueryRe
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
buf, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
@ -291,7 +291,7 @@ func Do(t testing.TB, method, urlStr string, body string) *httpResponse {
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
buf, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}

9
testhook/hook.go
View file

@ -4,7 +4,6 @@ package testhook
import (
"fmt"
"io/ioutil"
"os"
"sync"
"testing"
@ -71,7 +70,7 @@ func RunTestsWithHooks(m *testing.M) {
// TempDir creates a temp directory that will be automatically deleted when
// this test completes, using go1.14's [TB].Cleanup() if available.
func TempDir(tb testing.TB, pattern string) (path string, err error) {
path, err = ioutil.TempDir("", pattern)
path, err = os.MkdirTemp("", pattern)
if err == nil {
Cleanup(tb, func() {
os.RemoveAll(path)
@ -83,7 +82,7 @@ func TempDir(tb testing.TB, pattern string) (path string, err error) {
// TempFile creates a temp file that will be automatically deleted when
// this test completes, using go1.14's [TB].Cleanup() if available.
func TempFile(tb testing.TB, pattern string) (file *os.File, err error) {
file, err = ioutil.TempFile("", pattern)
file, err = os.CreateTemp("", pattern)
if err == nil {
path := file.Name()
Cleanup(tb, func() {
@ -99,7 +98,7 @@ func TempFile(tb testing.TB, pattern string) (file *os.File, err error) {
// path instead of the default Go TMPDIR. Only some tests use this, which is
// possibly an error...
func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err error) {
path, err = ioutil.TempDir(dir, pattern)
path, err = os.MkdirTemp(dir, pattern)
if err == nil {
Cleanup(tb, func() {
os.RemoveAll(path)
@ -113,7 +112,7 @@ func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err e
// path instead of the default Go TMPDIR. Only some tests use this, which is
// possibly an error...
func TempFileInDir(tb testing.TB, dir string, pattern string) (file *os.File, err error) {
file, err = ioutil.TempFile(dir, pattern)
file, err = os.CreateTemp(dir, pattern)
if err == nil {
path := file.Name()
Cleanup(tb, func() {

18
translate.go
View file

@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"sort"
"sync"
@ -35,13 +34,14 @@ var (
// TranslateStore is the storage for translation string-to-uint64 values.
// For BoltDB implementation an empty string will be converted into the sentinel byte slice:
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
//
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
type TranslateStore interface { // TODO: refactor this interface; readonly should be part of the type and replication should be an impl detail
io.Closer
@ -585,7 +585,7 @@ func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error) {
s.mu.Lock()
defer s.mu.Unlock()
var bytes []byte
bytes, err = ioutil.ReadAll(r)
bytes, err = io.ReadAll(r)
count = int64(len(bytes))
if err != nil {
return count, err

25
txfactory.go
View file

@ -29,7 +29,6 @@ const (
// with -2 when the Tx completes.
//
// Should be false for production.
//
const DetectMemAccessPastTx = false
var sep = string(os.PathSeparator)
@ -43,11 +42,11 @@ var sep = string(os.PathSeparator)
// The most common use of Qcx is to call GetTx() to obtain a Tx locally,
// once the index/shard pair is known:
//
// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) {
// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...})
// defer finisher(&err0)
// ...
// }
// someFunc(qcx Qcx, idx *Index, shard uint64) (err0 error) {
// tx, finisher := qcx.GetTx(Txo{Write: true, Index:idx, Shard:shard, ...})
// defer finisher(&err0)
// ...
// }
//
// Qcx reuses read-only Tx on the same index/shard pair. See
// the Qcx.GetTx() for further discussion. The caveat is of
@ -82,7 +81,6 @@ var sep = string(os.PathSeparator)
// This is then committed at the final, top-level, Qcx.Finish() call.
//
// See also the Qcx.GetTx() example and the TxGroup description below.
//
type Qcx struct {
Grp *TxGroup
Txf *TxFactory
@ -203,12 +201,12 @@ var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset
//
// Note we are tracking the returned err0 error value of someFunc(). An option instead is to say
//
// defer finisher(nil)
// defer finisher(nil)
//
// This means always Commit writes, ignoring if there were errors. This style
// is expected to be rare compared to the typical
//
// defer finisher(&err0)
// defer finisher(&err0)
//
// invocation, where err0 is your return from the enclosing function error.
// If the Tx is local and not a part of a group, then the finisher
@ -223,7 +221,6 @@ var ErrQcxDone = fmt.Errorf("Qcx already Aborted or Finished, so must call reset
// locally by another _, err := f() call. For this reason, it can
// be clearer (and much safer) to rename the enclosing functions 'err' to 'err0',
// to make it clear we are referring to the first and final error.
//
func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
if qcx.workers != nil {
qcx.workers.Block()
@ -665,14 +662,6 @@ func dirExists(name string) bool {
return false
}
func fileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
var _ = anyGlobalDBWrappersStillOpen // happy linter
func anyGlobalDBWrappersStillOpen() bool {