scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

staticcheck fixes (#2278)

Browse source
This commit is contained in:
tgruben 2022-11-07 10:51:55 -06:00 • committed by GitHub
parent 407d52baa0
commit 0aa5efcc51
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 271 additions and 276 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
authz/authorization_test.go
View file

@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,

15
boltdb/translate.go
View file

@ -59,13 +59,14 @@ var _ pilosa.TranslateStore = &TranslateStore{}
// TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
// An empty string will be converted into the sentinel byte slice:
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
//
// var emptyKey = []byte{
// 0x00, 0x00, 0x00,
// 0x4d, 0x54, 0x4d, 0x54, // MTMT
// 0x00,
// 0xc2, 0xa0, // NO-BREAK SPACE
// 0x00,
// }
type TranslateStore struct {
mu sync.RWMutex
db *bolt.DB

31
cmd/badloader/badloader.go
View file

@ -5,21 +5,19 @@ import (
"archive/tar"
"compress/gzip"
"context"
"time"
"fmt"
"io"
"io/ioutil"
"os"
"strconv"
"strings"
"time"
gohttp "net/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/vprint"
"os"
"strconv"
"strings"
)
func UploadTar(srcFile string, client *pilosa.InternalClient) error {
@ -40,8 +38,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
tarReader = tar.NewReader(f)
}
viewData := make(map[string][]byte)
//given ordered by index/field/view
//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
// given ordered by index/field/view
// trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
lastIndex := ""
lastField := ""
lastShard := uint64(0)
@ -52,7 +50,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
if header != nil {
vprint.PanicOn("header should not be nil on err io.EOF")
}
//submit any stuff we have left
// submit any stuff we have left
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
@ -69,7 +67,7 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
vprint.VV("n = %v, progress, elapsed '%v'", n, time.Since(t0))
}
parts := strings.Split(header.Name, "/")
//vv("parts = '%#v'", parts)
// vv("parts = '%#v'", parts)
index := parts[1]
field := parts[2]
view := parts[4]
@ -83,15 +81,15 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
// vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
vprint.PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
viewData = make(map[string][]byte)
//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
// vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
}
}
roaringData, err := ioutil.ReadAll(tarReader)
roaringData, err := io.ReadAll(tarReader)
if err != nil {
return err
}
@ -102,8 +100,8 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
lastIndex = index
lastField = field
//lastShard = shard
//vv("bottom of loop")
// lastShard = shard
// vv("bottom of loop")
}
}
@ -111,7 +109,6 @@ func UploadTar(srcFile string, client *pilosa.InternalClient) error {
// the new "good" loader, and should always be preferred now
// when not trying to repro that bug. pulled from 85fa67e8
func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))

4
cmd/pilosa-bench/main.go
View file

@ -6,7 +6,7 @@ import (
"expvar"
"flag"
"fmt"
"io/ioutil"
"io"
"log"
"math/rand"
"net/http"
@ -71,7 +71,7 @@ func run(ctx context.Context, args []string) (err error) {
// Clear time prefix on log.
log.SetFlags(0)
if !*verbose {
log.SetOutput(ioutil.Discard)
log.SetOutput(io.Discard)
}
// Setup PRNG to have consistent values for the same set of data.

68
cmd/roaring-migrate/main.go
View file

@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
@ -23,8 +22,10 @@ import (
"github.com/spf13/cobra"
)
var visited map[string]int64
var glogger = logger.NewStandardLogger(os.Stdout)
var (
visited map[string]int64
glogger = logger.NewStandardLogger(os.Stdout)
)
const (
Version = "1.0"
@ -33,6 +34,7 @@ const (
func main() {
os.Exit(realMain())
}
func realMain() int {
visited = make(map[string]int64)
var dataDir, backupPath string
@ -82,7 +84,6 @@ func FetchFragments(base string) []string {
var fragments []string
ff := func(pathX string, infoX os.FileInfo, errX error) error {
// first thing to do, check error. and decide what to do about it
if errX != nil {
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
@ -98,7 +99,6 @@ func FetchFragments(base string) []string {
}
err := filepath.Walk(base, ff)
if err != nil {
glogger.Errorf("error walking the path %q: %v\n", base, err)
}
@ -121,14 +121,13 @@ func fileExists(filename string) (bool, int64) {
}
func BuildSchema(dataDir string) ([]byte, error) {
//need to find all the ".meta" files and load as field options
// need to find all the ".meta" files and load as field options
schemaSerializer := struct {
Indexes []*local `json:"indexes,omitempty"`
}{Indexes: make([]*local, 0)}
var l *local
ff := func(pathX string, infoX os.FileInfo, errX error) error {
// first thing to do, check error. and decide what to do about it
if errX != nil {
glogger.Infof("error 「%v」 at a path 「%q」\n", errX, pathX)
@ -136,16 +135,16 @@ func BuildSchema(dataDir string) ([]byte, error) {
}
pathX = pathX[len(dataDir):]
if infoX.IsDir() {
//filepath.Walk(pathX, ff)
// filepath.Walk(pathX, ff)
} else {
if strings.Contains(pathX, ".meta") {
//convert the file to a fieldOptions
// convert the file to a fieldOptions
// ex: metaPath /trait_store/aba/.meta
glogger.Infof("PATHX %v", pathX)
t := strings.Split(pathX, "/")
index := t[1]
src := dataDir + pathX
content, err := ioutil.ReadFile(src)
content, err := os.ReadFile(src)
if err != nil {
return err
}
@ -165,7 +164,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
CreatedAt: uint64(CTimeNano(stat)),
Options: *io,
}
//index options
// index options
schemaSerializer.Indexes = append(schemaSerializer.Indexes, l)
return nil
}
@ -184,20 +183,20 @@ func BuildSchema(dataDir string) ([]byte, error) {
}
err := filepath.Walk(dataDir, ff)
if err != nil {
glogger.Errorf("error walking the path %q: %v\n", dataDir, err)
}
return json.MarshalIndent(schemaSerializer, "", " ")
}
func Extract(filename string) (index, field, view string, shard uint64) {
//trait_store/aba/views/standard/fragments
// trait_store/aba/views/standard/fragments
parts := strings.Split(filename, "/")
shard, _ = strconv.ParseUint(parts[6], 10, 64)
return parts[1], parts[2], parts[4], shard
}
//just a way to collect all the open dbs
// just a way to collect all the open dbs
type rbfFile struct {
working *rbf.DB
last string
@ -222,6 +221,7 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) {
}
return d.working, nil
}
func (d *rbfFile) Close() error {
defer func() error {
// clean up the temp directory
@ -235,11 +235,11 @@ func (d *rbfFile) Close() error {
if d.last != "" {
d.working.Close()
//if d.last exists only keep the biggest
// if d.last exists only keep the biggest
exists, sz := fileExists(d.last)
src := filepath.Join(d.temp, "data")
if !exists {
err := os.MkdirAll(filepath.Dir(d.last), 0750)
err := os.MkdirAll(filepath.Dir(d.last), 0o750)
if err != nil {
return err
}
@ -257,6 +257,7 @@ func (d *rbfFile) Close() error {
}
return nil
}
func copyFile(src, dest string) error {
from, err := os.Open(src)
if err != nil {
@ -264,7 +265,7 @@ func copyFile(src, dest string) error {
}
defer from.Close()
to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644)
to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0o644)
if err != nil {
return err
}
@ -280,7 +281,7 @@ func copyFile(src, dest string) error {
func Migrate(dataDir, backupPath string, verbose bool) error {
dataDir = strings.TrimSuffix(dataDir, "/")
err := os.MkdirAll(backupPath, 0750)
err := os.MkdirAll(backupPath, 0o750)
if err != nil {
return err
}
@ -293,14 +294,14 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
return err
}
err = ioutil.WriteFile(filepath.Join(backupPath, "schema"), schema, 0644)
err = os.WriteFile(filepath.Join(backupPath, "schema"), schema, 0o644)
if err != nil {
return err
}
raw := FetchFragments(dataDir)
sort.Slice(raw, func(i, j int) bool {
//trait_store/zip_code/views/standard/fragments/
// trait_store/zip_code/views/standard/fragments/
ti := strings.LastIndex(raw[i], "/") + 1
shardi, err := strconv.ParseUint(raw[i][ti:], 10, 16)
if err != nil {
@ -318,7 +319,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
}
return false
})
//raw is now sorted by shard
// raw is now sorted by shard
cache := &rbfFile{
temp: filepath.Join(backupPath, "_SCRATCH"),
@ -341,7 +342,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
if verbose {
glogger.Infof("processing: %v", dataDir+filename)
}
content, err := ioutil.ReadFile(dataDir + filename)
content, err := os.ReadFile(dataDir + filename)
if err != nil {
return err
}
@ -356,8 +357,14 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
}
key := string(txkey.Prefix(index, field, view, shard))
tx, err := db.Begin(true)
if err != nil {
return err
}
tx.AddRoaring(key, bm)
err = tx.Commit()
if err != nil {
return err
}
}
cache.Close()
keys := FetchIndexKeys(dataDir)
@ -372,7 +379,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
}
}
//deal with index field(row)keys
// deal with index field(row)keys
keys = FetchRowkeys(dataDir)
for _, filename := range keys {
glogger.Infof("field %v", filename)
@ -389,7 +396,7 @@ func Migrate(dataDir, backupPath string, verbose bool) error {
func writeIfBigger(dst string, srcFile string) error {
if stats, err := os.Stat(dst); os.IsNotExist(err) {
err = os.MkdirAll(filepath.Dir(dst), 0750)
err = os.MkdirAll(filepath.Dir(dst), 0o750)
if err != nil {
return err
}
@ -404,8 +411,9 @@ func writeIfBigger(dst string, srcFile string) error {
return copyFile(srcFile, dst)
}
}
return nil //simply skip it
return nil // simply skip it
}
func ignore(path string, items ...string) bool {
f := filepath.Base(path)
for i := range items {
@ -420,7 +428,6 @@ func FetchIndexKeys(base string) []string {
var directory []string
ff := func(pathX string, infoX os.FileInfo, errX error) error {
// first thing to do, check error. and decide what to do about it
if errX != nil {
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
@ -428,7 +435,7 @@ func FetchIndexKeys(base string) []string {
}
pathX = pathX[len(base):]
if infoX.IsDir() {
//filepath.Walk(pathX, ff)
// filepath.Walk(pathX, ff)
} else {
if strings.Contains(pathX, "_keys") && !ignore(pathX, ".data", "keys") {
directory = append(directory, pathX)
@ -438,7 +445,6 @@ func FetchIndexKeys(base string) []string {
}
err := filepath.Walk(base, ff)
if err != nil {
glogger.Errorf("error walking the path %q: %v\n", base, err)
}
@ -449,7 +455,6 @@ func FetchRowkeys(base string) []string {
var directory []string
ff := func(pathX string, infoX os.FileInfo, errX error) error {
// first thing to do, check error. and decide what to do about it
if errX != nil {
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
@ -457,13 +462,13 @@ func FetchRowkeys(base string) []string {
}
pathX = pathX[len(base):]
if infoX.IsDir() {
//filepath.Walk(pathX, ff)
// filepath.Walk(pathX, ff)
} else {
fp := filepath.Base(pathX)
if fp == "keys" {
p := strings.Split(pathX, "/")
if len(p) != 4 {
return nil //skip all but field/key files
return nil // skip all but field/key files
}
directory = append(directory, pathX)
}
@ -472,7 +477,6 @@ func FetchRowkeys(base string) []string {
}
err := filepath.Walk(base, ff)
if err != nil {
glogger.Errorf("error walking the path %q: %v\n", base, err)
}

13
cmd/roaring-migrate/main_test.go
View file

@ -1,7 +1,6 @@
package main
import (
"io/ioutil"
"os"
"testing"
)
@ -27,24 +26,27 @@ func TestFileExists(t *testing.T) {
}
func TestMainProgram(t *testing.T) {
os.Args = []string{"roaring-migrate",
os.Args = []string{
"roaring-migrate",
"--verbose",
}
if realMain() == 0 {
t.Fatal("should fail and it succeeded")
}
os.Args = []string{"roaring-migrate",
os.Args = []string{
"roaring-migrate",
"--verbose",
}
if realMain() == 0 {
t.Fatal("should fail and it succeeded")
}
dir, err := ioutil.TempDir("", "backup")
dir, err := os.MkdirTemp("", "backup")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir) // clean up
os.Args = []string{"roaring-migrate",
os.Args = []string{
"roaring-migrate",
"--verbose=true",
"--data-dir=testdata/data-dir/",
"--backup-dir=" + dir,
@ -52,5 +54,4 @@ func TestMainProgram(t *testing.T) {
if realMain() == 1 {
t.Fatal("shouldn't fail")
}
}

12
cmd/slurp/slurp.go
View file

@ -9,7 +9,6 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"net/url"
"os"
@ -54,7 +53,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
return err
}
}
roaringData, err := ioutil.ReadAll(tr)
roaringData, err := io.ReadAll(tr)
if err != nil {
return err
}
@ -85,12 +84,11 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
index := parts[1]
fieldName := parts[2]
if fieldName == "_keys" {
//skip index keys are not not real fields so will have no need for field keys
// skip index keys are not not real fields so will have no need for field keys
return nil
}
byteData, err := ioutil.ReadAll(tr)
byteData, err := io.ReadAll(tr)
vprint.PanicOn(err)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil
@ -106,7 +104,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
if err != nil {
return err
}
byteData, err := ioutil.ReadAll(tr)
byteData, err := io.ReadAll(tr)
vprint.PanicOn(err)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil
@ -121,6 +119,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
r.state = parts[0]
return nil
}
func (r *stateMachine) Upload() error {
if len(r.viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
@ -242,7 +241,6 @@ func stopProfile(host, outfile string) {
defer fd.Close()
_, err = io.Copy(fd, resp.Body)
vprint.PanicOn(err)
}
var globURI *pnet.URI

2
ctl/auth_token.go
View file

@ -168,7 +168,7 @@ func (cmd *AuthTokenCommand) Run(ctx context.Context) (err error) {
}
// Prompt the user to visit verification_uri and enter code.
fmt.Printf(formatPromptBox(dar.VerificationURI, dar.UserCode))
fmt.Print(formatPromptBox(dar.VerificationURI, dar.UserCode))
// Request a token until success or error response, slowing down if requested.
interval := dar.Interval

1
dbshard.go
View file

@ -456,7 +456,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir
//
// when a new DBShard is made, we will update the list of shards then. Thus
// the per.index2shard should always be up to date AFTER the first call here.
//
func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]struct{}, err error) {
// use the cache, always

21
idk/api/source_test.go
View file

@ -2,7 +2,6 @@ package api
import (
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
@ -47,19 +46,19 @@ func TestIngest(t *testing.T) {
unkeyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
unkeyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true))
unkeyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
//unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
// unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
unkeyedIdx.Field("int", pilosaclient.OptFieldTypeInt())
unkeyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2))
//unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
// unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
keyedIdx := schema.Index("apitestingest_keyed", pilosaclient.OptIndexTrackExistence(true), pilosaclient.OptIndexKeys(true))
keyedIdx.Field("idset")
keyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
keyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true))
keyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
//keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
// keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
keyedIdx.Field("int", pilosaclient.OptFieldTypeInt())
keyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2))
//keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
// keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
err = client.SyncSchema(schema)
if !assert.NoError(t, err) {
@ -119,13 +118,13 @@ func TestIngest(t *testing.T) {
return
}
defer resp.Body.Close()
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
defer io.Copy(io.Discard, resp.Body) //nolint: errcheck
if !assert.Equal(t, 200, resp.StatusCode) {
body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
t.Logf("request error: %s", body)
return
}
data, err := ioutil.ReadAll(resp.Body)
data, err := io.ReadAll(resp.Body)
if !assert.NoError(t, err) {
return
}
@ -190,13 +189,13 @@ func TestIngest(t *testing.T) {
return
}
defer resp.Body.Close()
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
defer io.Copy(io.Discard, resp.Body) //nolint: errcheck
if !assert.Equal(t, 200, resp.StatusCode) {
body, _ := ioutil.ReadAll(resp.Body)
body, _ := io.ReadAll(resp.Body)
t.Logf("request error: %s", body)
return
}
data, err := ioutil.ReadAll(resp.Body)
data, err := io.ReadAll(resp.Body)
if !assert.NoError(t, err) {
return
}

6
idk/cmd/molecula-consumer-github/main.go
View file

@ -7,7 +7,6 @@ import (
"expvar"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
_ "net/http/pprof"
@ -286,6 +285,7 @@ func (s *Source) issueSchema() []idk.Field {
idk.RecordTimeField{NameVal: "created_at", Layout: time.RFC3339},
}
}
func (s *Source) Close() error {
return nil
}
@ -384,9 +384,9 @@ func (m *Main) openURLReader(t time.Time) (io.ReadCloser, error) {
}
// If cache enabled, write to file first and then return.
if buf, err := ioutil.ReadAll(resp.Body); err != nil {
if buf, err := io.ReadAll(resp.Body); err != nil {
return nil, err
} else if err := ioutil.WriteFile(cachePath+".tmp", buf, 0666); err != nil {
} else if err := os.WriteFile(cachePath+".tmp", buf, 0o666); err != nil {
return nil, err
} else if err := os.Rename(cachePath+".tmp", cachePath); err != nil {
return nil, err

9
idk/common/configMap.go
View file

@ -3,7 +3,7 @@ package common
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"sync/atomic"
@ -43,11 +43,12 @@ func LaunchKafkaEventConfirmer(producer *confluent.Producer, finished *int32, it
}()
return doneChan
}
func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) {
var err error
configMap := &confluent.ConfigMap{}
if m.KafkaConfiguration != "" {
file, er := ioutil.ReadFile(m.KafkaConfiguration)
file, er := os.ReadFile(m.KafkaConfiguration)
if er != nil {
return nil, er
}
@ -84,7 +85,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) {
}
}
//SSL
// SSL
if m.KafkaSslCaLocation != "" {
err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation)
if err != nil {
@ -121,7 +122,7 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) {
}
}
//SSL
// SSL
if m.KafkaSslCaLocation != "" {
err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation)
if err != nil {

14
idk/csv/csv_test.go
View file

@ -3,7 +3,6 @@ package csv
import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
@ -257,7 +256,7 @@ func tim(t *testing.T, tstr string) time.Time {
}
func writeTempFile(t *testing.T, data string) string {
f, err := ioutil.TempFile("", "")
f, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("getting temp file: %v", err)
}
@ -321,12 +320,12 @@ func TestStreamFileNames(t *testing.T) {
}
func testDirTree(tree string) (string, error) {
tmp, err := ioutil.TempDir("", "")
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
err = os.MkdirAll(filepath.Join(tmp, tree), 0755)
err = os.MkdirAll(filepath.Join(tmp, tree), 0o755)
if err != nil {
os.RemoveAll(tmp)
return "", err
@ -490,7 +489,6 @@ func testTimestampRunner(t *testing.T, m *Main, testCase TimestampTestCase) {
check := testCase.expect.([]interface{})
testExtractRowsQuery(t, m.Index, testCase.fieldName, check)
}
// Test that out of range int values are ingested as nil when AllowIntOutOfRange is true.
@ -532,7 +530,6 @@ id__ID,negneg__Int_-10_-5,negpos__Int_-10_10,pospos__Int_5_10,negzero__Int_-10_0
}
})
}
}
// Test that out of range timestamp values are ingested as nil when AllowTimestampOutOfRange is true.
@ -556,7 +553,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15:
checker["ts4"] = []interface{}{nil, "0001-01-01T00:00:01Z", "0001-01-01T00:00:02Z", "9999-12-31T23:59:58Z", "9999-12-31T23:59:59Z", nil}
batchSizes := []int{3, 1, 4, 10}
for _, bsize := range batchSizes {
t.Run(fmt.Sprintf("batchsize=%d", bsize), func(t *testing.T) {
m := newMainOORFactory(t, file, false, false, true)
m.BatchSize = bsize
@ -573,7 +569,6 @@ id__ID,ts1__Timestamp_ns_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15:
}
})
}
}
// Tests various conditions that should halt ingest
@ -618,7 +613,8 @@ func TestFailureConditions(t *testing.T) {
{name: "int string overflow", csv: `id__ID,pospos__Int
0,"89273948723984729387492387492987"
`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}}
`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {

4
idk/datagen/common.go
View file

@ -8,7 +8,9 @@ import (
// data by providing one or more idk.Source. It also contains
// a method for getting information about the supported sources.
// Info() - return information about the generator along with
// a list of supported types
//
// a list of supported types
//
// Sources() - provided a string key and configuration, returns
// a list of primary key fields and a list of sources.
type SourceGenerator interface {

6
idk/datagen/custom.go
View file

@ -5,7 +5,6 @@ import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
@ -31,7 +30,7 @@ type Custom struct {
// NewCustom returns a new instance of Custom.
func NewCustom(cfg SourceGeneratorConfig) Sourcer {
conf := &CustomConfig{}
if bytes, err := ioutil.ReadFile(cfg.CustomConfig); err != nil {
if bytes, err := os.ReadFile(cfg.CustomConfig); err != nil {
return &Custom{err: errors.Wrap(err, "reading custom config file")}
} else if err = yaml.Unmarshal(bytes, conf); err != nil {
return &Custom{err: errors.Wrap(err, "unmarshaling custom config file")}
@ -45,7 +44,6 @@ func NewCustom(cfg SourceGeneratorConfig) Sourcer {
IDKAndGenFields: ig,
}
}
}
// CustomConfig represents the JSON/yaml configuration for the "custom" datagen source.
@ -374,7 +372,7 @@ func (cs *CustomSource) Record() (idk.Record, error) {
last = cs.record[i]
}
if cs.recordsToGenerate > 0 && cs.recordCounter >= cs.recordsToGenerate {
//break when number records produced
// break when number records produced
return nil, io.EOF
} else {
cs.recordCounter++

6
idk/datagen/custom_test.go
View file

@ -1,7 +1,6 @@
package datagen
import (
"io/ioutil"
"math/rand"
"os"
"strings"
@ -144,11 +143,10 @@ func TestGetIDKFields(t *testing.T) {
t.Fatalf("mismatched genFields at %d: got:\n%+v\nexp:\n%+v", i, gotGenField, expGenField)
}
}
}
func TestNewCustomUnmarshalling(t *testing.T) {
tmp, err := ioutil.TempFile("", "")
tmp, err := os.CreateTemp("", "")
if err != nil {
t.Fatalf("creating temp file: %v", err)
}
@ -181,7 +179,6 @@ fields:
if cc.CustomConfig.Fields[0].TimeFormat != unixFormat {
t.Fatalf("wrong time format: %s", cc.CustomConfig.Fields[0].TimeFormat)
}
}
func TestIncreasingTimestampGenerator(t *testing.T) {
@ -233,7 +230,6 @@ func TestIncreasingTimestampGenerator(t *testing.T) {
} else if recInt < testcase.min || recInt > testcase.max {
t.Fatalf("unexpected value for time: %v", recInt)
}
})
}
}

4
idk/datagen/dell.go
View file

@ -156,12 +156,12 @@ func (e *DellSource) StringWithCharset(length int, charset string) string {
return string(b)
}
//returns a string with random English alphabets of the specified length
// returns a string with random English alphabets of the specified length
func (e *DellSource) StringDell(length int) string {
return e.StringWithCharset(length, charsetDell)
}
//generates a random int between range max and min (inclusive)
// generates a random int between range max and min (inclusive)
func (e *DellSource) generateRandomInt(max int, min int) int {
return e.rand.Intn(max-min) + min
}

4
idk/datagen/hughes.go
View file

@ -122,7 +122,7 @@ func (e *HughesSource) Schema() []idk.Field {
return e.schema
}
//returns a random float between a range
// returns a random float between a range
func (e *HughesSource) floatRand(max float64, min float64) float64 {
return min + e.rand.Float64()*(max-min)
}
@ -138,7 +138,7 @@ func (e *HughesSource) StringWithCharset(length int, charset string) string {
return string(b)
}
//returns a string with random English alphabets of the specified length
// returns a string with random English alphabets of the specified length
func (e *HughesSource) SIDHughes(length int) string {
return e.StringWithCharset(length, charsetHughes)
}

12
idk/datagen/merck.go
View file

@ -665,12 +665,12 @@ func (e *MerckSource) StringWithCharset(length int, charset string) string {
return string(b)
}
//returns a string with random English alphabets of the specified length
// returns a string with random English alphabets of the specified length
func (e *MerckSource) String(length int) string {
return e.StringWithCharset(length, charset)
}
//returns a random int between a range with a 50% chance of returning a zero.
// returns a random int between a range with a 50% chance of returning a zero.
func (e *MerckSource) generate50(max int, min int) int {
num := e.rand.Intn(100)
numbertoReturn := e.rand.Intn(max-min) + min
@ -682,7 +682,7 @@ func (e *MerckSource) generate50(max int, min int) int {
}
}
//returns a random int between a range with a 10% chance of returning a zero.
// returns a random int between a range with a 10% chance of returning a zero.
func (e *MerckSource) generate10(max int, min int) int {
num := e.rand.Intn(100)
numbertoReturn := e.rand.Intn(max-min) + min
@ -694,7 +694,7 @@ func (e *MerckSource) generate10(max int, min int) int {
}
}
//returns a random float between a range with a 50% chance of returning a zero.
// returns a random float between a range with a 50% chance of returning a zero.
func (e *MerckSource) float50(max float64, min float64) float64 {
numbertoReturn := min + e.rand.Float64()*(max-min)
num := e.rand.Intn(100)
@ -706,12 +706,12 @@ func (e *MerckSource) float50(max float64, min float64) float64 {
}
}
//returns a random float between a range
// returns a random float between a range
func (e *MerckSource) floatRand(max float64, min float64) float64 {
return min + e.rand.Float64()*(max-min)
}
//returns a random int between a range
// returns a random int between a range
func (e *MerckSource) generateRandomInt(max int, min int) int {
return e.rand.Intn(max-min) + min
}

12
idk/header.go
View file

@ -42,12 +42,12 @@ var (
// destname are separated by "___", triple underscore) and converts it
// to an idk Field like:
//
// FieldTypeField {
// NameVal: sourcename,
// DestNameVal: destname,
// Thing1: Arg,
// Thing2: Arg2,
// }
// FieldTypeField {
// NameVal: sourcename,
// DestNameVal: destname,
// Thing1: Arg,
// Thing2: Arg2,
// }
//
// It does this using a variety of reflective magic. The unwritten
// rules are that all idk Fields must be structs and have their first

4
idk/idktest/idktest.go
View file

@ -3,7 +3,7 @@ package idktest
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"os"
"strings"
@ -62,7 +62,7 @@ func DoExtractQuery(pql, index string) (ExtractResponse, error) {
}
defer resp.Body.Close()
s, err := ioutil.ReadAll(resp.Body)
s, err := io.ReadAll(resp.Body)
if err != nil {
return eResp, errors.Errorf("reading response: %v", err)
}

8
idk/interfaces.go
View file

@ -893,10 +893,10 @@ func validateTimestamp(unit Unit, ts time.Time) error {
return nil
}
// validateDuration checks if the duration will overflow. Users can provide a custom epoch but
// Featurebase will ultimately convert this to some duration relative to the Unix epoch.
// So if the custom epoch + the provided value in the desired units is too far from
// Unix epoch such that it causes an interger overflow, this will return an error.
// validateDuration checks if the duration will overflow. Users can provide a custom epoch but
// Featurebase will ultimately convert this to some duration relative to the Unix epoch.
// So if the custom epoch + the provided value in the desired units is too far from
// Unix epoch such that it causes an interger overflow, this will return an error.
func validateDuration(dur int64, offset int64, granularity Unit) error {
var minInt, maxInt int64
switch granularity {

5
idk/internal/s3.go
View file

@ -2,7 +2,6 @@ package internal
import (
"bytes"
"io/ioutil"
"net/url"
"os"
"strings"
@ -57,7 +56,7 @@ func ReadFileOrURL(name string, s3client s3iface.S3API) ([]byte, error) {
}
content = buf.Bytes()
} else {
content, err = ioutil.ReadFile(name)
content, err = os.ReadFile(name)
if err != nil {
if os.IsNotExist(err) {
return nil, FileOrURLNotFound
@ -91,7 +90,7 @@ func WriteFileOrURL(name string, contents []byte, s3client s3iface.S3API) error
return errors.Wrapf(err, "putting S3 object %v", name)
}
} else {
err = ioutil.WriteFile(name, contents, 0644)
err = os.WriteFile(name, contents, 0o644)
if err != nil {
return errors.Wrapf(err, "reading file %v", name)
}

20
idk/kafka/cmd_test.go
View file

@ -5,7 +5,7 @@ package kafka
import (
"fmt"
"io/ioutil"
"io"
"math/rand"
"net/http"
"os"
@ -24,12 +24,14 @@ import (
"github.com/molecula/featurebase/v3/logger"
)
var pilosaHost string
var pilosaTLSHost string
var pilosaGrpcHost string
var kafkaHost string
var registryHost string
var certPath string
var (
pilosaHost string
pilosaTLSHost string
pilosaGrpcHost string
kafkaHost string
registryHost string
certPath string
)
func init() {
var ok bool
@ -181,7 +183,6 @@ func TestConfigOptions(t *testing.T) {
if val, err := cfg.Get("auto.offset.reset", nil); err != nil || val.(string) != "latest" {
t.Fatalf("unexpected val for auto.offset.reset val: %v, err: %v", val, err)
}
}
func TestCmdMainOne(t *testing.T) {
@ -652,6 +653,7 @@ func (s sortableCRI) Less(i, j int) bool {
}
return true
}
func (s sortableCRI) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
@ -673,7 +675,7 @@ func tDoHTTPPost(t *testing.T, url, contentType, body string) string {
t.Fatalf("making POST request: %v", err)
}
bod, err := ioutil.ReadAll(resp.Body)
bod, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("reading POST response bdoy: %v", err)
}

4
idk/kafka/csrc/csrc.go
View file

@ -4,7 +4,7 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"strings"
@ -114,7 +114,7 @@ func unmarshalRespErr(resp *http.Response, err error, into interface{}) error {
return errors.Wrap(err, "making http request")
}
if resp.StatusCode != 200 {
bod, err := ioutil.ReadAll(resp.Body)
bod, err := io.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "reading body")
}

9
idk/kafka/put.go
View file

@ -3,7 +3,6 @@ package kafka
import (
"context"
"encoding/binary"
"io/ioutil"
"os"
"time"
@ -60,7 +59,6 @@ func (p *PutCmd) Run() (err error) {
}
var auth *csrc.BasicAuth
if p.SchemaRegistryUsername != "" {
auth = &csrc.BasicAuth{
KafkaSchemaApiKey: p.SchemaRegistryUsername,
KafkaSchemaApiSecret: p.SchemaRegistryPassword,
@ -129,6 +127,7 @@ func (p *PutCmd) Run() (err error) {
return nil
}
func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer, numPartitions int, replicationFactor int) error {
a, err := confluent.NewAdminClientFromProducer(p)
if err != nil {
@ -153,7 +152,8 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer,
[]confluent.TopicSpecification{{
Topic: topic,
NumPartitions: numPartitions,
ReplicationFactor: replicationFactor}},
ReplicationFactor: replicationFactor,
}},
// Admin options
confluent.SetAdminOperationTimeout(maxDur))
if err != nil {
@ -166,6 +166,7 @@ func CreateKafkaTopic(ctx context.Context, topic string, p *confluent.Producer,
}
return nil
}
func (p *PutCmd) getSchema() (string, error) {
if p.Schema == "" && p.SchemaFile == "" {
return "", errors.New("need a string schema or schema file")
@ -173,7 +174,7 @@ func (p *PutCmd) getSchema() (string, error) {
if p.Schema != "" {
return p.Schema, nil
}
bytes, err := ioutil.ReadFile(p.SchemaFile)
bytes, err := os.ReadFile(p.SchemaFile)
if err != nil {
return "", errors.Wrap(err, "reading schema file")
}

7
idk/kafka/source.go
View file

@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
@ -1009,7 +1008,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
}
defer schemaUrlResponse.Body.Close()
if schemaUrlResponse.StatusCode >= 300 {
bod, err := ioutil.ReadAll(schemaUrlResponse.Body)
bod, err := io.ReadAll(schemaUrlResponse.Body)
if err != nil {
return nil, errors.Wrapf(err, "Failed to get schema, code: %d, no body", schemaUrlResponse.StatusCode)
}
@ -1043,7 +1042,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
s.Log.Infof("Problem getting subject/version info for schema: %v", err)
} else {
if subVerResponse.StatusCode >= 300 {
bod, err := ioutil.ReadAll(subVerResponse.Body)
bod, err := io.ReadAll(subVerResponse.Body)
s.Log.Infof("Problem getting subject/version info for schema, response: %s. Err reading body: %v", bod, err)
}
defer subVerResponse.Body.Close()
@ -1053,7 +1052,7 @@ func (s *Source) getCodec(id int32) (avro.Schema, error) {
Version int `json:"version"`
}
if bod, err := ioutil.ReadAll(subVerResponse.Body); err != nil {
if bod, err := io.ReadAll(subVerResponse.Body); err != nil {
s.Log.Infof("decoding subj/version %s body: %v", schemaSubVerUrl, err)
} else if err := json.Unmarshal(bod, &tempSchemaStruct); err != nil {
s.Log.Infof("decoding schema subject & version from registry: %v", err)

63
idk/kafka/source_test.go
View file

@ -9,7 +9,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"math/rand"
"os"
@ -81,7 +80,7 @@ func TestAvroToPDKSchema(t *testing.T) {
}
// check that we've covered all the test schemas
files, err := ioutil.ReadDir("./testdata/schemas")
files, err := os.ReadDir("./testdata/schemas")
if err != nil {
t.Fatalf("reading directory: %v", err)
}
@ -251,7 +250,7 @@ func decodeTestSchema(t *testing.T, filename string) avro.Schema {
}
func readTestSchema(t *testing.T, filename string) string {
bytes, err := ioutil.ReadFile("./testdata/schemas/" + filename)
bytes, err := os.ReadFile("./testdata/schemas/" + filename)
if err != nil {
t.Fatalf("reading schema file: %v", err)
}
@ -299,7 +298,8 @@ var tests = []struct {
},
exp: [][]interface{}{
{"a", true, int64(101), []byte{9, 196}, float64(9.4921)},
{nil, nil, nil, nil, nil}},
{nil, nil, nil, nil, nil},
},
},
{
schemaFile: "floatscale.json",
@ -435,27 +435,28 @@ func TestKafkaSourceSchemaChangeCommitRegression(t *testing.T) {
}
go func() {
topic := "xyzzy"
src.recordChannel <- recordWithError{Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
Partition: 0,
Offset: confluent.Offset(0),
src.recordChannel <- recordWithError{
Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
Partition: 0,
Offset: confluent.Offset(0),
},
Timestamp: time.Now(),
Value: data1,
},
Timestamp: time.Now(),
Value: data1,
},
}
src.recordChannel <- recordWithError{Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
Partition: 0,
Offset: confluent.Offset(1),
src.recordChannel <- recordWithError{
Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
Partition: 0,
Offset: confluent.Offset(1),
},
Timestamp: time.Now(),
Value: data2,
},
Timestamp: time.Now(),
Value: data2,
},
}
close(src.recordChannel)
}()
@ -509,14 +510,14 @@ func TestKafkaSourceTimeout(t *testing.T) {
go func() {
topic := "test"
src.recordChannel <- recordWithError{Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
src.recordChannel <- recordWithError{
Record: &confluent.Message{
TopicPartition: confluent.TopicPartition{
Topic: &topic,
},
Value: buf,
},
Value: buf,
},
}
}()
// ensure we can get a message if one is available
@ -622,7 +623,6 @@ func TestKafkaSourceIntegration(t *testing.T) {
}
})
}
}
func mustNewProducer(t *testing.T, kafkaHost string) *confluent.Producer {
@ -768,7 +768,6 @@ func TestKafkaSourceNotAutoCommitting(t *testing.T) {
} else if off := offsets[0]; int64(off.Offset) != numRecords {
t.Fatalf("after commit, offset is not %d: %d", numRecords-1, off.Offset)
}
}
func TestRegistryURLParsing(t *testing.T) {
@ -845,10 +844,8 @@ func TestRegistryURLParsing(t *testing.T) {
if codecURL != test.expectedCodecURL {
t.Errorf("codec URL exp:\n%s\ngot:\n%s", test.expectedCodecURL, codecURL)
}
})
}
}
func postSchema(t *testing.T, schemaFile, subj, regURL string, tlsConfig *tls.Config) (schemaID int) {
@ -884,7 +881,8 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) {
[]confluent.TopicSpecification{{
Topic: topic,
NumPartitions: 64,
ReplicationFactor: 1}},
ReplicationFactor: 1,
}},
// Admin options
confluent.SetAdminOperationTimeout(maxDur))
if err != nil {
@ -896,7 +894,6 @@ func tCreateTopic(t *testing.T, topic string, p *confluent.Producer) {
}
}
a.Close()
}
func tPutRecordsKafka(t *testing.T, p *confluent.Producer, topic string, schemaID int, schema *liavro.Codec, key string, records ...map[string]interface{}) {

4
idk/kafka_sasl/source.go
View file

@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"os"
"sort"
"strconv"
"sync"
@ -200,7 +200,7 @@ func (s *Source) Open() error {
return errors.New("needs header specification file")
}
headerData, err := ioutil.ReadFile(s.Header)
headerData, err := os.ReadFile(s.Header)
if err != nil {
return errors.Wrap(err, "reading header file")
}

4
idk/kafka_static/source.go
View file

@ -5,8 +5,8 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/url"
"os"
"strconv"
"strings"
"time"
@ -262,7 +262,7 @@ func (s *Source) readFileOrURL(name string) ([]byte, error) {
s.Log.Printf("read %d bytes from %s\n", bytesRead, name)
content = buf.Bytes()
} else {
content, err = ioutil.ReadFile(name)
content, err = os.ReadFile(name)
if err != nil {
return nil, errors.Wrapf(err, "reading file %v", name)
}

6
idk/kafka_static/source_test.go
View file

@ -6,9 +6,9 @@ package kafka_static
import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net"
"os"
"reflect"
"testing"
"time"
@ -68,7 +68,7 @@ func TestKafkaStaticSourceLocal(t *testing.T) {
src := NewSource()
configureSourceTestFlags(src)
{
headerData, err := ioutil.ReadFile(test.header)
headerData, err := os.ReadFile(test.header)
if err != nil {
t.Fatalf("reading header file: %v", err)
}
@ -84,7 +84,6 @@ func TestKafkaStaticSourceLocal(t *testing.T) {
for _, exp := range test.exp {
rec, err := src.Record()
if err != nil {
t.Fatalf("unexpected error getting record: %v", err)
}
@ -171,7 +170,6 @@ func TestKafkaStaticSourceIntegration(t *testing.T) {
}
})
}
}
func tCreateTopic(t *testing.T, topic string, addr net.Addr) {

7
idk/kafkagen/cmd.go
View file

@ -3,7 +3,6 @@ package kafkagen
import (
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
@ -58,7 +57,7 @@ func (m *Main) Run() (err error) {
{"2", "1", 159, map[string]interface{}{"boolean": true}, map[string]interface{}{"boolean": false}, map[string]interface{}{"string": "cgr"}, map[string]interface{}{"array": []string{"a", "b"}}, nil, map[string]interface{}{"int": 7}, nil, nil, map[string]interface{}{"float": 5.4}, nil, map[string]interface{}{"org.test.survey1234": "yes"}, map[string]interface{}{"float": 8.0}, nil},
}
//kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) {
// kakfa.SetupSasl(configMap *confluent.ConfigMap, SASLConfig SaslConfig) (err error) {
p, err := confluent.NewProducer(m.configMap)
if err != nil {
@ -102,7 +101,7 @@ func (m *Main) Run() (err error) {
return errors.Wrap(err, "putting record")
}
}
<-doneChan //wait till all messages are acked
<-doneChan // wait till all messages are acked
p.Flush(10 * 1000)
return nil
@ -125,7 +124,7 @@ func (m *Main) putRecordKafka(p *confluent.Producer, schemaID int, schema *liavr
}
func readSchema(filename string) (string, error) {
bytes, err := ioutil.ReadFile(filename)
bytes, err := os.ReadFile(filename)
if err != nil {
return "", err
}

12
internal/clustertests/pause_node_test.go
View file

@ -5,7 +5,7 @@ import (
"context"
"fmt"
"hash/fnv"
"io/ioutil"
"io"
"math/rand"
"os"
"os/exec"
@ -101,7 +101,7 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient,
if err != nil {
return err
}
buf, err := ioutil.ReadAll(r)
buf, err := io.ReadAll(r)
if err != nil {
return err
}
@ -129,7 +129,7 @@ func readIndexTranslateData(ctx context.Context, client *pilosa.InternalClient,
}
func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) {
dirEntries, err := ioutil.ReadDir(dirPath)
dirEntries, err := os.ReadDir(dirPath)
if err != nil {
return nil, err
}
@ -145,7 +145,7 @@ func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore,
// filter out non-file entries
filePaths := make([]string, 0, len(dirEntries))
for _, entry := range dirEntries {
if entry.Mode().IsDir() {
if entry.IsDir() {
continue
}
filePath := filepath.Join(dirPath, entry.Name())
@ -186,7 +186,7 @@ func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, ke
// create dir to store boltdbs for this node
nodeDirPath := filepath.Join(dirPath, node)
err = os.Mkdir(nodeDirPath, 0755)
err = os.Mkdir(nodeDirPath, 0o755)
if err != nil {
return err
}
@ -385,7 +385,7 @@ func TestPauseReplica(t *testing.T) {
t.Fatal(err)
}
dirPath = filepath.Join(dirPath, keysDirName)
err = os.Mkdir(dirPath, 0755)
err = os.Mkdir(dirPath, 0o755)
if err != nil {
t.Fatal(err)
}

13
net/uri.go
View file

@ -29,12 +29,13 @@ var (
// 3) Port: Port of the URI. Default: 10101.
//
// All parts of the URI are optional. The following are equivalent:
// http://localhost:10101
// http://localhost
// http://:10101
// localhost:10101
// localhost
// :10101
//
// http://localhost:10101
// http://localhost
// http://:10101
// localhost:10101
// localhost
// :10101
type URI struct {
Scheme string `json:"scheme"`
Host string `json:"host"`

4
pql/parser_test.go
View file

@ -217,7 +217,7 @@ func TestParser_Parse(t *testing.T) {
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
q, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`)
_, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`)
if err == nil {
t.Fatal("no error parsing invalid date")
} else if !strings.Contains(err.Error(), "not a valid timestamp") {
@ -238,7 +238,6 @@ func TestParser_Parse(t *testing.T) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
}
})
}
func TestUnquote(t *testing.T) {
@ -286,7 +285,6 @@ func TestUnquote(t *testing.T) {
}
})
}
}
func testErr(t *testing.T, exp string, actual error) (done bool) {

2
rbf/cursorx.go
View file

@ -12,7 +12,7 @@ import (
"github.com/pkg/errors"
)
//probably should just implement the container interface
// probably should just implement the container interface
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
shardVsContainerExponent := uint(4) //needs constant exported from roaring package

21
rbf/db.go
View file

@ -19,9 +19,7 @@ import (
"github.com/molecula/featurebase/v3/syswrap"
)
var (
ErrClosed = errors.New("rbf: database closed")
)
var ErrClosed = errors.New("rbf: database closed")
// shared cursor pool across all DB instances.
// Cursors are returned on Cursor.Close().
@ -118,14 +116,14 @@ func (db *DB) Open() (err error) {
db.mu.Lock()
defer db.mu.Unlock()
if err := os.MkdirAll(db.Path, 0755); err != nil {
if err := os.MkdirAll(db.Path, 0o755); err != nil {
return err
} else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil {
} else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil {
return fmt.Errorf("open file: %w", err)
}
// Open read-only database mmap.
if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0600); err != nil {
if f, err := os.OpenFile(db.DataPath(), os.O_RDONLY, 0o600); err != nil {
return fmt.Errorf("open mmap file: %w", err)
} else if db.data, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
f.Close()
@ -163,12 +161,12 @@ func (db *DB) Open() (err error) {
func (db *DB) openWAL() (err error) {
// Open WAL file writer.
if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0600); err != nil {
if db.walFile, err = os.OpenFile(db.WALPath(), os.O_WRONLY|os.O_CREATE, 0o600); err != nil {
return fmt.Errorf("open wal file: %w", err)
}
// Open read-only mmap.
if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0600); err != nil {
if f, err := os.OpenFile(db.WALPath(), os.O_RDONLY, 0o600); err != nil {
return fmt.Errorf("open wal mmap file: %w", err)
} else if db.wal, err = syswrap.Mmap(int(f.Fd()), 0, int(db.cfg.MaxWALSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
f.Close()
@ -332,7 +330,7 @@ func (db *DB) checkpoint() (err error) {
if IsBitmapHeader(page) {
pgno = readPageNo(page)
if i+1 < db.walPageN {
if page, err = db.readWALPageAt(i + 1); err != nil {
if _, err = db.readWALPageAt(i + 1); err != nil {
return err
}
} else {
@ -501,7 +499,6 @@ func (db *DB) Close() (err error) {
// We will internally create and rollback a read-only
// transaction to answer this query.
func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) {
// Read a list of all bitmaps in Tx.
tx, err := db.Begin(false)
if err != nil {
@ -578,7 +575,6 @@ func (db *DB) init() error {
// initMetaPage initializes the meta page.
func (db *DB) initMetaPage() error {
page := allocPage()
writeMetaMagic(page)
writeMetaPageN(page, 3)
@ -590,7 +586,6 @@ func (db *DB) initMetaPage() error {
// initRootRecordPage initializes the initial root record page.
func (db *DB) initRootRecordPage() error {
page := allocPage()
writePageNo(page, 1)
writeFlags(page, PageTypeRootRecord)
@ -600,7 +595,6 @@ func (db *DB) initRootRecordPage() error {
// initFreelistPage initializes the initial freelist btree page.
func (db *DB) initFreelistPage() error {
page := allocPage()
writePageNo(page, 2)
writeFlags(page, PageTypeLeaf)
@ -786,7 +780,6 @@ func (db *DB) removeTx(tx *Tx) error {
// Check performs an integrity check.
func (db *DB) Check() error {
tx, err := db.Begin(false)
if err != nil {
return err

2
rbf/page_map.go
View file

@ -1,7 +1,7 @@
/*
start with https://github.com/benbjohnson/immutable and specialize Map<uint32,int64>
Copyright 2019 Ben Johnson
# Copyright 2019 Ben Johnson
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in

4
roaring/btree.go
View file

@ -635,9 +635,9 @@ func (t *tree) Set(k uint64, v *Container) {
// (whatever, false) if it decides not to create or not to update the value of
// the KV pair.
//
// tree.Set(k, v) call conceptually equals calling
// tree.Set(k, v) call conceptually equals calling
//
// tree.Put(k, func(uint64, bool){ return v, true })
// tree.Put(k, func(uint64, bool){ return v, true })
//
// modulo the differing return values.
func (t *tree) Put(k uint64, upd func(oldV *Container, exists bool) (newV *Container, write bool)) (oldV *Container, written bool) {

6
roaring/roaring_helpers_test.go
View file

@ -7,7 +7,7 @@ import "sync"
const containerWidth = 1 << 16
////////////////// array
// //////////////// array
func arrayEmpty() []uint16 {
return make([]uint16, 0)
}
@ -76,7 +76,7 @@ func arrayEvenBitsSet() []uint16 {
return array
}
////////////////// bitmap
// //////////////// bitmap
func bitmapEmpty() []uint64 {
return make([]uint64, bitmapN)
}
@ -161,7 +161,7 @@ func bitmapEvenBitsSet() []uint64 {
return bitmap
}
////////////////// run
// //////////////// run
func runEmpty() []Interval16 {
return make([]Interval16, 0)
}

2
roaring/roaring_test.go
View file

@ -1155,7 +1155,7 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) {
}
//empty array test
// empty array test
func TestBitmap_Xor_Empty(t *testing.T) {
bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002)
empty := roaring.NewFileBitmap()

26
short_txkey/txkey.go
View file

@ -32,7 +32,7 @@ func FieldViewFromFullKey(fullKey []byte) FieldView {
// The roaringContainerKey argument to Key() is a container key into a roaring Container.
// The return value from Key() is constructed as follows:
//
// ~field;view<ckey#
// ~field;view<ckey#
//
// where ckey is always exactly 8 bytes, uint64 big-endian encoded.
//
@ -43,20 +43,21 @@ func FieldViewFromFullKey(fullKey []byte) FieldView {
// The ckey is the 8 bytes between the '<' and the '#'.
// The Prefix of a key ends at, and includes, the '<'. It is at least 13 bytes long.
// The index, field, and view are not allowed to contain these reserved bytes:
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable
// base-64 encoding.
//
// The shortest possible key is 14 bytes. It would be laid out like this:
// ~f;v<12345678#
// 12345678901234
//
// ~f;v<12345678#
// 12345678901234
//
// keys starting with '~' are regular value keys.
// keys starting with '>' are symlink keys.
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
//
func Key(index, field, view string, shard, roaringContainerKey uint64) (r []byte) {
prefix := Prefix(index, field, view, shard)
@ -100,9 +101,10 @@ func MustValidateKey(bkey []byte) {
// KeyExtractContainerKey extracts the containerKey from bkey.
// key example: field;view<ckey
// shortest: ~f;v<12345678#
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
//
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
n := len(bkey)
MustValidateKey(bkey)
@ -135,7 +137,6 @@ func Prefix(index, field, view string, shard uint64) (r []byte) {
// IndexOnlyPrefix returns a "~" prefix suitable for DeleteIndex and a key-scan to
// remove all storage. We assume only one index in this database, so delete everything.
//
func IndexOnlyPrefix(indexName string) (r []byte) {
return []byte("~")
}
@ -150,9 +151,12 @@ func FieldPrefix(index, field string) (r []byte) {
}
// PrefixFromKey key example: ~field;view<ckey#
// n-9 n-1
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
// view ckey
//
// view ckey
func PrefixFromKey(bkey []byte) (prefix []byte) {
n := len(bkey)
return bkey[:(n - 9)]

14
sql/extract.go
View file

@ -84,8 +84,10 @@ func (j parseTables) secondary() *parseTable {
// tableWhere represents a parseTable from a sql query along
// with the portion of the where clause that relates to
// that table. For example, if a sql query had:
// from tbl1, tbl2
// where tbl1.field1=1 and tbl2.field2=2
//
// from tbl1, tbl2
// where tbl1.field1=1 and tbl2.field2=2
//
// then each table would have a separate tableWhere object
// with the where made up of only the field with matching qualifier.
type tableWhere struct {
@ -846,10 +848,12 @@ func extractComparisonOp(expr *sqlparser.ComparisonExpr) pql.Token {
// extractJoinTables returns a slice of parseTable containing two
// items, the primary and secondary join tables. This function does
// not extract join tables of the form:
// from tbl1, tbl2
// The from clause must be of the form:
// from tbl1 INNER JOIN tbl2 ON ...
//
// from tbl1, tbl2
//
// The from clause must be of the form:
//
// from tbl1 INNER JOIN tbl2 ON ...
func extractJoinTables(stmt *sqlparser.Select) (parseTables, error) {
if len(stmt.From) != 1 {
return nil, errors.New("selecting from multiple tables is not supported")

2
sql3/parser/token.go
View file

@ -22,7 +22,7 @@ func init() {
// Token is the set of lexical tokens of the Go programming language.
type Token int
//TODO (pok) remove unnecessary tokens
// TODO (pok) remove unnecessary tokens
// The list of tokens.
const (
// Special tokens

4
sql3/sql_defs_between_test.go
View file

@ -1,6 +1,6 @@
package sql3_test
//BETWEEN tests
// BETWEEN tests
var betweenTests = tableTest{
table: tbl(
"between_all_types",
@ -101,7 +101,7 @@ var betweenTests = tableTest{
},
}
//NOT BETWEEN tests
// NOT BETWEEN tests
var notBetweenTests = tableTest{
table: tbl(
"not_between_all_types",

2
sql3/sql_defs_groupby_test.go
View file

@ -5,7 +5,7 @@ import (
"github.com/molecula/featurebase/v3/sql3/parser"
)
//groupby tests
// groupby tests
var groupByTests = tableTest{
table: tbl(
"groupby_test",

4
sql3/sql_defs_like_test.go
View file

@ -1,6 +1,6 @@
package sql3_test
//LIKE tests
// LIKE tests
var likeTests = tableTest{
table: tbl(
"like_all_types",
@ -83,7 +83,7 @@ var likeTests = tableTest{
},
}
//NOT LIKE tests
// NOT LIKE tests
var notLikeTests = tableTest{
table: tbl(
"not_like_all_types",

4
sql3/sql_defs_timequantum_test.go
View file

@ -1,6 +1,6 @@
package sql3_test
//time quantum insert tests
// time quantum insert tests
var timeQuantumInsertTest = tableTest{
table: tbl(
"time_quantum_insert",
@ -23,7 +23,7 @@ var timeQuantumInsertTest = tableTest{
},
}
//time quantum query tests
// time quantum query tests
var timeQuantumQueryTest = tableTest{
table: tbl(
"timeQuantumQueryTest",

4
statik/filesystem.go
View file

@ -1,9 +1,9 @@
// Copyright 2021 Molecula Corp. All rights reserved.
//
//go:generate statik -src=../lattice/build -dest=../
//
// Package statik contains static assets for the Lattice UI. `go generate` or
// `make generate-statik` will produce statik.go, which is ignored by git.
//
//go:generate statik -src=../lattice/build -dest=../
package statik
import (

1
time.go
View file

@ -533,6 +533,7 @@ func viewTimePart(v string) string {
// getLowestGranularityQuantum returns lowest granularity quantum from a list of views
// e.g.
//
// [std_2001, std_200102, std_20010203, std_2001020304] - returns "Y" since year is the lowest granularity
// [std_2001020304, std_200102, std_20010203] - returns "M", the order of views should not affect lowest granularity
func getLowestGranularityQuantum(views []string) TimeQuantum {

33
txkey/txkey.go
View file

@ -29,7 +29,7 @@ func FieldViewFromFullKey(fullKey []byte) FieldView {
// The roaringContainerKey argument to Key() is a container key into a roaring Container.
// The return value from Key() is constructed as follows:
//
// ~index%field;view:shard<ckey#
// ~index%field;view:shard<ckey#
//
// where shard and ckey are always exactly 8 bytes, uint64 big-endian encoded.
//
@ -42,20 +42,21 @@ func FieldViewFromFullKey(fullKey []byte) FieldView {
// The ckey is the 8 bytes between the '<' and the '#'.
// The Prefix of a key ends at, and includes, the '<'. It is at least 16 bytes long.
// The index, field, and view are not allowed to contain these reserved bytes:
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// {'~', '>', ';', ':', '<', '#', '$', '%', '^', '(', ')', '*', '!'}
//
// The bytes {'+', '/', '-', '_', '.', and '=' can be used in index, field, and view; to enable
// base-64 encoding.
//
// The shortest possible key is 25 bytes. It would be laid out like this:
// ~i%f;v:12345678<12345678#
// 1234567890123456789012345
//
// ~i%f;v:12345678<12345678#
// 1234567890123456789012345
//
// keys starting with '~' are regular value keys.
// keys starting with '>' are symlink keys.
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
//
func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r []byte) {
prefix := Prefix(index, field, view, shard)
@ -67,9 +68,12 @@ func Key(index, field, view string, shard uint64, roaringContainerKey uint64) (r
}
// ShardFromKey key example: index/field;view:shard<ckey
// n-9 n-1
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
// shard ckey
//
// shard ckey
func ShardFromKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
@ -121,9 +125,10 @@ func MustValidateKey(bkey []byte) {
// KeyExtractContainerKey extracts the containerKey from bkey.
// key example: index/field;view:shard<ckey
// shortest: =i%f;v:12345678<12345678#
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
//
// 1234567890123456789012345
// numbering len(bkey) - i:
// 5432109876543210987654321
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
n := len(bkey)
MustValidateKey(bkey)
@ -169,7 +174,6 @@ func Prefix(index, field, view string, shard uint64) (r []byte) {
// The full name of the index must be provided, no partial index names will work.
//
// The returned prefix is terminated by '%' and so DeleteIndex("i") will not delete the index "i2".
//
func IndexOnlyPrefix(indexName string) (r []byte) {
r = make([]byte, 0, 32)
r = append(r, '~')
@ -190,9 +194,12 @@ func FieldPrefix(index, field string) (r []byte) {
}
// PrefixFromKey key example: index/field;view:shard<ckey
// n-9 n-1
//
// n-9 n-1
//
// ... : 01234567 < 01234567 #
// shard ckey
//
// shard ckey
func PrefixFromKey(bkey []byte) (prefix []byte) {
n := len(bkey)
return bkey[:(n - 9)]