diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 46f814723..3ebd5d990 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -3,13 +3,38 @@ include: - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml +.go-cache: + variables: + GOPATH: $CI_PROJECT_DIR/.go + cache: + - key: $CI_COMMIT_REF_SLUG + paths: + - .go/pkg/mod/ variables: GOVERSION: "1.16.9" stages: + - lint - test - build + #before_script: + #- echo "before_script" + #- git version + #- go env -w GOPRIVATE=github.com/molecula + #- mkdir -p .go + #- go version + #- go env -w GO111MODULE=on + +golangci-lint: + image: golangci/golangci-lint:v1.39.0 + stage: lint + extends: .go-cache + allow_failure: false + script: + - echo "Checking for issues in new code" + - golangci-lint run -v + install lattice: stage: test image: node:14 @@ -52,9 +77,10 @@ run jest tests: run go tests: stage: test image: golang:1.16.9 + extends: .go-cache script: - echo "Running featurebase unit tests..." - - go test -covermode=atomic -coverprofile=coverage.out + - ./cover-everything.sh artifacts: paths: - coverage.out @@ -69,18 +95,6 @@ run go tests with output: paths: - test-report.out -generate coverage html: - stage: test - image: golang:1.16.9 - script: - - echo "Generating coverage HTML.." - - go tool cover -html coverage.out -o coverage.html - artifacts: - paths: - - coverage.html - needs: - - job: run go tests - upload to sonarcloud: stage: test image: sonarsource/sonar-scanner-cli:4.6 @@ -96,6 +110,7 @@ upload to sonarcloud: build for linux amd64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + extends: .go-cache variables: GOOS: "linux" GOARCH: "amd64" @@ -108,6 +123,7 @@ build for linux amd64: build for linux arm64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + extends: .go-cache variables: GOOS: "linux" GOARCH: "arm64" @@ -120,6 +136,7 @@ build for linux arm64: build for darwin amd64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + extends: .go-cache variables: GOOS: "darwin" GOARCH: "amd64" @@ -132,6 +149,7 @@ build for darwin amd64: build for darwin arm64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + extends: .go-cache variables: GOOS: "darwin" GOARCH: "arm64" diff --git a/.golangci.yml b/.golangci.yml index 3c51570cc..ca1abeed0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,60 @@ run: + #skip the protobuf generated files + deadline: 5m + timeout: 5m + skip-dirs-use-default: true + skip-dirs: + - pb + - proto skip-files: - pql/pql.peg.go +linters: + enable: + - govet + - gofmt + enable-all: false + disable-all: true + +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + # print lines of code with issue, default is true + print-issued-lines: true + # print linter name in the end of issue text, default is true + print-linter-name: true + +linters-settings: + gofmt: + simplify: true + govet: + # report about shadowed variables + check-shadowing: true + + + # settings per analyzer + settings: + printf: # analyzer name, run `go tool vet help` to see all analyzers + funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + + # enable or disable analyzers by name + # run `go tool vet help` to see all analyzers + enable: + - atomicalign + enable-all: false + disable: + - shadow + disable-all: false + + +issues: + exclude-use-default: false + max-issues-per-linter: 0 + max-same-issues: 0 + exclude: + - 'declaration of "(err|ctx)" shadows declaration at' + - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked' + diff --git a/cache.go b/cache.go index 7991d1aaa..00c70a129 100644 --- a/cache.go +++ b/cache.go @@ -398,8 +398,8 @@ func (p PairField) ToRows(callback func(*pb.RowResponse) error) error { {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, }, }) } else { @@ -409,8 +409,8 @@ func (p PairField) ToRows(callback func(*pb.RowResponse) error) error { {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}}, }, }) } @@ -540,8 +540,8 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error { if err := callback(&pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -549,8 +549,8 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error { if err := callback(&pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}}, }}); err != nil { return errors.Wrap(err, "calling callback") } diff --git a/client/batch.go b/client/batch.go index 19cdc52d9..0b2ae36bb 100644 --- a/client/batch.go +++ b/client/batch.go @@ -672,9 +672,9 @@ func (b *Batch) Import() error { return errors.Wrap(err, "starting transaction") } defer func() { - trns, err := b.client.FinishTransaction(trns.ID) + trnsl, err := b.client.FinishTransaction(trns.ID) if err != nil { - b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trns) + b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) } b.client.Stats.Timing(MetricBatchImportDurationSeconds, time.Since(start), 1.0) }() @@ -729,9 +729,9 @@ func (b *Batch) Flush() error { return errors.Wrap(err, "starting transaction") } defer func() { - trns, err := b.client.FinishTransaction(trns.ID) + trnsl, err := b.client.FinishTransaction(trns.ID) if err != nil { - b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trns) + b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) } b.client.Stats.Timing(MetricBatchFlushDurationSeconds, time.Since(start), 1.0) }() @@ -1053,17 +1053,17 @@ func (b *Batch) doImport(frags, clearFrags fragments) error { eg.Go(func() error { clearViewMap := clearFrags.GetViewMap(shard, field) if len(clearViewMap) > 0 { - start := time.Now() + startx := time.Now() err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, clearViewMap, true) if err != nil { return errors.Wrapf(err, "import clearing clearing data for %s", field) } - b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start)) + b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(startx)) } - start := time.Now() + starty := time.Now() err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, viewMap, false) - b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start)) + b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) return errors.Wrapf(err, "importing data for %s", field) }) } diff --git a/client/client.go b/client/client.go index cac4b9ec3..4076069c2 100644 --- a/client/client.go +++ b/client/client.go @@ -180,8 +180,8 @@ func newClientWithCluster(cluster *Cluster, options *ClientOptions) *Client { func newClientWithURI(uri *pnet.URI, options *ClientOptions) *Client { client := newClientWithOptions(options) if options.manualServerAddress { - fragmentNode := newFragmentNodeFromURI(uri) - client.manualFragmentNode = &fragmentNode + fNode := newFragmentNodeFromURI(uri) + client.manualFragmentNode = &fNode client.manualServerURI = uri client.cluster = NewClusterWithHost() } diff --git a/client/orm_test.go b/client/orm_test.go index cbb7f0c0b..112d7556f 100644 --- a/client/orm_test.go +++ b/client/orm_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/pql" "github.com/pkg/errors" ) @@ -122,15 +122,15 @@ func TestORM(t *testing.T) { }) t.Run("NewIndexOptions", func(t *testing.T) { - schema := NewSchema() + schemal := NewSchema() // test the defaults - index := schema.Index("index-default-options") + index := schemal.Index("index-default-options") target := `{"options":{}}` if target != index.options.String() { t.Fatalf("%s != %s", target, index.options.String()) } - index = schema.Index("index-keys", OptIndexKeys(true)) + index = schemal.Index("index-keys", OptIndexKeys(true)) if true != index.Opts().Keys() { t.Fatalf("index keys %v != %v", true, index.Opts().Keys()) } @@ -139,7 +139,7 @@ func TestORM(t *testing.T) { t.Fatalf("%s != %s", target, index.options.String()) } - index = schema.Index("index-trackexistence", OptIndexTrackExistence(false)) + index = schemal.Index("index-trackexistence", OptIndexTrackExistence(false)) if false != index.Opts().TrackExistence() { t.Fatalf("index trackExistene %v != %v", true, index.Opts().TrackExistence()) } diff --git a/cluster.go b/cluster.go index 2843b732f..9b161ea41 100644 --- a/cluster.go +++ b/cluster.go @@ -1534,7 +1534,7 @@ func (c *cluster) translateFieldListIDs(field *Field, ids []uint64) (keys []stri // TODO: remove this when it is no longer used func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string, writable bool) (uint64, error) { - keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}}, writable) + keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: {}}, writable) if err != nil { return 0, err } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 3d204ae92..5092cba73 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,6 +15,7 @@ package pilosa import ( + "context" "fmt" "math/rand" "reflect" @@ -625,3 +626,14 @@ func TestAE(t *testing.T) { } }) } + +func TestTranslateIndexKey(t *testing.T) { + c := newCluster() + node0 := &topology.Node{ID: "node0"} + c.addNodeBasicSorted(node0) + c.holder = newHolderWithTempPath(t, "rbf") + _, e := c.translateIndexKey(context.Background(), "i", "a", false) + if e == nil { + t.Fatal("expecting error") + } +} diff --git a/cmd/roaring-migrate/ctim_linux.go b/cmd/roaring-migrate/ctim_linux.go index b9e5d70e1..09a68435c 100644 --- a/cmd/roaring-migrate/ctim_linux.go +++ b/cmd/roaring-migrate/ctim_linux.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build linux // +build linux package main diff --git a/const_amd64.go b/const_amd64.go index 5298800ff..e0cc22abb 100644 --- a/const_amd64.go +++ b/const_amd64.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build amd64 // +build amd64 package pilosa diff --git a/cover-everything.sh b/cover-everything.sh new file mode 100755 index 000000000..f34640cfb --- /dev/null +++ b/cover-everything.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# actually get test coverage for every single package and subpackage +# very slow but oh well what are you gonna do, not test things? +echo "mode: atomic" > coverage.out +for pkg in $(go list all | grep featurebase); do + go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; + tail -n +2 pkgcoverage.out >> coverage.out; done diff --git a/ctl/config.go b/ctl/config.go index ed89dd481..830e9f526 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -19,9 +19,9 @@ import ( "fmt" "io" - toml "github.com/pelletier/go-toml" "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/server" + toml "github.com/pelletier/go-toml" ) // ConfigCommand represents a command for printing a default config. diff --git a/ctl/generate_config.go b/ctl/generate_config.go index f9f28bac9..c35dcc2b6 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -19,9 +19,9 @@ import ( "fmt" "io" - "github.com/pelletier/go-toml" "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/server" + "github.com/pelletier/go-toml" "github.com/pkg/errors" ) diff --git a/executor.go b/executor.go index 82f215fef..f99a0f667 100644 --- a/executor.go +++ b/executor.go @@ -2620,7 +2620,7 @@ func (r RowIdentifiers) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: key}}, + {ColumnVal: &proto.ColumnResponse_StringVal{StringVal: key}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -2632,7 +2632,7 @@ func (r RowIdentifiers) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Uint64Val{Uint64Val: uint64(id)}}, + {ColumnVal: &proto.ColumnResponse_Uint64Val{Uint64Val: uint64(id)}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -7362,7 +7362,7 @@ func (s SignedRow) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}}, }, }); err != nil { return errors.Wrap(err, "calling callback") @@ -7380,7 +7380,7 @@ func (s SignedRow) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}}, }, }); err != nil { return errors.Wrap(err, "calling callback") @@ -7480,8 +7480,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_DecimalVal{DecimalVal: &proto.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}}, - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, + {ColumnVal: &proto.ColumnResponse_DecimalVal{DecimalVal: &proto.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -7493,8 +7493,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Float64Val{Float64Val: v.FloatVal}}, - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, + {ColumnVal: &proto.ColumnResponse_Float64Val{Float64Val: v.FloatVal}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -7506,8 +7506,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: v.TimestampVal.Format(time.RFC3339Nano)}}, - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, + {ColumnVal: &proto.ColumnResponse_StringVal{StringVal: v.TimestampVal.Format(time.RFC3339Nano)}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -7519,8 +7519,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error { if err := callback(&proto.RowResponse{ Headers: ci, Columns: []*proto.ColumnResponse{ - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Val}}, - &proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Val}}, + {ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}}, }}); err != nil { return errors.Wrap(err, "calling callback") } diff --git a/executor_test.go b/executor_test.go index ba357d724..24395df8c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -8184,3 +8184,113 @@ func TestExternalLookup(t *testing.T) { } }) } +func TestToRows(t *testing.T) { + ids := &pilosa.RowIdentifiers{ + Rows: []uint64{1, 2, 3}, + } + c := ids.Clone() + if c == nil { + t.Fatal("Shouldn't be nil ") + } + e := ids.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + + keys := &pilosa.RowIdentifiers{ + Keys: []string{"a", "b"}, + } + c = keys.Clone() + if c == nil { + t.Fatal("Shouldn't be nil ") + } + e = keys.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + v := &pilosa.ValCount{ + TimestampVal: time.Now(), + Count: 1, + } + x := v.Clone() + if x == nil { + t.Fatal("Shouldn't be nil ") + } + e = v.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + v.DecimalVal = &pql.Decimal{Value: 1, Scale: 1} + e = v.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + v.DecimalVal = nil + v.FloatVal = 3.0 + e = v.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + + pfi := &pilosa.PairField{ + Pair: pilosa.Pair{ID: 1, Count: 1}, + Field: "f", + } + z := pfi.Clone() + if z.Pair.ID != pfi.Pair.ID { + t.Fatal("Should be equal ", z, pfi) + } + e = pfi.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + pfk := &pilosa.PairField{ + Pair: pilosa.Pair{Key: "a", Count: 1}, + Field: "f", + } + o := pfk.Clone() + if o.Pair.Key != pfk.Pair.Key { + t.Fatal("Should be equal ") + } + e = pfk.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + pfs := &pilosa.PairsField{ + Pairs: []pilosa.Pair{{ID: 1, Count: 1}}, + Field: "f", + } + f := pfs.Clone() + if f.Pairs[0].ID != pfs.Pairs[0].ID { + t.Fatal("Should be equal ") + } + e = pfs.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + + r4 := server.ResultUint64(1) + e = r4.ToRows(func(*proto.RowResponse) error { + return nil + }) + if e != nil { + t.Fatal("Shouldn't be err ", e) + } + +} diff --git a/fragment.go b/fragment.go index d3c52c7dc..674f2da65 100644 --- a/fragment.go +++ b/fragment.go @@ -961,9 +961,9 @@ func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err er err = f.gen.Transaction(wp, func() error { var rowChanged bool for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ { - if changed, err := f.unprotectedClearRow(tx, rowID); err != nil { + if chang, err := f.unprotectedClearRow(tx, rowID); err != nil { return errors.Wrapf(err, "clearing row: %d", rowID) - } else if changed { + } else if chang { rowChanged = true } } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 23c2063f1..a878091b7 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -5375,25 +5375,25 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { eg := &errgroup.Group{} eg.Go(func() error { - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + ltx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) for i := uint64(0); i < 1000; i++ { - _, err := f.setBit(tx, i%4, i) + _, err := f.setBit(ltx, i%4, i) if err != nil { return errors.Wrap(err, "setting bit") } } - PanicOn(tx.Commit()) + PanicOn(ltx.Commit()) return nil }) // need read-only Tx so as not to block on the writer finishing above. - tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - defer tx.Rollback() + tx1 := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + defer tx1.Rollback() acc := uint64(0) for i := uint64(0); i < 100; i++ { - r := f.mustRow(tx, i%4) + r := f.mustRow(tx1, i%4) acc += r.Count() } if err := eg.Wait(); err != nil { diff --git a/generation_nodebug.go b/generation_nodebug.go index a2d2dd2f6..03ec5d5e3 100644 --- a/generation_nodebug.go +++ b/generation_nodebug.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !generationdebug // +build !generationdebug package pilosa diff --git a/go.mod b/go.mod index 25e8925e5..e3c2e395e 100644 --- a/go.mod +++ b/go.mod @@ -49,9 +49,8 @@ require ( go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 golang.org/x/mod v0.4.2 - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.3.0 // indirect modernc.org/mathutil v1.0.0 diff --git a/go.sum b/go.sum index 7f90a4524..38f2e4578 100644 --- a/go.sum +++ b/go.sum @@ -410,8 +410,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -446,7 +446,7 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -454,8 +454,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= diff --git a/holder.go b/holder.go index 756d75443..909bd6ecd 100644 --- a/holder.go +++ b/holder.go @@ -2082,7 +2082,7 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { if !recurse { continue } - fragNums := fragNums[:0] + fragNums = fragNums[:0] view.mu.Lock() for fragNum := range view.fragments { fragNums = append(fragNums, fragNum) diff --git a/http/handler.go b/http/handler.go index 59a3c3162..a9c5d2010 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1326,7 +1326,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos := fieldOptionsToFunctionalOpts(req.Options) field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...) - if _, ok := err.(pilosa.BadRequestError); ok { + if _, ok = err.(pilosa.BadRequestError); ok { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/main_test.go b/main_test.go index 90f3cf214..73254f4e8 100644 --- a/main_test.go +++ b/main_test.go @@ -17,9 +17,9 @@ package pilosa_test import ( "fmt" "net" - "net/http" "testing" + "net/http" _ "net/http/pprof" "github.com/molecula/featurebase/v2/testhook" diff --git a/rbf.go b/rbf.go index cee25f35c..910ecce78 100644 --- a/rbf.go +++ b/rbf.go @@ -254,25 +254,25 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove boo if rc.N() == 0 { return 0, nil } - rc, chng := rc.Remove(lo) + rc1, chng := rc.Remove(lo) if !chng { return 0, nil } - if rc.N() == 0 { + if rc1.N() == 0 { err = tx.tx.RemoveContainer(name, hi) } else { - err = tx.tx.PutContainer(name, hi, rc) + err = tx.tx.PutContainer(name, hi, rc1) } if err != nil { return 0, err } return 1, nil } else { - rc, chng := rc.Add(lo) + rc2, chng := rc.Add(lo) if !chng { return 0, nil } - err = tx.tx.PutContainer(name, hi, rc) + err = tx.tx.PutContainer(name, hi, rc2) if err != nil { return 0, err } diff --git a/rbf/cfg/os.go b/rbf/cfg/os.go index 28605af8b..88bb68891 100644 --- a/rbf/cfg/os.go +++ b/rbf/cfg/os.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !386 // +build !386 package cfg diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 8c321fcec..6694cd026 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -18,6 +18,7 @@ import ( "io" "math/bits" "math/rand" + "os" "reflect" "sort" "strings" @@ -558,10 +559,12 @@ func TestCursor_RLETesting(t *testing.T) { } want := []uint16{0} - if got, want := c.Values(), want; !reflect.DeepEqual(got, want) { + got := c.Values() + if !reflect.DeepEqual(got, want) { t.Fatalf("Values()=%#v, want %#v", got, want) - } else if got, want := c.Key(), uint64(1); !reflect.DeepEqual(got, want) { - t.Fatalf("Key()=%#v, want %#v", got, want) + } + if g, w := c.Key(), uint64(1); !reflect.DeepEqual(g, w) { + t.Fatalf("Key()=%#v, want %#v", g, w) } }) } @@ -845,6 +848,25 @@ func (e *EasyWalker) String() string { return e.path.String() } +func TestDumpDot(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + c, err := tx.Cursor("x") + if err != nil { + t.Fatal(err) + } + + _, err = c.Add(1) + if err != nil { + t.Fatal(err) + } + rbf.Dumpdot(tx, 0, " ", os.Stdout) +} func TestCursor_UpdateBranchCells(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) @@ -962,6 +984,9 @@ func TestCursor_SplitBranchCells(t *testing.T) { t.Fatalf("Expecting RBLL (a branch split) got %v", after.String()) } + // + c, _ := tx.Cursor("x") //added just for dot code coverage + c.Dump("ignore for coverage") } @@ -1140,7 +1165,7 @@ func TestForEachRange(t *testing.T) { } rb.Put(0, roaring.NewContainerBitmap(n, bits)) - crun := roaring.NewContainerRun([]roaring.Interval16{roaring.Interval16{Start: 0, Last: 1<<16 - 1}}) + crun := roaring.NewContainerRun([]roaring.Interval16{{Start: 0, Last: 1<<16 - 1}}) rb.Put(1, crun) rb.Put(2, roaring.NewContainerArray([]uint16{1, 1024, 1<<16 - 1})) diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 9b2a61111..5fcd2ae7c 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -106,7 +106,7 @@ func (c *Cursor) Dump(name string) { fmt.Fprintf(bufStdout, "rankdir=\"LR\"\n") fmt.Fprintf(bufStdout, "node [shape=record height=.1]\n") - dumpdot(c.tx, 0, " ", bufStdout) + Dumpdot(c.tx, 0, " ", bufStdout) fmt.Fprintf(bufStdout, "\n}") bufStdout.Flush() } diff --git a/rbf/dot.go b/rbf/dot.go index 74953e6ff..69ef11961 100644 --- a/rbf/dot.go +++ b/rbf/dot.go @@ -60,8 +60,8 @@ func dotCell(b []byte, parent string, writer io.Writer) { } } -// dumpdot recursively writes the tree representation starting from a given page to STDERR. -func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { +// Dumpdot recursively writes the tree representation starting from a given page to STDERR. +func Dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { page, _, err := tx.readPage(pgno) if err != nil { panic(err) @@ -78,8 +78,8 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { for _, record := range records { root := fmt.Sprintf("root%d", record.Pgno) fmt.Fprintf(writer, "%s[label=\"ROOT(%d)| %s\"]\n%s->%s\n", root, record.Pgno, record.Name, rr, root) - parent := fmt.Sprintf("root%d", record.Pgno) - dumpdot(tx, record.Pgno, parent, writer) + p := fmt.Sprintf("root%d", record.Pgno) + Dumpdot(tx, record.Pgno, p, writer) } } @@ -96,7 +96,7 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) { for i, n := 0, readCellN(page); i < n; i++ { cell := readBranchCell(page, i) if cell.Flags&uint32(ContainerTypeBitmap) == 0 { // leaf/branch child page - dumpdot(tx, cell.ChildPgno, p, writer) + Dumpdot(tx, cell.ChildPgno, p, writer) } else { b := fmt.Sprintf("bm%d", cell.ChildPgno) fmt.Fprintf(writer, "%s[label=\"BITMAP(%d) key=%d \"]\n %s -> %s\n", b, cell.ChildPgno, cell.LeftKey, p, b) diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index a5baff1e7..5626e67cd 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -39,7 +39,7 @@ func TestReadWriteRootRecord(t *testing.T) { // Write records. if remaining, err := rbf.WriteRootRecord(buf, &rbf.RootRecord{Pgno: 10, Name: "foo"}); err != nil { t.Fatal(err) - } else if remaining, err := rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 11, Name: "bar"}); err != nil { + } else if remaining, err = rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 11, Name: "bar"}); err != nil { t.Fatal(err) } else if _, err := rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 12, Name: "baz"}); err != io.ErrShortBuffer { t.Fatalf("unexpected error: %#v", err) // buffer too short @@ -50,7 +50,7 @@ func TestReadWriteRootRecord(t *testing.T) { t.Fatal(err) } else if got, want := *rec, (rbf.RootRecord{Pgno: 10, Name: "foo"}); got != want { t.Fatalf("ReadRootRecord=%#v, want %#v", got, want) - } else if rec, remaining, err := rbf.ReadRootRecord(remaining); err != nil { + } else if rec, remaining, err = rbf.ReadRootRecord(remaining); err != nil { t.Fatal(err) } else if got, want := *rec, (rbf.RootRecord{Pgno: 11, Name: "bar"}); got != want { t.Fatalf("ReadRootRecord=%#v, want %#v", got, want) diff --git a/roaring/add_test.go b/roaring/add_test.go index 1da117c56..46e1e0d62 100644 --- a/roaring/add_test.go +++ b/roaring/add_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !race // +build !race package roaring diff --git a/roaring/generation_nodebug.go b/roaring/generation_nodebug.go index 4ce3f4ab1..7b507dcbe 100644 --- a/roaring/generation_nodebug.go +++ b/roaring/generation_nodebug.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !generationdebug // +build !generationdebug package roaring diff --git a/roaring/nop_inst.go b/roaring/nop_inst.go index ad02831f8..73a8bd99e 100644 --- a/roaring/nop_inst.go +++ b/roaring/nop_inst.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !btreeInstrumentation // +build !btreeInstrumentation package roaring diff --git a/roaring/roaring.go b/roaring/roaring.go index 4fd8a50c0..bc4c57214 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2402,13 +2402,13 @@ func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log b } err = nil if log && changed > 0 { - op := op{opN: changed, roaring: itr.Data()} + o := op{opN: changed, roaring: itr.Data()} if clear { - op.typ = opTypeRemoveRoaring + o.typ = opTypeRemoveRoaring } else { - op.typ = opTypeAddRoaring + o.typ = opTypeAddRoaring } - err = b.writeOp(&op) + err = b.writeOp(&o) } return changed, rowSet, err } @@ -4559,26 +4559,26 @@ func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) { j++ } else if va.Last > vb.Last && va.Start >= vb.Start { // |--vb-|-|-va--| - for i := int(va.Start); i <= int(vb.Last); i++ { - fn(uint16(i)) + for z := int(va.Start); z <= int(vb.Last); z++ { + fn(uint16(z)) } j++ } else if va.Last > vb.Last && va.Start < vb.Start { // |--va|--vb--|--| - for i := int(vb.Start); i <= int(vb.Last); i++ { - fn(uint16(i)) + for z := int(vb.Start); z <= int(vb.Last); z++ { + fn(uint16(z)) } j++ } else if va.Last <= vb.Last && va.Start >= vb.Start { // |--vb|--va--|--| - for i := int(va.Start); i <= int(va.Last); i++ { - fn(uint16(i)) + for z := int(va.Start); z <= int(va.Last); z++ { + fn(uint16(z)) } i++ } else if va.Last <= vb.Last && va.Start < vb.Start { // |--va-|-|-vb--| - for i := int(vb.Start); i <= int(va.Last); i++ { - fn(uint16(i)) + for z := int(vb.Start); z <= int(va.Last); z++ { + fn(uint16(z)) } i++ } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index c23adc6f4..da65e2306 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -34,6 +34,29 @@ func (iv Interval16) String() string { return fmt.Sprintf("[%d, %d]", iv.Start, iv.Last) } +func GetRoaringIter(bitsToSet ...uint64) RoaringIterator { + + b := NewBitmap() + changed := b.DirectAddN(bitsToSet...) + n := len(bitsToSet) + if changed != n { + e := fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n) + panic(e) + } + buf := bytes.NewBuffer(make([]byte, 0, 100000)) + _, er := b.WriteTo(buf) + if er != nil { + if er != nil { + panic(er) + } + } + itr, err := NewRoaringIterator(buf.Bytes()) + if err != nil { + panic(err) + } + return itr +} + func TestRunAppendInterval(t *testing.T) { a := NewContainerRun(nil) tests := []struct { @@ -99,9 +122,10 @@ func TestContainerRunAdd(t *testing.T) { {0, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 7}, {Start: 10, Last: 10}}}, {8, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 8}, {Start: 10, Last: 10}}}, } + var changed bool for _, test := range tests { c.setMapped(true) - c, changed := c.add(test.op) + c, changed = c.add(test.op) if !changed { t.Fatalf("result of adding new bit should be true: %v", c.runs()) } @@ -731,65 +755,65 @@ func TestUnionInterval16InPlace(t *testing.T) { }{ { name: "firstBitUnset lastBitSet", - a: []Interval16{Interval16{1, 10}}, - b: []Interval16{Interval16{10, 10}}, - expected: []Interval16{Interval16{1, 10}}, + a: []Interval16{{1, 10}}, + b: []Interval16{{10, 10}}, + expected: []Interval16{{1, 10}}, expectedN: 10, }, { name: "single overlap", - a: []Interval16{Interval16{1, 10}, Interval16{21, 28}}, - b: []Interval16{Interval16{8, 12}}, - expected: []Interval16{Interval16{1, 12}, Interval16{21, 28}}, + a: []Interval16{{1, 10}, {21, 28}}, + b: []Interval16{{8, 12}}, + expected: []Interval16{{1, 12}, {21, 28}}, expectedN: 20, }, { name: "nested intervals", - a: []Interval16{Interval16{3, 13}, Interval16{17, 20}}, - b: []Interval16{Interval16{1, 4}, Interval16{6, 7}, Interval16{8, 9}, Interval16{10, 11}, Interval16{14, 17}}, - expected: []Interval16{Interval16{1, 20}}, + a: []Interval16{{3, 13}, {17, 20}}, + b: []Interval16{{1, 4}, {6, 7}, {8, 9}, {10, 11}, {14, 17}}, + expected: []Interval16{{1, 20}}, expectedN: 20, }, { name: "no overlap", - a: []Interval16{Interval16{3, 4}, Interval16{7, 8}}, - b: []Interval16{Interval16{1, 2}, Interval16{5, 6}, Interval16{9, 10}}, - expected: []Interval16{Interval16{1, 10}}, + a: []Interval16{{3, 4}, {7, 8}}, + b: []Interval16{{1, 2}, {5, 6}, {9, 10}}, + expected: []Interval16{{1, 10}}, expectedN: 10, }, { name: "b in a", - a: []Interval16{Interval16{1, 10}}, - b: []Interval16{Interval16{5, 7}}, - expected: []Interval16{Interval16{1, 10}}, + a: []Interval16{{1, 10}}, + b: []Interval16{{5, 7}}, + expected: []Interval16{{1, 10}}, expectedN: 10, }, { name: "a eq b", - a: []Interval16{Interval16{1, 10}}, - b: []Interval16{Interval16{1, 10}}, - expected: []Interval16{Interval16{1, 10}}, + a: []Interval16{{1, 10}}, + b: []Interval16{{1, 10}}, + expected: []Interval16{{1, 10}}, expectedN: 10, }, { name: "a in b", - a: []Interval16{Interval16{5, 7}}, - b: []Interval16{Interval16{1, 10}}, - expected: []Interval16{Interval16{1, 10}}, + a: []Interval16{{5, 7}}, + b: []Interval16{{1, 10}}, + expected: []Interval16{{1, 10}}, expectedN: 10, }, { name: "a ahead b", - a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, - b: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}}, - expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}}, + a: []Interval16{{1, 2}, {3, 4}, {5, 7}}, + b: []Interval16{{10, 11}, {12, 13}, {14, 15}}, + expected: []Interval16{{1, 7}, {10, 15}}, expectedN: 13, }, { name: "b ahead a", - a: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}}, - b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, - expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}}, + a: []Interval16{{10, 11}, {12, 13}, {14, 15}}, + b: []Interval16{{1, 2}, {3, 4}, {5, 7}}, + expected: []Interval16{{1, 7}, {10, 15}}, expectedN: 13, }, { @@ -802,78 +826,78 @@ func TestUnionInterval16InPlace(t *testing.T) { { name: "empty a", a: []Interval16{}, - b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, - expected: []Interval16{Interval16{1, 7}}, + b: []Interval16{{1, 2}, {3, 4}, {5, 7}}, + expected: []Interval16{{1, 7}}, expectedN: 7, }, { name: "empty b", - a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}}, + a: []Interval16{{1, 2}, {3, 4}, {5, 7}}, b: []Interval16{}, - expected: []Interval16{Interval16{1, 7}}, + expected: []Interval16{{1, 7}}, expectedN: 7, }, { name: "single a", - a: []Interval16{Interval16{1, 2}}, + a: []Interval16{{1, 2}}, b: []Interval16{}, - expected: []Interval16{Interval16{1, 2}}, + expected: []Interval16{{1, 2}}, expectedN: 2, }, { name: "single b", a: []Interval16{}, - b: []Interval16{Interval16{1, 2}}, - expected: []Interval16{Interval16{1, 2}}, + b: []Interval16{{1, 2}}, + expected: []Interval16{{1, 2}}, expectedN: 2, }, { name: "single a single b", - a: []Interval16{Interval16{3, 4}}, - b: []Interval16{Interval16{1, 2}}, - expected: []Interval16{Interval16{1, 4}}, + a: []Interval16{{3, 4}}, + b: []Interval16{{1, 2}}, + expected: []Interval16{{1, 4}}, expectedN: 4, }, { name: "oddBitsSet lastBitUnset", - a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}}, - b: []Interval16{Interval16{0, 4}}, - expected: []Interval16{Interval16{0, 5}}, + a: []Interval16{{1, 1}, {3, 3}, {5, 5}}, + b: []Interval16{{0, 4}}, + expected: []Interval16{{0, 5}}, expectedN: 6, }, { name: "all bits", - a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}}, - b: []Interval16{Interval16{0, 0}, Interval16{2, 2}, Interval16{4, 4}}, - expected: []Interval16{Interval16{0, 5}}, + a: []Interval16{{1, 1}, {3, 3}, {5, 5}}, + b: []Interval16{{0, 0}, {2, 2}, {4, 4}}, + expected: []Interval16{{0, 5}}, expectedN: 6, }, { name: "short a long b", - a: []Interval16{Interval16{5, 5}, Interval16{7, 7}, Interval16{9, 10}, Interval16{12, 12}, Interval16{15, 17}, Interval16{19, 20}}, - b: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 18}}, - expected: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 20}}, + a: []Interval16{{5, 5}, {7, 7}, {9, 10}, {12, 12}, {15, 17}, {19, 20}}, + b: []Interval16{{1, 10}, {12, 12}, {14, 18}}, + expected: []Interval16{{1, 10}, {12, 12}, {14, 20}}, expectedN: 18, }, { name: "common endings", - a: []Interval16{Interval16{1, 5}, Interval16{15, 20}, Interval16{25, 35}}, - b: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{30, 35}}, - expected: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{25, 35}}, + a: []Interval16{{1, 5}, {15, 20}, {25, 35}}, + b: []Interval16{{1, 10}, {15, 20}, {30, 35}}, + expected: []Interval16{{1, 10}, {15, 20}, {25, 35}}, expectedN: 27, }, { name: "common endings and overlap", - a: []Interval16{Interval16{1, 5}, Interval16{10, 15}}, - b: []Interval16{Interval16{5, 10}, Interval16{12, 17}}, - expected: []Interval16{Interval16{1, 17}}, + a: []Interval16{{1, 5}, {10, 15}}, + b: []Interval16{{5, 10}, {12, 17}}, + expected: []Interval16{{1, 17}}, expectedN: 17, }, { name: "no common endings and overlap", - a: []Interval16{Interval16{5, 10}, Interval16{12, 17}}, - b: []Interval16{Interval16{0, 11}, Interval16{15, 20}}, - expected: []Interval16{Interval16{0, 20}}, + a: []Interval16{{5, 10}, {12, 17}}, + b: []Interval16{{0, 11}, {15, 20}}, + expected: []Interval16{{0, 20}}, expectedN: 21, }, } @@ -4736,3 +4760,81 @@ func TestIntersectionCallback(t *testing.T) { } } } +func TestImportBitmap(t *testing.T) { + b := NewBitmap() + i := GetRoaringIter(1, 3, 5) + + changed, _, err := b.ImportRoaringRawIterator(i, false, true, 16) + if err != nil { + t.Fatal("no error should happen changed") + } + if changed != 3 { + t.Fatal("Should have changed") + } + i = GetRoaringIter(1, 3, 5) + changed, _, err = b.ImportRoaringRawIterator(i, true, true, 16) + if err != nil { + t.Fatal("no error should happen changed") + } + if changed != 3 { + t.Fatalf("Should have changed %v", changed) + } +} +func TestVariousBitmap(t *testing.T) { + b := NewBitmap(3) + + c, e := b.Add(8) + if e != nil { + t.Fatal("add:", e) + } + if c == false { + t.Fatal("add: should have changed") + } + c, _ = b.Add(8) + if c == true { + + t.Fatal("add: should not changed") + } + z, _ := b.AddN(9) + if z != 1 { + t.Fatal("add: should changed 1") + } + z, _ = b.RemoveN(9) + if z != 1 { + t.Fatal("add: should changed 1") + } + if b.Contains(100) { + t.Fatal("should not contain 100") + } + if !b.Any() { + t.Fatal("should have bits ") + } + if b.Size() == 0 { + t.Fatal("should have storage") + } + if b.Count() == 0 { + t.Fatal("should have bits") + } + if b.Max() == 0 { + t.Fatal("should max >0") + } + if m, e := b.Min(); !(m == 3 && e) { + t.Fatal("min should be 3 and containers exist", m, e) + } + b = nil + x := b.Clone() + if x != nil { + + t.Fatal("nil clone should be nil") + } + n := b.Freeze() + if n != nil { + + t.Fatal("nil freeze should be nil") + } + r, _ := b.AddN() + if r != 0 { + + t.Fatal("nil AddN should be 0") + } +} diff --git a/roaring/roaring_nop_paranoia.go b/roaring/roaring_nop_paranoia.go index d89ade062..c37e6b9ec 100644 --- a/roaring/roaring_nop_paranoia.go +++ b/roaring/roaring_nop_paranoia.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !roaringparanoia // +build !roaringparanoia package roaring diff --git a/roaring/roaring_nop_sentinel.go b/roaring/roaring_nop_sentinel.go index daa0b266b..cba2ec106 100644 --- a/roaring/roaring_nop_sentinel.go +++ b/roaring/roaring_nop_sentinel.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !roaringsentinel // +build !roaringsentinel package roaring diff --git a/roaring/roaring_nop_stats.go b/roaring/roaring_nop_stats.go index 2b1506953..c25ab553b 100644 --- a/roaring/roaring_nop_stats.go +++ b/roaring/roaring_nop_stats.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !roaringstats // +build !roaringstats package roaring diff --git a/row.go b/row.go index 022ee76f3..89bec66b1 100644 --- a/row.go +++ b/row.go @@ -146,7 +146,7 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error { if err := callback(&pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}}, + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -161,7 +161,7 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error { if err := callback(&pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}}, }}); err != nil { return errors.Wrap(err, "calling callback") } @@ -308,7 +308,7 @@ func (r *Row) Union(others ...*Row) *Row { } } nextSegs = nextSegs[:0] - toProcess := toProcess[:0] + toProcess = toProcess[:0] for _, segs := range segments { if segs[0].shard == shard { toProcess = append(toProcess, &segs[0]) diff --git a/server/dup.go b/server/dup.go index 55c918758..d6b2fe32c 100644 --- a/server/dup.go +++ b/server/dup.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build darwin || (linux && !arm64) // +build darwin linux,!arm64 package server diff --git a/server/grpc.go b/server/grpc.go index f2f4d3246..270a5dd47 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -471,7 +471,7 @@ func (r ResultUint64) ToRows(callback func(*pb.RowResponse) error) error { return callback(&pb.RowResponse{ Headers: []*pb.ColumnInfo{{Name: "count", Datatype: "uint64"}}, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}}, }}) } @@ -490,7 +490,7 @@ func (r ResultBool) ToRows(callback func(*pb.RowResponse) error) error { return callback(&pb.RowResponse{ Headers: []*pb.ColumnInfo{{Name: "result", Datatype: "bool"}}, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bool(r)}}, + {ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bool(r)}}, }}) } @@ -714,7 +714,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: col}}, + {ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: col}}, }, } ci = nil // only include headers with the first row @@ -1029,7 +1029,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: col}}, + {ColumnVal: &pb.ColumnResponse_StringVal{StringVal: col}}, }, } ci = nil // only include headers with the first row diff --git a/server/grpc_test.go b/server/grpc_test.go index 19353838f..5bb39902a 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -141,21 +141,21 @@ func TestGRPC(t *testing.T) { // []GroupCount (uint64) { pilosa.NewGroupCounts("", []pilosa.GroupCount{ - pilosa.GroupCount{ + { Group: []pilosa.FieldRow{ {Field: "a", RowID: 10}, {Field: "b", RowID: 11}, }, Count: 123, }, - pilosa.GroupCount{ + { Group: []pilosa.FieldRow{ {Field: "a", RowID: 10}, {Field: "b", RowID: 12}, }, Count: 456, }, - pilosa.GroupCount{ + { Group: []pilosa.FieldRow{ {Field: "va", Value: &va}, {Field: "vb", Value: &vb}, @@ -177,7 +177,7 @@ func TestGRPC(t *testing.T) { // []GroupCount (string) + sum { pilosa.NewGroupCounts("sum", []pilosa.GroupCount{ - pilosa.GroupCount{ + { Group: []pilosa.FieldRow{ {Field: "a", RowKey: "ten"}, {Field: "b", RowKey: "eleven"}, diff --git a/server/handler_test.go b/server/handler_test.go index 96b0f12aa..0e76255c3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1136,8 +1136,8 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h := clus.GetNode(0).Handler.(*http.Handler).Handler - h.ServeHTTP(w, req) + h1 := clus.GetNode(0).Handler.(*http.Handler).Handler + h1.ServeHTTP(w, req) result = w.Result() if result.StatusCode != 200 { diff --git a/shardwidth/20.go b/shardwidth/20.go index 5505656dd..f5e6ad968 100644 --- a/shardwidth/20.go +++ b/shardwidth/20.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !shardwidth16 && !shardwidth17 && !shardwidth18 && !shardwidth19 && !shardwidth21 && !shardwidth22 && !shardwidth23 && !shardwidth24 && !shardwidth25 && !shardwidth26 && !shardwidth27 && !shardwidth28 && !shardwidth29 && !shardwidth30 && !shardwidth31 && !shardwidth32 // +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32 package shardwidth diff --git a/sql/handler_test.go b/sql/handler_test.go new file mode 100644 index 000000000..e07b67f1e --- /dev/null +++ b/sql/handler_test.go @@ -0,0 +1,42 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package sql_test + +import ( + "context" + "testing" + + "github.com/molecula/featurebase/v2/sql" + "github.com/molecula/featurebase/v2/test" +) + +func TestHandler(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + api := cluster.GetNode(0).API + queryStr := "select * from nowhere" + mapper := sql.NewMapper() + query, err := mapper.MapSQL(queryStr) + if err != nil { + t.Fatal("failed to map SQL") + } + handler := sql.NewSelectHandler(api) + _, err = handler.Handle(context.Background(), query) + if err.Error() != "mapping select: handling: nowhere: index not found" { + //expecting it to fail with index not found + //can be more elaborate later + t.Fatal(err) + } + +} diff --git a/sql/select.go b/sql/select.go index ca4b22042..1a3a829f9 100644 --- a/sql/select.go +++ b/sql/select.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/pql" pproto "github.com/molecula/featurebase/v2/proto" "github.com/pkg/errors" @@ -56,8 +56,8 @@ func (s *SelectHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.T func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Select, qm QueryMask) (*MappingResult, error) { // Get the handler for this query mask. - handler := s.router.handler(qm) - if handler == nil { + hndlr := s.router.handler(qm) + if hndlr == nil { return nil, ErrUnsupportedQuery } indexFunc := func(indexName string) *pilosa.Index { @@ -68,7 +68,7 @@ func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Sel return idx } - mr, err := handler.Apply(selectStmt, qm, indexFunc) + mr, err := hndlr.Apply(selectStmt, qm, indexFunc) if err != nil { return nil, errors.Wrap(err, "handling") } diff --git a/sql2/parser_test.go b/sql2/parser_test.go index 67927515d..0d371954f 100644 --- a/sql2/parser_test.go +++ b/sql2/parser_test.go @@ -569,7 +569,7 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), }, @@ -586,11 +586,11 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), Set: pos(60), @@ -608,11 +608,11 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), Set: pos(60), @@ -630,11 +630,11 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), Cascade: pos(60), @@ -651,11 +651,11 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), Restrict: pos(60), @@ -672,11 +672,11 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), No: pos(60), @@ -694,16 +694,16 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Args: []*sql.ForeignKeyArg{ - &sql.ForeignKeyArg{ + { On: pos(50), OnDelete: pos(53), Cascade: pos(60), }, - &sql.ForeignKeyArg{ + { On: pos(68), OnUpdate: pos(71), Restrict: pos(78), @@ -720,7 +720,7 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Deferrable: pos(50), @@ -735,7 +735,7 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Not: pos(50), @@ -751,7 +751,7 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Deferrable: pos(50), @@ -768,7 +768,7 @@ func TestParser_ParseStatement(t *testing.T) { ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)}, ForeignLparen: pos(43), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "col2", NamePos: pos(44)}, + {Name: "col2", NamePos: pos(44)}, }, ForeignRparen: pos(48), Deferrable: pos(50), @@ -802,8 +802,8 @@ func TestParser_ParseStatement(t *testing.T) { Key: pos(37), Lparen: pos(41), Columns: []*sql.Ident{ - &sql.Ident{Name: "col1", NamePos: pos(42)}, - &sql.Ident{Name: "col2", NamePos: pos(48)}, + {Name: "col1", NamePos: pos(42)}, + {Name: "col2", NamePos: pos(48)}, }, Rparen: pos(52), }, @@ -838,8 +838,8 @@ func TestParser_ParseStatement(t *testing.T) { Unique: pos(45), Lparen: pos(52), Columns: []*sql.Ident{ - &sql.Ident{Name: "col1", NamePos: pos(53)}, - &sql.Ident{Name: "col2", NamePos: pos(59)}, + {Name: "col1", NamePos: pos(53)}, + {Name: "col2", NamePos: pos(59)}, }, Rparen: pos(63), }, @@ -903,16 +903,16 @@ func TestParser_ParseStatement(t *testing.T) { ForeignKey: pos(37), Lparen: pos(41), Columns: []*sql.Ident{ - &sql.Ident{Name: "col1", NamePos: pos(42)}, - &sql.Ident{Name: "col2", NamePos: pos(48)}, + {Name: "col1", NamePos: pos(42)}, + {Name: "col2", NamePos: pos(48)}, }, Rparen: pos(52), References: pos(54), ForeignTable: &sql.Ident{Name: "tbl2", NamePos: pos(65)}, ForeignLparen: pos(70), ForeignColumns: []*sql.Ident{ - &sql.Ident{Name: "x", NamePos: pos(71)}, - &sql.Ident{Name: "y", NamePos: pos(74)}, + {Name: "x", NamePos: pos(71)}, + {Name: "y", NamePos: pos(74)}, }, ForeignRparen: pos(75), }, @@ -963,8 +963,8 @@ func TestParser_ParseStatement(t *testing.T) { Name: &sql.Ident{NamePos: pos(12), Name: "vw"}, Lparen: pos(15), Columns: []*sql.Ident{ - &sql.Ident{NamePos: pos(16), Name: "col1"}, - &sql.Ident{NamePos: pos(22), Name: "col2"}, + {NamePos: pos(16), Name: "col1"}, + {NamePos: pos(22), Name: "col2"}, }, Rparen: pos(26), As: pos(28), @@ -1709,8 +1709,8 @@ func TestParser_ParseStatement(t *testing.T) { Order: pos(9), OrderBy: pos(15), OrderingTerms: []*sql.OrderingTerm{ - &sql.OrderingTerm{X: &sql.Ident{NamePos: pos(18), Name: "foo"}, Asc: pos(22)}, - &sql.OrderingTerm{X: &sql.Ident{NamePos: pos(27), Name: "bar"}, Desc: pos(31)}, + {X: &sql.Ident{NamePos: pos(18), Name: "foo"}, Asc: pos(22)}, + {X: &sql.Ident{NamePos: pos(27), Name: "bar"}, Desc: pos(31)}, }, }) @@ -1757,7 +1757,7 @@ func TestParser_ParseStatement(t *testing.T) { Order: pos(24), OrderBy: pos(30), OrderingTerms: []*sql.OrderingTerm{ - &sql.OrderingTerm{X: &sql.Ident{NamePos: pos(33), Name: "foo"}}, + {X: &sql.Ident{NamePos: pos(33), Name: "foo"}}, }, }) AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &sql.SelectStatement{ diff --git a/sql2/walk.go b/sql2/walk.go index c04ca76ba..855103987 100644 --- a/sql2/walk.go +++ b/sql2/walk.go @@ -228,9 +228,7 @@ func walk(v Visitor, node Node) (_ Node, err error) { if n.Source != nil { if src, err := walk(v, n.Source); err != nil { return node, err - } else if src != nil { - n.Source = n.Source.(Source) - } else { + } else if src == nil { n.Source = nil } } diff --git a/synthload/synthload_test.go b/synthload/synthload_test.go deleted file mode 100644 index f816fd2ff..000000000 --- a/synthload/synthload_test.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package synthload - -/* // still work in progress, comment out for now, since we - // are getting a hang on go1.13 in CI -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "encoding/json" - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/molecula/featurebase/v2" - "github.com/molecula/featurebase/v2/server" - "github.com/molecula/featurebase/v2/test" -) - -func Test_SynthLoad_ImportSchema(t *testing.T) { - c := test.MustRunCluster(t, 1, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("node0"), - )}, - ) - defer c.Close() - - m0 := c.GetNode(0) - - tarball := "testindex.tar.gz" - - // get the holder.Path to write to - h := m0.API.Holder() - target := h.Path() - PanicOn(h.Close()) - - PanicOn(unpackTarball(tarball, target)) - - // reopen - PanicOn(h.Open()) - - qs := strings.Split(pql, "\n\n") - //vv("qs = '%#v'", qs) - for _, q := range qs { - req := &pilosa.QueryRequest{ - // Index to execute query against. - Index: "testindex", - - // The query string to parse and execute. - Query: q, - - // // The shards to include in the query execution. - // // If empty, all shards are included. - // Shards []uint64 - - // // Return column attributes, if true. - // ColumnAttrs bool - - // // Do not return row attributes, if true. - // ExcludeRowAttrs bool - - // // Do not return columns, if true. - // ExcludeColumns bool - - // // If true, indicates that query is part of a larger distributed query. - // // If false, this request is on the originating node. - // Remote bool - - // // Should we profile this query? - // Profile bool - - // // Additional data associated with the query, in cases where there's - // // row-style inputs for precomputed values. - // EmbeddedData []*Row - } - - qr, err := m0.API.Query(context.Background(), req) - PanicOn(err) - vv("qr = '%#v'", qr) - } -} - -var _ = applySchema - -func applySchema(m0 *test.Command, schemaStr string) { - // don't need schema now that we import the tarball, it has it all. - schema := &pilosa.Schema{} - err := json.NewDecoder(bytes.NewBufferString(schemaStr)).Decode(schema) - PanicOn(err) - - ctx := context.Background() - remote := false - err = m0.API.ApplySchema(ctx, schema, remote) - PanicOn(err) -} - -func unpackTarball(tarball, target string) error { - - vv("target = '%v'", target) - fd, err := os.Open(tarball) - PanicOn(err) - defer fd.Close() - gz, err := gzip.NewReader(fd) - PanicOn(err) - defer gz.Close() - - tarReader := tar.NewReader(gz) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - panic(err) - } - - path := filepath.Join(target, header.Name) - info := header.FileInfo() - if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { - PanicOn(err) - } - continue - } - - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) - PanicOn(err) - - _, err = io.Copy(file, tarReader) - PanicOn(err) - file.Close() - } - return nil -} - -// mkdir o; bangbang schemator -o o -var pql = ` -Count(All()) - -Rows(field='bools',limit=10) -Count(Row(bools='bool')) -Count(Not(Row(bools='bool'))) -Count(Intersect(Row(bools='bool'))) -Count(Difference(Union(Row(bools='bool')),Intersect(Row(bools='bool')))) -GroupBy(Rows(field='bools'),limit=10) - -Rows(field='bools-exists',limit=10) -Count(Row(bools-exists='bool')) -Count(Not(Row(bools-exists='bool'))) -Count(Intersect(Row(bools-exists='bool'))) -Count(Difference(Union(Row(bools-exists='bool')),Intersect(Row(bools-exists='bool')))) -GroupBy(Rows(field='bools-exists'),limit=10) - -GroupBy(Rows(field='int'),limit=10) -Min(field='int') -Max(field='int') -Sum(field='int') - -Rows(field='mutex',limit=10) -Count(Row(mutex='12')) -Count(Not(Row(mutex='12'))) -Count(Row(mutex='13')) -Count(Not(Row(mutex='13'))) -Count(Row(mutex='0')) -Count(Not(Row(mutex='0'))) -Count(Row(mutex='5')) -Count(Not(Row(mutex='5'))) -Count(Row(mutex='2')) -Count(Not(Row(mutex='2'))) -Count(Row(mutex='14')) -Count(Not(Row(mutex='14'))) -Count(Row(mutex='3')) -Count(Not(Row(mutex='3'))) -Count(Row(mutex='18')) -Count(Not(Row(mutex='18'))) -Count(Row(mutex='7')) -Count(Not(Row(mutex='7'))) -Count(Row(mutex='19')) -Count(Not(Row(mutex='19'))) -Count(Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19'))) -Count(Difference(Union(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')),Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')))) -GroupBy(Rows(field='mutex'),limit=10) - -Rows(field='set',limit=10) -Count(Row(set='v621')) -Count(Not(Row(set='v621'))) -Count(Row(set='v997')) -Count(Not(Row(set='v997'))) -Count(Row(set='v772')) -Count(Not(Row(set='v772'))) -Count(Row(set='v340')) -Count(Not(Row(set='v340'))) -Count(Row(set='v766')) -Count(Not(Row(set='v766'))) -Count(Row(set='v416')) -Count(Not(Row(set='v416'))) -Count(Row(set='v481')) -Count(Not(Row(set='v481'))) -Count(Row(set='v581')) -Count(Not(Row(set='v581'))) -Count(Row(set='v591')) -Count(Not(Row(set='v591'))) -Count(Row(set='v675')) -Count(Not(Row(set='v675'))) -Count(Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675'))) -Count(Difference(Union(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')),Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')))) -GroupBy(Rows(field='set'),limit=10) - -Rows(field='string',limit=10) -Count(Row(string='KPFGWOYUGTCF')) -Count(Not(Row(string='KPFGWOYUGTCF'))) -Count(Row(string='VNSHWDTNELDA')) -Count(Not(Row(string='VNSHWDTNELDA'))) -Count(Row(string='LTDOGKKFGZPW')) -Count(Not(Row(string='LTDOGKKFGZPW'))) -Count(Row(string='EETNIIDCDZHB')) -Count(Not(Row(string='EETNIIDCDZHB'))) -Count(Row(string='ATWOMTRASGHP')) -Count(Not(Row(string='ATWOMTRASGHP'))) -Count(Row(string='KOJUBJQVMXZL')) -Count(Not(Row(string='KOJUBJQVMXZL'))) -Count(Row(string='NOXDEYPTGZBH')) -Count(Not(Row(string='NOXDEYPTGZBH'))) -Count(Row(string='DLSRPRADBPKX')) -Count(Not(Row(string='DLSRPRADBPKX'))) -Count(Row(string='WJUEQABAEEUX')) -Count(Not(Row(string='WJUEQABAEEUX'))) -Count(Row(string='PFRGBFDLHHPK')) -Count(Not(Row(string='PFRGBFDLHHPK'))) -Count(Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK'))) -Count(Difference(Union(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')),Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')))) -GroupBy(Rows(field='string'),limit=10) - -GroupBy(Rows(field='time'),limit=10) -` - -// datagen --source=kitchensink --pilosa.index=testindex --start-from=1 --end-at=1000 --pilosa.batch-size=10000 --pilosa.hosts localhost:10101 -// curl localhost:10101/schema - -var _ = schemaString -var schemaString = ` -{ - "indexes": [ - { - "name": "testindex", - "createdAt": 1599601704641744600, - "options": { - "keys": false, - "trackExistence": true - }, - "fields": [ - { - "name": "bools", - "createdAt": 1599601704647053000, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - }, - { - "name": "bools-exists", - "createdAt": 1599601704643225900, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - }, - { - "name": "decimal", - "createdAt": 1599601704642064400, - "options": { - "type": "decimal", - "base": 0, - "scale": 2, - "bitDepth": 63, - "min": -92233720368547760, - "max": 92233720368547760, - "keys": false - } - }, - { - "name": "int", - "createdAt": 1599601704645510100, - "options": { - "type": "int", - "base": 0, - "bitDepth": 63, - "min": -9223372036854776000, - "max": 9223372036854776000, - "keys": false, - "foreignIndex": "" - } - }, - { - "name": "mutex", - "createdAt": 1599601704647187200, - "options": { - "type": "mutex", - "cacheType": "ranked", - "cacheSize": 500, - "keys": true - } - }, - { - "name": "set", - "createdAt": 1599601704643664400, - "options": { - "type": "set", - "cacheType": "lru", - "cacheSize": 1, - "keys": true - } - }, - { - "name": "string", - "createdAt": 1599601704644342000, - "options": { - "type": "set", - "cacheType": "ranked", - "cacheSize": 50000, - "keys": true - } - }, - { - "name": "time", - "createdAt": 1599601704643013400, - "options": { - "type": "time", - "timeQuantum": "YMD", - "keys": true, - "noStandardView": false - } - } - ], - "shardWidth": 1048576 - } - ] -} -` - -*/ diff --git a/synthload/testindex.tar.gz b/synthload/testindex.tar.gz deleted file mode 100644 index 15cdfe2b6..000000000 Binary files a/synthload/testindex.tar.gz and /dev/null differ diff --git a/syswrap/mmap.go b/syswrap/mmap.go index 2d89ce39b..2858183b2 100644 --- a/syswrap/mmap.go +++ b/syswrap/mmap.go @@ -17,10 +17,10 @@ package syswrap import ( + "strings" "sync" "sync/atomic" "syscall" - "strings" "github.com/pkg/errors" ) diff --git a/testhook/cleanup1.14.go b/testhook/cleanup1.14.go index 9ca6024be..a88c91eae 100644 --- a/testhook/cleanup1.14.go +++ b/testhook/cleanup1.14.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build go1.14 // +build go1.14 package testhook diff --git a/time.go b/time.go index 4253a4d1e..403f00da6 100644 --- a/time.go +++ b/time.go @@ -288,7 +288,7 @@ func parseTime(t interface{}) (time.Time, error) { if calcTime, err = time.Parse(TimeFormat, v); err != nil { // if the default parsing fails, check if user tried to // supply partial time eg year and month - calcTime, err := parsePartialTime(v) + calcTime, err = parsePartialTime(v) return calcTime, err } case int64: diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index a0afe8669..ae336a5d0 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -18,10 +18,10 @@ import ( "context" "net/http" - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/tracing" + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" ) // Ensure type implements interface. diff --git a/translator_test.go b/translator_test.go index 2144d2b9a..7d8b09a42 100644 --- a/translator_test.go +++ b/translator_test.go @@ -24,7 +24,7 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/mock" @@ -550,9 +550,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) { var g errgroup.Group defer g.Wait() //nolint:errcheck for i, n := range c.Nodes { - i, api := i, n.API + x, api := i, n.API g.Go(func() (err error) { - defer func() { err = errors.Wrapf(err, "translating on node %d", i) }() + defer func() { err = errors.Wrapf(err, "translating on node %d", x) }() localTranslations, err := api.FindIndexKeys(ctx, "i", keyList...) if err != nil { return errors.Wrap(err, "finding translations") @@ -567,9 +567,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) { // Check that re-invoking create returns the original translations. for i, n := range c.Nodes { - i, api := i, n.API + x, api := i, n.API g.Go(func() (err error) { - defer func() { err = errors.Wrapf(err, "translating on node %d", i) }() + defer func() { err = errors.Wrapf(err, "translating on node %d", x) }() localTranslations, err := api.CreateIndexKeys(ctx, "i", keyList...) if err != nil { return errors.Wrap(err, "finding translations") @@ -636,9 +636,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) { var g errgroup.Group defer g.Wait() //nolint:errcheck for i, n := range c.Nodes { - i, api := i, n.API + x, api := i, n.API g.Go(func() (err error) { - defer func() { err = errors.Wrapf(err, "translating on node %d", i) }() + defer func() { err = errors.Wrapf(err, "translating on node %d", x) }() localTranslations, err := api.FindFieldKeys(ctx, "i", "f", keyList...) if err != nil { return errors.Wrap(err, "finding translations") @@ -653,9 +653,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) { // Check that re-invoking create returns the original translations. for i, n := range c.Nodes { - i, api := i, n.API + x, api := i, n.API g.Go(func() (err error) { - defer func() { err = errors.Wrapf(err, "translating on node %d", i) }() + defer func() { err = errors.Wrapf(err, "translating on node %d", x) }() localTranslations, err := api.CreateFieldKeys(ctx, "i", "f", keyList...) if err != nil { return errors.Wrap(err, "finding translations")