scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1740 from molecula/golangci-lint

Browse source 
[FB-895] gofmt govet
This commit is contained in:
tgruben 2021-11-03 10:14:47 -05:00 • committed by GitHub
commit 63fd5100d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
54 changed files with 603 additions and 580 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
.gitlab/.gitlab-ci.yml
View file

@ -3,13 +3,38 @@ include:
- template: Security/License-Scanning.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
.go-cache:
variables:
GOPATH: $CI_PROJECT_DIR/.go
cache:
- key: $CI_COMMIT_REF_SLUG
paths:
- .go/pkg/mod/
variables:
GOVERSION: "1.16.9"
stages:
- lint
- test
- build
#before_script:
#- echo "before_script"
#- git version
#- go env -w GOPRIVATE=github.com/molecula
#- mkdir -p .go
#- go version
#- go env -w GO111MODULE=on
golangci-lint:
image: golangci/golangci-lint:v1.39.0
stage: lint
extends: .go-cache
allow_failure: false
script:
- echo "Checking for issues in new code"
- golangci-lint run -v
install lattice:
stage: test
image: node:14
@ -52,9 +77,10 @@ run jest tests:
run go tests:
stage: test
image: golang:1.16.9
extends: .go-cache
script:
- echo "Running featurebase unit tests..."
- go test -covermode=atomic -coverprofile=coverage.out
- ./cover-everything.sh
artifacts:
paths:
- coverage.out
@ -69,18 +95,6 @@ run go tests with output:
paths:
- test-report.out
generate coverage html:
stage: test
image: golang:1.16.9
script:
- echo "Generating coverage HTML.."
- go tool cover -html coverage.out -o coverage.html
artifacts:
paths:
- coverage.html
needs:
- job: run go tests
upload to sonarcloud:
stage: test
image: sonarsource/sonar-scanner-cli:4.6
@ -96,6 +110,7 @@ upload to sonarcloud:
build for linux amd64:
stage: build
image: registry.gitlab.com/molecula/featurebase/builder:0.0.3
extends: .go-cache
variables:
GOOS: "linux"
GOARCH: "amd64"
@ -108,6 +123,7 @@ build for linux amd64:
build for linux arm64:
stage: build
image: registry.gitlab.com/molecula/featurebase/builder:0.0.3
extends: .go-cache
variables:
GOOS: "linux"
GOARCH: "arm64"
@ -120,6 +136,7 @@ build for linux arm64:
build for darwin amd64:
stage: build
image: registry.gitlab.com/molecula/featurebase/builder:0.0.3
extends: .go-cache
variables:
GOOS: "darwin"
GOARCH: "amd64"
@ -132,6 +149,7 @@ build for darwin amd64:
build for darwin arm64:
stage: build
image: registry.gitlab.com/molecula/featurebase/builder:0.0.3
extends: .go-cache
variables:
GOOS: "darwin"
GOARCH: "arm64"

57
.golangci.yml
View file

@ -1,3 +1,60 @@
run:
#skip the protobuf generated files
deadline: 5m
timeout: 5m
skip-dirs-use-default: true
skip-dirs:
- pb
- proto
skip-files:
- pql/pql.peg.go
linters:
enable:
- govet
- gofmt
enable-all: false
disable-all: true
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: tab
# print lines of code with issue, default is true
print-issued-lines: true
# print linter name in the end of issue text, default is true
print-linter-name: true
linters-settings:
gofmt:
simplify: true
govet:
# report about shadowed variables
check-shadowing: true
# settings per analyzer
settings:
printf: # analyzer name, run `go tool vet help` to see all analyzers
funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
# enable or disable analyzers by name
# run `go tool vet help` to see all analyzers
enable:
- atomicalign
enable-all: false
disable:
- shadow
disable-all: false
issues:
exclude-use-default: false
max-issues-per-linter: 0
max-same-issues: 0
exclude:
- 'declaration of "(err|ctx)" shadows declaration at'
- 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked'

16
cache.go
View file

@ -398,8 +398,8 @@ func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
},
})
} else {
@ -409,8 +409,8 @@ func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
{Name: "count", Datatype: "uint64"},
},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
},
})
}
@ -540,8 +540,8 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -549,8 +549,8 @@ func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}

16
client/batch.go
View file

@ -672,9 +672,9 @@ func (b *Batch) Import() error {
return errors.Wrap(err, "starting transaction")
}
defer func() {
trns, err := b.client.FinishTransaction(trns.ID)
trnsl, err := b.client.FinishTransaction(trns.ID)
if err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trns)
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
b.client.Stats.Timing(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
}()
@ -729,9 +729,9 @@ func (b *Batch) Flush() error {
return errors.Wrap(err, "starting transaction")
}
defer func() {
trns, err := b.client.FinishTransaction(trns.ID)
trnsl, err := b.client.FinishTransaction(trns.ID)
if err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trns)
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
b.client.Stats.Timing(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
}()
@ -1053,17 +1053,17 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
eg.Go(func() error {
clearViewMap := clearFrags.GetViewMap(shard, field)
if len(clearViewMap) > 0 {
start := time.Now()
startx := time.Now()
err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, clearViewMap, true)
if err != nil {
return errors.Wrapf(err, "import clearing clearing data for %s", field)
}
b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start))
b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(startx))
}
start := time.Now()
starty := time.Now()
err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, viewMap, false)
b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start))
b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
return errors.Wrapf(err, "importing data for %s", field)
})
}

4
client/client.go
View file

@ -180,8 +180,8 @@ func newClientWithCluster(cluster *Cluster, options *ClientOptions) *Client {
func newClientWithURI(uri *pnet.URI, options *ClientOptions) *Client {
client := newClientWithOptions(options)
if options.manualServerAddress {
fragmentNode := newFragmentNodeFromURI(uri)
client.manualFragmentNode = &fragmentNode
fNode := newFragmentNodeFromURI(uri)
client.manualFragmentNode = &fNode
client.manualServerURI = uri
client.cluster = NewClusterWithHost()
}

10
client/orm_test.go
View file

@ -26,7 +26,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/pql"
"github.com/pkg/errors"
)
@ -122,15 +122,15 @@ func TestORM(t *testing.T) {
})
t.Run("NewIndexOptions", func(t *testing.T) {
schema := NewSchema()
schemal := NewSchema()
// test the defaults
index := schema.Index("index-default-options")
index := schemal.Index("index-default-options")
target := `{"options":{}}`
if target != index.options.String() {
t.Fatalf("%s != %s", target, index.options.String())
}
index = schema.Index("index-keys", OptIndexKeys(true))
index = schemal.Index("index-keys", OptIndexKeys(true))
if true != index.Opts().Keys() {
t.Fatalf("index keys %v != %v", true, index.Opts().Keys())
}
@ -139,7 +139,7 @@ func TestORM(t *testing.T) {
t.Fatalf("%s != %s", target, index.options.String())
}
index = schema.Index("index-trackexistence", OptIndexTrackExistence(false))
index = schemal.Index("index-trackexistence", OptIndexTrackExistence(false))
if false != index.Opts().TrackExistence() {
t.Fatalf("index trackExistene %v != %v", true, index.Opts().TrackExistence())
}

2
cluster.go
View file

@ -1534,7 +1534,7 @@ func (c *cluster) translateFieldListIDs(field *Field, ids []uint64) (keys []stri
// TODO: remove this when it is no longer used
func (c *cluster) translateIndexKey(ctx context.Context, indexName string, key string, writable bool) (uint64, error) {
keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: struct{}{}}, writable)
keyMap, err := c.translateIndexKeySet(ctx, indexName, map[string]struct{}{key: {}}, writable)
if err != nil {
return 0, err
}

12
cluster_internal_test.go
View file

@ -15,6 +15,7 @@
package pilosa
import (
"context"
"fmt"
"math/rand"
"reflect"
@ -625,3 +626,14 @@ func TestAE(t *testing.T) {
}
})
}
func TestTranslateIndexKey(t *testing.T) {
c := newCluster()
node0 := &topology.Node{ID: "node0"}
c.addNodeBasicSorted(node0)
c.holder = newHolderWithTempPath(t, "rbf")
_, e := c.translateIndexKey(context.Background(), "i", "a", false)
if e == nil {
t.Fatal("expecting error")
}
}

1
cmd/roaring-migrate/ctim_linux.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
// +build linux
package main

1
const_amd64.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build amd64
// +build amd64
package pilosa

8
cover-everything.sh Executable file
View file

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# actually get test coverage for every single package and subpackage
# very slow but oh well what are you gonna do, not test things?
echo "mode: atomic" > coverage.out
for pkg in $(go list all | grep featurebase); do
go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg;
tail -n +2 pkgcoverage.out >> coverage.out; done

2
ctl/config.go
View file

@ -19,9 +19,9 @@ import (
"fmt"
"io"
toml "github.com/pelletier/go-toml"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/server"
toml "github.com/pelletier/go-toml"
)
// ConfigCommand represents a command for printing a default config.

2
ctl/generate_config.go
View file

@ -19,9 +19,9 @@ import (
"fmt"
"io"
"github.com/pelletier/go-toml"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/server"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
)

24
executor.go
View file

@ -2620,7 +2620,7 @@ func (r RowIdentifiers) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: key}},
{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: key}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -2632,7 +2632,7 @@ func (r RowIdentifiers) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Uint64Val{Uint64Val: uint64(id)}},
{ColumnVal: &proto.ColumnResponse_Uint64Val{Uint64Val: uint64(id)}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -7362,7 +7362,7 @@ func (s SignedRow) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
@ -7380,7 +7380,7 @@ func (s SignedRow) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: val}},
},
}); err != nil {
return errors.Wrap(err, "calling callback")
@ -7480,8 +7480,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_DecimalVal{DecimalVal: &proto.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}},
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
{ColumnVal: &proto.ColumnResponse_DecimalVal{DecimalVal: &proto.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -7493,8 +7493,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Float64Val{Float64Val: v.FloatVal}},
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
{ColumnVal: &proto.ColumnResponse_Float64Val{Float64Val: v.FloatVal}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -7506,8 +7506,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: v.TimestampVal.Format(time.RFC3339Nano)}},
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
{ColumnVal: &proto.ColumnResponse_StringVal{StringVal: v.TimestampVal.Format(time.RFC3339Nano)}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -7519,8 +7519,8 @@ func (v ValCount) ToRows(callback func(*proto.RowResponse) error) error {
if err := callback(&proto.RowResponse{
Headers: ci,
Columns: []*proto.ColumnResponse{
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Val}},
&proto.ColumnResponse{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Val}},
{ColumnVal: &proto.ColumnResponse_Int64Val{Int64Val: v.Count}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}

110
executor_test.go
View file

@ -8184,3 +8184,113 @@ func TestExternalLookup(t *testing.T) {
}
})
}
func TestToRows(t *testing.T) {
ids := &pilosa.RowIdentifiers{
Rows: []uint64{1, 2, 3},
}
c := ids.Clone()
if c == nil {
t.Fatal("Shouldn't be nil ")
}
e := ids.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
keys := &pilosa.RowIdentifiers{
Keys: []string{"a", "b"},
}
c = keys.Clone()
if c == nil {
t.Fatal("Shouldn't be nil ")
}
e = keys.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
v := &pilosa.ValCount{
TimestampVal: time.Now(),
Count: 1,
}
x := v.Clone()
if x == nil {
t.Fatal("Shouldn't be nil ")
}
e = v.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
v.DecimalVal = &pql.Decimal{Value: 1, Scale: 1}
e = v.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
v.DecimalVal = nil
v.FloatVal = 3.0
e = v.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
pfi := &pilosa.PairField{
Pair: pilosa.Pair{ID: 1, Count: 1},
Field: "f",
}
z := pfi.Clone()
if z.Pair.ID != pfi.Pair.ID {
t.Fatal("Should be equal ", z, pfi)
}
e = pfi.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
pfk := &pilosa.PairField{
Pair: pilosa.Pair{Key: "a", Count: 1},
Field: "f",
}
o := pfk.Clone()
if o.Pair.Key != pfk.Pair.Key {
t.Fatal("Should be equal ")
}
e = pfk.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
pfs := &pilosa.PairsField{
Pairs: []pilosa.Pair{{ID: 1, Count: 1}},
Field: "f",
}
f := pfs.Clone()
if f.Pairs[0].ID != pfs.Pairs[0].ID {
t.Fatal("Should be equal ")
}
e = pfs.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
r4 := server.ResultUint64(1)
e = r4.ToRows(func(*proto.RowResponse) error {
return nil
})
if e != nil {
t.Fatal("Shouldn't be err ", e)
}
}

4
fragment.go
View file

@ -961,9 +961,9 @@ func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err er
err = f.gen.Transaction(wp, func() error {
var rowChanged bool
for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ {
if changed, err := f.unprotectedClearRow(tx, rowID); err != nil {
if chang, err := f.unprotectedClearRow(tx, rowID); err != nil {
return errors.Wrapf(err, "clearing row: %d", rowID)
} else if changed {
} else if chang {
rowChanged = true
}
}

12
fragment_internal_test.go
View file

@ -5375,25 +5375,25 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
eg := &errgroup.Group{}
eg.Go(func() error {
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
ltx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
for i := uint64(0); i < 1000; i++ {
_, err := f.setBit(tx, i%4, i)
_, err := f.setBit(ltx, i%4, i)
if err != nil {
return errors.Wrap(err, "setting bit")
}
}
PanicOn(tx.Commit())
PanicOn(ltx.Commit())
return nil
})
// need read-only Tx so as not to block on the writer finishing above.
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
tx1 := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx1.Rollback()
acc := uint64(0)
for i := uint64(0); i < 100; i++ {
r := f.mustRow(tx, i%4)
r := f.mustRow(tx1, i%4)
acc += r.Count()
}
if err := eg.Wait(); err != nil {

1
generation_nodebug.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !generationdebug
// +build !generationdebug
package pilosa

3
go.mod
View file

@ -49,9 +49,8 @@ require (
go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
golang.org/x/mod v0.4.2
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/text v0.3.5 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/yaml.v2 v2.3.0 // indirect
modernc.org/mathutil v1.0.0

10
go.sum
View file

@ -410,8 +410,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -446,7 +446,7 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -454,8 +454,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=

2
holder.go
View file

@ -2082,7 +2082,7 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
if !recurse {
continue
}
fragNums := fragNums[:0]
fragNums = fragNums[:0]
view.mu.Lock()
for fragNum := range view.fragments {
fragNums = append(fragNums, fragNum)

2
http/handler.go
View file

@ -1326,7 +1326,7 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
fos := fieldOptionsToFunctionalOpts(req.Options)
field, err := h.api.CreateField(r.Context(), indexName, fieldName, fos...)
if _, ok := err.(pilosa.BadRequestError); ok {
if _, ok = err.(pilosa.BadRequestError); ok {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}

2
main_test.go
View file

@ -17,9 +17,9 @@ package pilosa_test
import (
"fmt"
"net"
"net/http"
"testing"
"net/http"
_ "net/http/pprof"
"github.com/molecula/featurebase/v2/testhook"

10
rbf.go
View file

@ -254,25 +254,25 @@ func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove boo
if rc.N() == 0 {
return 0, nil
}
rc, chng := rc.Remove(lo)
rc1, chng := rc.Remove(lo)
if !chng {
return 0, nil
}
if rc.N() == 0 {
if rc1.N() == 0 {
err = tx.tx.RemoveContainer(name, hi)
} else {
err = tx.tx.PutContainer(name, hi, rc)
err = tx.tx.PutContainer(name, hi, rc1)
}
if err != nil {
return 0, err
}
return 1, nil
} else {
rc, chng := rc.Add(lo)
rc2, chng := rc.Add(lo)
if !chng {
return 0, nil
}
err = tx.tx.PutContainer(name, hi, rc)
err = tx.tx.PutContainer(name, hi, rc2)
if err != nil {
return 0, err
}

1
rbf/cfg/os.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !386
// +build !386
package cfg

33
rbf/cursor_test.go
View file

@ -18,6 +18,7 @@ import (
"io"
"math/bits"
"math/rand"
"os"
"reflect"
"sort"
"strings"
@ -558,10 +559,12 @@ func TestCursor_RLETesting(t *testing.T) {
}
want := []uint16{0}
if got, want := c.Values(), want; !reflect.DeepEqual(got, want) {
got := c.Values()
if !reflect.DeepEqual(got, want) {
t.Fatalf("Values()=%#v, want %#v", got, want)
} else if got, want := c.Key(), uint64(1); !reflect.DeepEqual(got, want) {
t.Fatalf("Key()=%#v, want %#v", got, want)
}
if g, w := c.Key(), uint64(1); !reflect.DeepEqual(g, w) {
t.Fatalf("Key()=%#v, want %#v", g, w)
}
})
}
@ -845,6 +848,25 @@ func (e *EasyWalker) String() string {
return e.path.String()
}
func TestDumpDot(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
c, err := tx.Cursor("x")
if err != nil {
t.Fatal(err)
}
_, err = c.Add(1)
if err != nil {
t.Fatal(err)
}
rbf.Dumpdot(tx, 0, " ", os.Stdout)
}
func TestCursor_UpdateBranchCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
@ -962,6 +984,9 @@ func TestCursor_SplitBranchCells(t *testing.T) {
t.Fatalf("Expecting RBLL (a branch split) got %v", after.String())
}
//
c, _ := tx.Cursor("x") //added just for dot code coverage
c.Dump("ignore for coverage")
}
@ -1140,7 +1165,7 @@ func TestForEachRange(t *testing.T) {
}
rb.Put(0, roaring.NewContainerBitmap(n, bits))
crun := roaring.NewContainerRun([]roaring.Interval16{roaring.Interval16{Start: 0, Last: 1<<16 - 1}})
crun := roaring.NewContainerRun([]roaring.Interval16{{Start: 0, Last: 1<<16 - 1}})
rb.Put(1, crun)
rb.Put(2, roaring.NewContainerArray([]uint16{1, 1024, 1<<16 - 1}))

2
rbf/cursorx.go
View file

@ -106,7 +106,7 @@ func (c *Cursor) Dump(name string) {
fmt.Fprintf(bufStdout, "rankdir=\"LR\"\n")
fmt.Fprintf(bufStdout, "node [shape=record height=.1]\n")
dumpdot(c.tx, 0, " ", bufStdout)
Dumpdot(c.tx, 0, " ", bufStdout)
fmt.Fprintf(bufStdout, "\n}")
bufStdout.Flush()
}

10
rbf/dot.go
View file

@ -60,8 +60,8 @@ func dotCell(b []byte, parent string, writer io.Writer) {
}
}
// dumpdot recursively writes the tree representation starting from a given page to STDERR.
func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
// Dumpdot recursively writes the tree representation starting from a given page to STDERR.
func Dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
page, _, err := tx.readPage(pgno)
if err != nil {
panic(err)
@ -78,8 +78,8 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
for _, record := range records {
root := fmt.Sprintf("root%d", record.Pgno)
fmt.Fprintf(writer, "%s[label=\"ROOT(%d)| %s\"]\n%s->%s\n", root, record.Pgno, record.Name, rr, root)
parent := fmt.Sprintf("root%d", record.Pgno)
dumpdot(tx, record.Pgno, parent, writer)
p := fmt.Sprintf("root%d", record.Pgno)
Dumpdot(tx, record.Pgno, p, writer)
}
}
@ -96,7 +96,7 @@ func dumpdot(tx *Tx, pgno uint32, parent string, writer io.Writer) {
for i, n := 0, readCellN(page); i < n; i++ {
cell := readBranchCell(page, i)
if cell.Flags&uint32(ContainerTypeBitmap) == 0 { // leaf/branch child page
dumpdot(tx, cell.ChildPgno, p, writer)
Dumpdot(tx, cell.ChildPgno, p, writer)
} else {
b := fmt.Sprintf("bm%d", cell.ChildPgno)
fmt.Fprintf(writer, "%s[label=\"BITMAP(%d) key=%d \"]\n %s -> %s\n", b, cell.ChildPgno, cell.LeftKey, p, b)

4
rbf/rbf_test.go
View file

@ -39,7 +39,7 @@ func TestReadWriteRootRecord(t *testing.T) {
// Write records.
if remaining, err := rbf.WriteRootRecord(buf, &rbf.RootRecord{Pgno: 10, Name: "foo"}); err != nil {
t.Fatal(err)
} else if remaining, err := rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 11, Name: "bar"}); err != nil {
} else if remaining, err = rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 11, Name: "bar"}); err != nil {
t.Fatal(err)
} else if _, err := rbf.WriteRootRecord(remaining, &rbf.RootRecord{Pgno: 12, Name: "baz"}); err != io.ErrShortBuffer {
t.Fatalf("unexpected error: %#v", err) // buffer too short
@ -50,7 +50,7 @@ func TestReadWriteRootRecord(t *testing.T) {
t.Fatal(err)
} else if got, want := *rec, (rbf.RootRecord{Pgno: 10, Name: "foo"}); got != want {
t.Fatalf("ReadRootRecord=%#v, want %#v", got, want)
} else if rec, remaining, err := rbf.ReadRootRecord(remaining); err != nil {
} else if rec, remaining, err = rbf.ReadRootRecord(remaining); err != nil {
t.Fatal(err)
} else if got, want := *rec, (rbf.RootRecord{Pgno: 11, Name: "bar"}); got != want {
t.Fatalf("ReadRootRecord=%#v, want %#v", got, want)

1
roaring/add_test.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !race
// +build !race
package roaring

1
roaring/generation_nodebug.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !generationdebug
// +build !generationdebug
package roaring

1
roaring/nop_inst.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !btreeInstrumentation
// +build !btreeInstrumentation
package roaring

24
roaring/roaring.go
View file

@ -2402,13 +2402,13 @@ func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log b
}
err = nil
if log && changed > 0 {
op := op{opN: changed, roaring: itr.Data()}
o := op{opN: changed, roaring: itr.Data()}
if clear {
op.typ = opTypeRemoveRoaring
o.typ = opTypeRemoveRoaring
} else {
op.typ = opTypeAddRoaring
o.typ = opTypeAddRoaring
}
err = b.writeOp(&op)
err = b.writeOp(&o)
}
return changed, rowSet, err
}
@ -4559,26 +4559,26 @@ func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) {
j++
} else if va.Last > vb.Last && va.Start >= vb.Start {
// |--vb-|-|-va--|
for i := int(va.Start); i <= int(vb.Last); i++ {
fn(uint16(i))
for z := int(va.Start); z <= int(vb.Last); z++ {
fn(uint16(z))
}
j++
} else if va.Last > vb.Last && va.Start < vb.Start {
// |--va|--vb--|--|
for i := int(vb.Start); i <= int(vb.Last); i++ {
fn(uint16(i))
for z := int(vb.Start); z <= int(vb.Last); z++ {
fn(uint16(z))
}
j++
} else if va.Last <= vb.Last && va.Start >= vb.Start {
// |--vb|--va--|--|
for i := int(va.Start); i <= int(va.Last); i++ {
fn(uint16(i))
for z := int(va.Start); z <= int(va.Last); z++ {
fn(uint16(z))
}
i++
} else if va.Last <= vb.Last && va.Start < vb.Start {
// |--va-|-|-vb--|
for i := int(vb.Start); i <= int(va.Last); i++ {
fn(uint16(i))
for z := int(vb.Start); z <= int(va.Last); z++ {
fn(uint16(z))
}
i++
}

216
roaring/roaring_internal_test.go
View file

@ -34,6 +34,29 @@ func (iv Interval16) String() string {
return fmt.Sprintf("[%d, %d]", iv.Start, iv.Last)
}
func GetRoaringIter(bitsToSet ...uint64) RoaringIterator {
b := NewBitmap()
changed := b.DirectAddN(bitsToSet...)
n := len(bitsToSet)
if changed != n {
e := fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n)
panic(e)
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, er := b.WriteTo(buf)
if er != nil {
if er != nil {
panic(er)
}
}
itr, err := NewRoaringIterator(buf.Bytes())
if err != nil {
panic(err)
}
return itr
}
func TestRunAppendInterval(t *testing.T) {
a := NewContainerRun(nil)
tests := []struct {
@ -99,9 +122,10 @@ func TestContainerRunAdd(t *testing.T) {
{0, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 7}, {Start: 10, Last: 10}}},
{8, []Interval16{{Start: 0, Last: 4}, {Start: 6, Last: 8}, {Start: 10, Last: 10}}},
}
var changed bool
for _, test := range tests {
c.setMapped(true)
c, changed := c.add(test.op)
c, changed = c.add(test.op)
if !changed {
t.Fatalf("result of adding new bit should be true: %v", c.runs())
}
@ -731,65 +755,65 @@ func TestUnionInterval16InPlace(t *testing.T) {
}{
{
name: "firstBitUnset lastBitSet",
a: []Interval16{Interval16{1, 10}},
b: []Interval16{Interval16{10, 10}},
expected: []Interval16{Interval16{1, 10}},
a: []Interval16{{1, 10}},
b: []Interval16{{10, 10}},
expected: []Interval16{{1, 10}},
expectedN: 10,
},
{
name: "single overlap",
a: []Interval16{Interval16{1, 10}, Interval16{21, 28}},
b: []Interval16{Interval16{8, 12}},
expected: []Interval16{Interval16{1, 12}, Interval16{21, 28}},
a: []Interval16{{1, 10}, {21, 28}},
b: []Interval16{{8, 12}},
expected: []Interval16{{1, 12}, {21, 28}},
expectedN: 20,
},
{
name: "nested intervals",
a: []Interval16{Interval16{3, 13}, Interval16{17, 20}},
b: []Interval16{Interval16{1, 4}, Interval16{6, 7}, Interval16{8, 9}, Interval16{10, 11}, Interval16{14, 17}},
expected: []Interval16{Interval16{1, 20}},
a: []Interval16{{3, 13}, {17, 20}},
b: []Interval16{{1, 4}, {6, 7}, {8, 9}, {10, 11}, {14, 17}},
expected: []Interval16{{1, 20}},
expectedN: 20,
},
{
name: "no overlap",
a: []Interval16{Interval16{3, 4}, Interval16{7, 8}},
b: []Interval16{Interval16{1, 2}, Interval16{5, 6}, Interval16{9, 10}},
expected: []Interval16{Interval16{1, 10}},
a: []Interval16{{3, 4}, {7, 8}},
b: []Interval16{{1, 2}, {5, 6}, {9, 10}},
expected: []Interval16{{1, 10}},
expectedN: 10,
},
{
name: "b in a",
a: []Interval16{Interval16{1, 10}},
b: []Interval16{Interval16{5, 7}},
expected: []Interval16{Interval16{1, 10}},
a: []Interval16{{1, 10}},
b: []Interval16{{5, 7}},
expected: []Interval16{{1, 10}},
expectedN: 10,
},
{
name: "a eq b",
a: []Interval16{Interval16{1, 10}},
b: []Interval16{Interval16{1, 10}},
expected: []Interval16{Interval16{1, 10}},
a: []Interval16{{1, 10}},
b: []Interval16{{1, 10}},
expected: []Interval16{{1, 10}},
expectedN: 10,
},
{
name: "a in b",
a: []Interval16{Interval16{5, 7}},
b: []Interval16{Interval16{1, 10}},
expected: []Interval16{Interval16{1, 10}},
a: []Interval16{{5, 7}},
b: []Interval16{{1, 10}},
expected: []Interval16{{1, 10}},
expectedN: 10,
},
{
name: "a ahead b",
a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}},
b: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}},
expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}},
a: []Interval16{{1, 2}, {3, 4}, {5, 7}},
b: []Interval16{{10, 11}, {12, 13}, {14, 15}},
expected: []Interval16{{1, 7}, {10, 15}},
expectedN: 13,
},
{
name: "b ahead a",
a: []Interval16{Interval16{10, 11}, Interval16{12, 13}, Interval16{14, 15}},
b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}},
expected: []Interval16{Interval16{1, 7}, Interval16{10, 15}},
a: []Interval16{{10, 11}, {12, 13}, {14, 15}},
b: []Interval16{{1, 2}, {3, 4}, {5, 7}},
expected: []Interval16{{1, 7}, {10, 15}},
expectedN: 13,
},
{
@ -802,78 +826,78 @@ func TestUnionInterval16InPlace(t *testing.T) {
{
name: "empty a",
a: []Interval16{},
b: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}},
expected: []Interval16{Interval16{1, 7}},
b: []Interval16{{1, 2}, {3, 4}, {5, 7}},
expected: []Interval16{{1, 7}},
expectedN: 7,
},
{
name: "empty b",
a: []Interval16{Interval16{1, 2}, Interval16{3, 4}, Interval16{5, 7}},
a: []Interval16{{1, 2}, {3, 4}, {5, 7}},
b: []Interval16{},
expected: []Interval16{Interval16{1, 7}},
expected: []Interval16{{1, 7}},
expectedN: 7,
},
{
name: "single a",
a: []Interval16{Interval16{1, 2}},
a: []Interval16{{1, 2}},
b: []Interval16{},
expected: []Interval16{Interval16{1, 2}},
expected: []Interval16{{1, 2}},
expectedN: 2,
},
{
name: "single b",
a: []Interval16{},
b: []Interval16{Interval16{1, 2}},
expected: []Interval16{Interval16{1, 2}},
b: []Interval16{{1, 2}},
expected: []Interval16{{1, 2}},
expectedN: 2,
},
{
name: "single a single b",
a: []Interval16{Interval16{3, 4}},
b: []Interval16{Interval16{1, 2}},
expected: []Interval16{Interval16{1, 4}},
a: []Interval16{{3, 4}},
b: []Interval16{{1, 2}},
expected: []Interval16{{1, 4}},
expectedN: 4,
},
{
name: "oddBitsSet lastBitUnset",
a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}},
b: []Interval16{Interval16{0, 4}},
expected: []Interval16{Interval16{0, 5}},
a: []Interval16{{1, 1}, {3, 3}, {5, 5}},
b: []Interval16{{0, 4}},
expected: []Interval16{{0, 5}},
expectedN: 6,
},
{
name: "all bits",
a: []Interval16{Interval16{1, 1}, Interval16{3, 3}, Interval16{5, 5}},
b: []Interval16{Interval16{0, 0}, Interval16{2, 2}, Interval16{4, 4}},
expected: []Interval16{Interval16{0, 5}},
a: []Interval16{{1, 1}, {3, 3}, {5, 5}},
b: []Interval16{{0, 0}, {2, 2}, {4, 4}},
expected: []Interval16{{0, 5}},
expectedN: 6,
},
{
name: "short a long b",
a: []Interval16{Interval16{5, 5}, Interval16{7, 7}, Interval16{9, 10}, Interval16{12, 12}, Interval16{15, 17}, Interval16{19, 20}},
b: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 18}},
expected: []Interval16{Interval16{1, 10}, Interval16{12, 12}, Interval16{14, 20}},
a: []Interval16{{5, 5}, {7, 7}, {9, 10}, {12, 12}, {15, 17}, {19, 20}},
b: []Interval16{{1, 10}, {12, 12}, {14, 18}},
expected: []Interval16{{1, 10}, {12, 12}, {14, 20}},
expectedN: 18,
},
{
name: "common endings",
a: []Interval16{Interval16{1, 5}, Interval16{15, 20}, Interval16{25, 35}},
b: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{30, 35}},
expected: []Interval16{Interval16{1, 10}, Interval16{15, 20}, Interval16{25, 35}},
a: []Interval16{{1, 5}, {15, 20}, {25, 35}},
b: []Interval16{{1, 10}, {15, 20}, {30, 35}},
expected: []Interval16{{1, 10}, {15, 20}, {25, 35}},
expectedN: 27,
},
{
name: "common endings and overlap",
a: []Interval16{Interval16{1, 5}, Interval16{10, 15}},
b: []Interval16{Interval16{5, 10}, Interval16{12, 17}},
expected: []Interval16{Interval16{1, 17}},
a: []Interval16{{1, 5}, {10, 15}},
b: []Interval16{{5, 10}, {12, 17}},
expected: []Interval16{{1, 17}},
expectedN: 17,
},
{
name: "no common endings and overlap",
a: []Interval16{Interval16{5, 10}, Interval16{12, 17}},
b: []Interval16{Interval16{0, 11}, Interval16{15, 20}},
expected: []Interval16{Interval16{0, 20}},
a: []Interval16{{5, 10}, {12, 17}},
b: []Interval16{{0, 11}, {15, 20}},
expected: []Interval16{{0, 20}},
expectedN: 21,
},
}
@ -4736,3 +4760,81 @@ func TestIntersectionCallback(t *testing.T) {
}
}
}
func TestImportBitmap(t *testing.T) {
b := NewBitmap()
i := GetRoaringIter(1, 3, 5)
changed, _, err := b.ImportRoaringRawIterator(i, false, true, 16)
if err != nil {
t.Fatal("no error should happen changed")
}
if changed != 3 {
t.Fatal("Should have changed")
}
i = GetRoaringIter(1, 3, 5)
changed, _, err = b.ImportRoaringRawIterator(i, true, true, 16)
if err != nil {
t.Fatal("no error should happen changed")
}
if changed != 3 {
t.Fatalf("Should have changed %v", changed)
}
}
func TestVariousBitmap(t *testing.T) {
b := NewBitmap(3)
c, e := b.Add(8)
if e != nil {
t.Fatal("add:", e)
}
if c == false {
t.Fatal("add: should have changed")
}
c, _ = b.Add(8)
if c == true {
t.Fatal("add: should not changed")
}
z, _ := b.AddN(9)
if z != 1 {
t.Fatal("add: should changed 1")
}
z, _ = b.RemoveN(9)
if z != 1 {
t.Fatal("add: should changed 1")
}
if b.Contains(100) {
t.Fatal("should not contain 100")
}
if !b.Any() {
t.Fatal("should have bits ")
}
if b.Size() == 0 {
t.Fatal("should have storage")
}
if b.Count() == 0 {
t.Fatal("should have bits")
}
if b.Max() == 0 {
t.Fatal("should max >0")
}
if m, e := b.Min(); !(m == 3 && e) {
t.Fatal("min should be 3 and containers exist", m, e)
}
b = nil
x := b.Clone()
if x != nil {
t.Fatal("nil clone should be nil")
}
n := b.Freeze()
if n != nil {
t.Fatal("nil freeze should be nil")
}
r, _ := b.AddN()
if r != 0 {
t.Fatal("nil AddN should be 0")
}
}

1
roaring/roaring_nop_paranoia.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !roaringparanoia
// +build !roaringparanoia
package roaring

1
roaring/roaring_nop_sentinel.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !roaringsentinel
// +build !roaringsentinel
package roaring

1
roaring/roaring_nop_stats.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !roaringstats
// +build !roaringstats
package roaring

6
row.go
View file

@ -146,7 +146,7 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}},
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: x}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -161,7 +161,7 @@ func (r *Row) ToRows(callback func(*pb.RowResponse) error) error {
if err := callback(&pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: x}},
}}); err != nil {
return errors.Wrap(err, "calling callback")
}
@ -308,7 +308,7 @@ func (r *Row) Union(others ...*Row) *Row {
}
}
nextSegs = nextSegs[:0]
toProcess := toProcess[:0]
toProcess = toProcess[:0]
for _, segs := range segments {
if segs[0].shard == shard {
toProcess = append(toProcess, &segs[0])

1
server/dup.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build darwin || (linux && !arm64)
// +build darwin linux,!arm64
package server

8
server/grpc.go
View file

@ -471,7 +471,7 @@ func (r ResultUint64) ToRows(callback func(*pb.RowResponse) error) error {
return callback(&pb.RowResponse{
Headers: []*pb.ColumnInfo{{Name: "count", Datatype: "uint64"}},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(r)}},
}})
}
@ -490,7 +490,7 @@ func (r ResultBool) ToRows(callback func(*pb.RowResponse) error) error {
return callback(&pb.RowResponse{
Headers: []*pb.ColumnInfo{{Name: "result", Datatype: "bool"}},
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bool(r)}},
{ColumnVal: &pb.ColumnResponse_BoolVal{BoolVal: bool(r)}},
}})
}
@ -714,7 +714,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
rowResp := &pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: col}},
{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: col}},
},
}
ci = nil // only include headers with the first row
@ -1029,7 +1029,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
rowResp := &pb.RowResponse{
Headers: ci,
Columns: []*pb.ColumnResponse{
&pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: col}},
{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: col}},
},
}
ci = nil // only include headers with the first row

8
server/grpc_test.go
View file

@ -141,21 +141,21 @@ func TestGRPC(t *testing.T) {
// []GroupCount (uint64)
{
pilosa.NewGroupCounts("", []pilosa.GroupCount{
pilosa.GroupCount{
{
Group: []pilosa.FieldRow{
{Field: "a", RowID: 10},
{Field: "b", RowID: 11},
},
Count: 123,
},
pilosa.GroupCount{
{
Group: []pilosa.FieldRow{
{Field: "a", RowID: 10},
{Field: "b", RowID: 12},
},
Count: 456,
},
pilosa.GroupCount{
{
Group: []pilosa.FieldRow{
{Field: "va", Value: &va},
{Field: "vb", Value: &vb},
@ -177,7 +177,7 @@ func TestGRPC(t *testing.T) {
// []GroupCount (string) + sum
{
pilosa.NewGroupCounts("sum", []pilosa.GroupCount{
pilosa.GroupCount{
{
Group: []pilosa.FieldRow{
{Field: "a", RowKey: "ten"},
{Field: "b", RowKey: "eleven"},

4
server/handler_test.go
View file

@ -1136,8 +1136,8 @@ func TestHandler_Endpoints(t *testing.T) {
clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
defer clus.Close()
w = httptest.NewRecorder()
h := clus.GetNode(0).Handler.(*http.Handler).Handler
h.ServeHTTP(w, req)
h1 := clus.GetNode(0).Handler.(*http.Handler).Handler
h1.ServeHTTP(w, req)
result = w.Result()
if result.StatusCode != 200 {

1
shardwidth/20.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !shardwidth16 && !shardwidth17 && !shardwidth18 && !shardwidth19 && !shardwidth21 && !shardwidth22 && !shardwidth23 && !shardwidth24 && !shardwidth25 && !shardwidth26 && !shardwidth27 && !shardwidth28 && !shardwidth29 && !shardwidth30 && !shardwidth31 && !shardwidth32
// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32
package shardwidth

42
sql/handler_test.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sql_test
import (
"context"
"testing"
"github.com/molecula/featurebase/v2/sql"
"github.com/molecula/featurebase/v2/test"
)
func TestHandler(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
api := cluster.GetNode(0).API
queryStr := "select * from nowhere"
mapper := sql.NewMapper()
query, err := mapper.MapSQL(queryStr)
if err != nil {
t.Fatal("failed to map SQL")
}
handler := sql.NewSelectHandler(api)
_, err = handler.Handle(context.Background(), query)
if err.Error() != "mapping select: handling: nowhere: index not found" {
//expecting it to fail with index not found
//can be more elaborate later
t.Fatal(err)
}
}

8
sql/select.go
View file

@ -19,7 +19,7 @@ import (
"fmt"
"strings"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/pql"
pproto "github.com/molecula/featurebase/v2/proto"
"github.com/pkg/errors"
@ -56,8 +56,8 @@ func (s *SelectHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.T
func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Select, qm QueryMask) (*MappingResult, error) {
// Get the handler for this query mask.
handler := s.router.handler(qm)
if handler == nil {
hndlr := s.router.handler(qm)
if hndlr == nil {
return nil, ErrUnsupportedQuery
}
indexFunc := func(indexName string) *pilosa.Index {
@ -68,7 +68,7 @@ func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Sel
return idx
}
mr, err := handler.Apply(selectStmt, qm, indexFunc)
mr, err := hndlr.Apply(selectStmt, qm, indexFunc)
if err != nil {
return nil, errors.Wrap(err, "handling")
}

62
sql2/parser_test.go
View file

@ -569,7 +569,7 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
},
@ -586,11 +586,11 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
Set: pos(60),
@ -608,11 +608,11 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
Set: pos(60),
@ -630,11 +630,11 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
Cascade: pos(60),
@ -651,11 +651,11 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
Restrict: pos(60),
@ -672,11 +672,11 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
No: pos(60),
@ -694,16 +694,16 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Args: []*sql.ForeignKeyArg{
&sql.ForeignKeyArg{
{
On: pos(50),
OnDelete: pos(53),
Cascade: pos(60),
},
&sql.ForeignKeyArg{
{
On: pos(68),
OnUpdate: pos(71),
Restrict: pos(78),
@ -720,7 +720,7 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Deferrable: pos(50),
@ -735,7 +735,7 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Not: pos(50),
@ -751,7 +751,7 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Deferrable: pos(50),
@ -768,7 +768,7 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignTable: &sql.Ident{Name: "foo", NamePos: pos(39)},
ForeignLparen: pos(43),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "col2", NamePos: pos(44)},
{Name: "col2", NamePos: pos(44)},
},
ForeignRparen: pos(48),
Deferrable: pos(50),
@ -802,8 +802,8 @@ func TestParser_ParseStatement(t *testing.T) {
Key: pos(37),
Lparen: pos(41),
Columns: []*sql.Ident{
&sql.Ident{Name: "col1", NamePos: pos(42)},
&sql.Ident{Name: "col2", NamePos: pos(48)},
{Name: "col1", NamePos: pos(42)},
{Name: "col2", NamePos: pos(48)},
},
Rparen: pos(52),
},
@ -838,8 +838,8 @@ func TestParser_ParseStatement(t *testing.T) {
Unique: pos(45),
Lparen: pos(52),
Columns: []*sql.Ident{
&sql.Ident{Name: "col1", NamePos: pos(53)},
&sql.Ident{Name: "col2", NamePos: pos(59)},
{Name: "col1", NamePos: pos(53)},
{Name: "col2", NamePos: pos(59)},
},
Rparen: pos(63),
},
@ -903,16 +903,16 @@ func TestParser_ParseStatement(t *testing.T) {
ForeignKey: pos(37),
Lparen: pos(41),
Columns: []*sql.Ident{
&sql.Ident{Name: "col1", NamePos: pos(42)},
&sql.Ident{Name: "col2", NamePos: pos(48)},
{Name: "col1", NamePos: pos(42)},
{Name: "col2", NamePos: pos(48)},
},
Rparen: pos(52),
References: pos(54),
ForeignTable: &sql.Ident{Name: "tbl2", NamePos: pos(65)},
ForeignLparen: pos(70),
ForeignColumns: []*sql.Ident{
&sql.Ident{Name: "x", NamePos: pos(71)},
&sql.Ident{Name: "y", NamePos: pos(74)},
{Name: "x", NamePos: pos(71)},
{Name: "y", NamePos: pos(74)},
},
ForeignRparen: pos(75),
},
@ -963,8 +963,8 @@ func TestParser_ParseStatement(t *testing.T) {
Name: &sql.Ident{NamePos: pos(12), Name: "vw"},
Lparen: pos(15),
Columns: []*sql.Ident{
&sql.Ident{NamePos: pos(16), Name: "col1"},
&sql.Ident{NamePos: pos(22), Name: "col2"},
{NamePos: pos(16), Name: "col1"},
{NamePos: pos(22), Name: "col2"},
},
Rparen: pos(26),
As: pos(28),
@ -1709,8 +1709,8 @@ func TestParser_ParseStatement(t *testing.T) {
Order: pos(9),
OrderBy: pos(15),
OrderingTerms: []*sql.OrderingTerm{
&sql.OrderingTerm{X: &sql.Ident{NamePos: pos(18), Name: "foo"}, Asc: pos(22)},
&sql.OrderingTerm{X: &sql.Ident{NamePos: pos(27), Name: "bar"}, Desc: pos(31)},
{X: &sql.Ident{NamePos: pos(18), Name: "foo"}, Asc: pos(22)},
{X: &sql.Ident{NamePos: pos(27), Name: "bar"}, Desc: pos(31)},
},
})
@ -1757,7 +1757,7 @@ func TestParser_ParseStatement(t *testing.T) {
Order: pos(24),
OrderBy: pos(30),
OrderingTerms: []*sql.OrderingTerm{
&sql.OrderingTerm{X: &sql.Ident{NamePos: pos(33), Name: "foo"}},
{X: &sql.Ident{NamePos: pos(33), Name: "foo"}},
},
})
AssertParseStatement(t, `SELECT * UNION ALL SELECT *`, &sql.SelectStatement{

4
sql2/walk.go
View file

@ -228,9 +228,7 @@ func walk(v Visitor, node Node) (_ Node, err error) {
if n.Source != nil {
if src, err := walk(v, n.Source); err != nil {
return node, err
} else if src != nil {
n.Source = n.Source.(Source)
} else {
} else if src == nil {
n.Source = nil
}
}

361
synthload/synthload_test.go
View file

@ -1,361 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package synthload
/* // still work in progress, comment out for now, since we
// are getting a hang on go1.13 in CI
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/test"
)
func Test_SynthLoad_ImportSchema(t *testing.T) {
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
)},
)
defer c.Close()
m0 := c.GetNode(0)
tarball := "testindex.tar.gz"
// get the holder.Path to write to
h := m0.API.Holder()
target := h.Path()
PanicOn(h.Close())
PanicOn(unpackTarball(tarball, target))
// reopen
PanicOn(h.Open())
qs := strings.Split(pql, "\n\n")
//vv("qs = '%#v'", qs)
for _, q := range qs {
req := &pilosa.QueryRequest{
// Index to execute query against.
Index: "testindex",
// The query string to parse and execute.
Query: q,
// // The shards to include in the query execution.
// // If empty, all shards are included.
// Shards []uint64
// // Return column attributes, if true.
// ColumnAttrs bool
// // Do not return row attributes, if true.
// ExcludeRowAttrs bool
// // Do not return columns, if true.
// ExcludeColumns bool
// // If true, indicates that query is part of a larger distributed query.
// // If false, this request is on the originating node.
// Remote bool
// // Should we profile this query?
// Profile bool
// // Additional data associated with the query, in cases where there's
// // row-style inputs for precomputed values.
// EmbeddedData []*Row
}
qr, err := m0.API.Query(context.Background(), req)
PanicOn(err)
vv("qr = '%#v'", qr)
}
}
var _ = applySchema
func applySchema(m0 *test.Command, schemaStr string) {
// don't need schema now that we import the tarball, it has it all.
schema := &pilosa.Schema{}
err := json.NewDecoder(bytes.NewBufferString(schemaStr)).Decode(schema)
PanicOn(err)
ctx := context.Background()
remote := false
err = m0.API.ApplySchema(ctx, schema, remote)
PanicOn(err)
}
func unpackTarball(tarball, target string) error {
vv("target = '%v'", target)
fd, err := os.Open(tarball)
PanicOn(err)
defer fd.Close()
gz, err := gzip.NewReader(fd)
PanicOn(err)
defer gz.Close()
tarReader := tar.NewReader(gz)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
path := filepath.Join(target, header.Name)
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
PanicOn(err)
}
continue
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
PanicOn(err)
_, err = io.Copy(file, tarReader)
PanicOn(err)
file.Close()
}
return nil
}
// mkdir o; bangbang schemator -o o
var pql = `
Count(All())
Rows(field='bools',limit=10)
Count(Row(bools='bool'))
Count(Not(Row(bools='bool')))
Count(Intersect(Row(bools='bool')))
Count(Difference(Union(Row(bools='bool')),Intersect(Row(bools='bool'))))
GroupBy(Rows(field='bools'),limit=10)
Rows(field='bools-exists',limit=10)
Count(Row(bools-exists='bool'))
Count(Not(Row(bools-exists='bool')))
Count(Intersect(Row(bools-exists='bool')))
Count(Difference(Union(Row(bools-exists='bool')),Intersect(Row(bools-exists='bool'))))
GroupBy(Rows(field='bools-exists'),limit=10)
GroupBy(Rows(field='int'),limit=10)
Min(field='int')
Max(field='int')
Sum(field='int')
Rows(field='mutex',limit=10)
Count(Row(mutex='12'))
Count(Not(Row(mutex='12')))
Count(Row(mutex='13'))
Count(Not(Row(mutex='13')))
Count(Row(mutex='0'))
Count(Not(Row(mutex='0')))
Count(Row(mutex='5'))
Count(Not(Row(mutex='5')))
Count(Row(mutex='2'))
Count(Not(Row(mutex='2')))
Count(Row(mutex='14'))
Count(Not(Row(mutex='14')))
Count(Row(mutex='3'))
Count(Not(Row(mutex='3')))
Count(Row(mutex='18'))
Count(Not(Row(mutex='18')))
Count(Row(mutex='7'))
Count(Not(Row(mutex='7')))
Count(Row(mutex='19'))
Count(Not(Row(mutex='19')))
Count(Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')))
Count(Difference(Union(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')),Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19'))))
GroupBy(Rows(field='mutex'),limit=10)
Rows(field='set',limit=10)
Count(Row(set='v621'))
Count(Not(Row(set='v621')))
Count(Row(set='v997'))
Count(Not(Row(set='v997')))
Count(Row(set='v772'))
Count(Not(Row(set='v772')))
Count(Row(set='v340'))
Count(Not(Row(set='v340')))
Count(Row(set='v766'))
Count(Not(Row(set='v766')))
Count(Row(set='v416'))
Count(Not(Row(set='v416')))
Count(Row(set='v481'))
Count(Not(Row(set='v481')))
Count(Row(set='v581'))
Count(Not(Row(set='v581')))
Count(Row(set='v591'))
Count(Not(Row(set='v591')))
Count(Row(set='v675'))
Count(Not(Row(set='v675')))
Count(Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')))
Count(Difference(Union(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')),Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675'))))
GroupBy(Rows(field='set'),limit=10)
Rows(field='string',limit=10)
Count(Row(string='KPFGWOYUGTCF'))
Count(Not(Row(string='KPFGWOYUGTCF')))
Count(Row(string='VNSHWDTNELDA'))
Count(Not(Row(string='VNSHWDTNELDA')))
Count(Row(string='LTDOGKKFGZPW'))
Count(Not(Row(string='LTDOGKKFGZPW')))
Count(Row(string='EETNIIDCDZHB'))
Count(Not(Row(string='EETNIIDCDZHB')))
Count(Row(string='ATWOMTRASGHP'))
Count(Not(Row(string='ATWOMTRASGHP')))
Count(Row(string='KOJUBJQVMXZL'))
Count(Not(Row(string='KOJUBJQVMXZL')))
Count(Row(string='NOXDEYPTGZBH'))
Count(Not(Row(string='NOXDEYPTGZBH')))
Count(Row(string='DLSRPRADBPKX'))
Count(Not(Row(string='DLSRPRADBPKX')))
Count(Row(string='WJUEQABAEEUX'))
Count(Not(Row(string='WJUEQABAEEUX')))
Count(Row(string='PFRGBFDLHHPK'))
Count(Not(Row(string='PFRGBFDLHHPK')))
Count(Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')))
Count(Difference(Union(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')),Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK'))))
GroupBy(Rows(field='string'),limit=10)
GroupBy(Rows(field='time'),limit=10)
`
// datagen --source=kitchensink --pilosa.index=testindex --start-from=1 --end-at=1000 --pilosa.batch-size=10000 --pilosa.hosts localhost:10101
// curl localhost:10101/schema
var _ = schemaString
var schemaString = `
{
"indexes": [
{
"name": "testindex",
"createdAt": 1599601704641744600,
"options": {
"keys": false,
"trackExistence": true
},
"fields": [
{
"name": "bools",
"createdAt": 1599601704647053000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": true
}
},
{
"name": "bools-exists",
"createdAt": 1599601704643225900,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": true
}
},
{
"name": "decimal",
"createdAt": 1599601704642064400,
"options": {
"type": "decimal",
"base": 0,
"scale": 2,
"bitDepth": 63,
"min": -92233720368547760,
"max": 92233720368547760,
"keys": false
}
},
{
"name": "int",
"createdAt": 1599601704645510100,
"options": {
"type": "int",
"base": 0,
"bitDepth": 63,
"min": -9223372036854776000,
"max": 9223372036854776000,
"keys": false,
"foreignIndex": ""
}
},
{
"name": "mutex",
"createdAt": 1599601704647187200,
"options": {
"type": "mutex",
"cacheType": "ranked",
"cacheSize": 500,
"keys": true
}
},
{
"name": "set",
"createdAt": 1599601704643664400,
"options": {
"type": "set",
"cacheType": "lru",
"cacheSize": 1,
"keys": true
}
},
{
"name": "string",
"createdAt": 1599601704644342000,
"options": {
"type": "set",
"cacheType": "ranked",
"cacheSize": 50000,
"keys": true
}
},
{
"name": "time",
"createdAt": 1599601704643013400,
"options": {
"type": "time",
"timeQuantum": "YMD",
"keys": true,
"noStandardView": false
}
}
],
"shardWidth": 1048576
}
]
}
`
*/

BIN
synthload/testindex.tar.gz
View file

Binary file not shown.

2
syswrap/mmap.go
View file

@ -17,10 +17,10 @@
package syswrap
import (
"strings"
"sync"
"sync/atomic"
"syscall"
"strings"
"github.com/pkg/errors"
)

1
testhook/cleanup1.14.go
View file

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.14
// +build go1.14
package testhook

2
time.go
View file

@ -288,7 +288,7 @@ func parseTime(t interface{}) (time.Time, error) {
if calcTime, err = time.Parse(TimeFormat, v); err != nil {
// if the default parsing fails, check if user tried to
// supply partial time eg year and month
calcTime, err := parsePartialTime(v)
calcTime, err = parsePartialTime(v)
return calcTime, err
}
case int64:

4
tracing/opentracing/opentracing.go
View file

@ -18,10 +18,10 @@ import (
"context"
"net/http"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/tracing"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
)
// Ensure type implements interface.

18
translator_test.go
View file

@ -24,7 +24,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/mock"
@ -550,9 +550,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
var g errgroup.Group
defer g.Wait() //nolint:errcheck
for i, n := range c.Nodes {
i, api := i, n.API
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.FindIndexKeys(ctx, "i", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
@ -567,9 +567,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
// Check that re-invoking create returns the original translations.
for i, n := range c.Nodes {
i, api := i, n.API
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.CreateIndexKeys(ctx, "i", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
@ -636,9 +636,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
var g errgroup.Group
defer g.Wait() //nolint:errcheck
for i, n := range c.Nodes {
i, api := i, n.API
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.FindFieldKeys(ctx, "i", "f", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")
@ -653,9 +653,9 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
// Check that re-invoking create returns the original translations.
for i, n := range c.Nodes {
i, api := i, n.API
x, api := i, n.API
g.Go(func() (err error) {
defer func() { err = errors.Wrapf(err, "translating on node %d", i) }()
defer func() { err = errors.Wrapf(err, "translating on node %d", x) }()
localTranslations, err := api.CreateFieldKeys(ctx, "i", "f", keyList...)
if err != nil {
return errors.Wrap(err, "finding translations")