diff --git a/Makefile b/Makefile index f93ea4f52..dd9b03606 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ testv-race: topt-race testvsub-race # find which test is hung/deadlocked. # testvsub: - set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ @@ -72,7 +72,7 @@ testvsub: done testvsub-race: - set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ @@ -163,7 +163,7 @@ upgrade-lattice: lattice # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast - $(GO) generate github.com/pilosa/pilosa/v2/internal + $(GO) generate github.com/pilosa/pilosa/v2/pb # `go generate` statik assets (lattice UI) generate-statik: build-lattice require-statik diff --git a/api.go b/api.go index d54dd77d6..665dcf600 100644 --- a/api.go +++ b/api.go @@ -1893,7 +1893,7 @@ func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOf defer func() { if err != nil { for i := range a { - a[i].Close() + a[i].Close() // nolint: errcheck } } }() diff --git a/api_test.go b/api_test.go index bbd79f36a..179d664c1 100644 --- a/api_test.go +++ b/api_test.go @@ -30,6 +30,7 @@ import ( "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" + . "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck ) // attrFun defines a mapping from columnID -> attr value @@ -46,7 +47,7 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { 10000 5.156 100000 38.179 */ - c := test.MustRunCluster(t, 2, + c := test.MustRunCluster(t, 3, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), @@ -57,11 +58,17 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&offsetModHasher{}), )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node2"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + )}, ) defer c.Close() m0 := c.GetNode(0) m1 := c.GetNode(1) + t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() indexName := "i" @@ -111,7 +118,7 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { IndexCreatedAt: index.CreatedAt(), } - if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { + if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { t.Fatal(err) } @@ -125,7 +132,7 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { IndexCreatedAt: index.CreatedAt(), } - if err := m0.API.ImportColumnAttrs(ctx, req); err != nil { + if err := m1.API.ImportColumnAttrs(ctx, req); err != nil { t.Fatal(err) } @@ -166,7 +173,7 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } func TestAPI_Import(t *testing.T) { - c := test.MustRunCluster(t, 2, + c := test.MustRunCluster(t, 3, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), @@ -181,6 +188,13 @@ func TestAPI_Import(t *testing.T) { pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node2"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + )}, ) defer c.Close() @@ -243,7 +257,7 @@ func TestAPI_Import(t *testing.T) { if err := m0.API.Import(ctx, qcx, req); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) @@ -287,7 +301,7 @@ func TestAPI_Import(t *testing.T) { } func TestAPI_ImportValue(t *testing.T) { - c := test.MustRunCluster(t, 2, + c := test.MustRunCluster(t, 3, []server.CommandOption{ server.OptCommandServerOptions( pilosa.OptServerNodeID("node0"), @@ -300,12 +314,19 @@ func TestAPI_ImportValue(t *testing.T) { pilosa.OptServerClusterHasher(&offsetModHasher{}), pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node2"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + )}, ) defer c.Close() coord := c.GetPrimary() m0 := c.GetNode(0) m1 := c.GetNode(1) + m2 := c.GetNode(2) t.Run("ValColumnKey", func(t *testing.T) { ctx := context.Background() @@ -343,7 +364,7 @@ func TestAPI_ImportValue(t *testing.T) { if err := coord.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) pql := fmt.Sprintf("Row(%s>0)", field) @@ -371,16 +392,14 @@ func TestAPI_ImportValue(t *testing.T) { ctx := context.Background() index := "valdec" field := "fdec" - - _, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) + _, err := m2.API.CreateIndex(ctx, index, pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = m1.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1)) + _, err = m2.API.CreateField(ctx, index, field, pilosa.OptFieldTypeDecimal(1)) if err != nil { t.Fatalf("creating field: %v", err) } - // Generate some records. values := []float64{} colIDs := []uint64{} @@ -388,7 +407,6 @@ func TestAPI_ImportValue(t *testing.T) { values = append(values, float64(i)+0.1) colIDs = append(colIDs, uint64(i)) } - // Import data with keys to node1 and verify that it gets translated and // forwarded to the owner of shard 0 (node0; because of offsetModHasher) req := &pilosa.ImportValueRequest{ @@ -397,15 +415,12 @@ func TestAPI_ImportValue(t *testing.T) { ColumnIDs: colIDs, FloatValues: values, } - - qcx := m1.API.Txf().NewQcx() - if err := m1.API.ImportValue(ctx, qcx, req); err != nil { + qcx := m0.API.Txf().NewQcx() + if err := m0.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) - + PanicOn(qcx.Finish()) query := fmt.Sprintf("Row(%s>6)", field) - // Query node0. if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil { t.Fatal(err) @@ -475,7 +490,7 @@ func TestAPI_ImportValue(t *testing.T) { if err := m0.API.ImportValue(ctx, qcx, req); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) pql := fmt.Sprintf(`Row(%s=="strval-110")`, field) @@ -565,12 +580,12 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) bitIsSet := func() bool { query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - panicOn(err) + PanicOn(err) cols := res.Results[0].(*pilosa.Row).Columns() for i := range cols { if cols[i] == acctOwnerID { @@ -581,13 +596,13 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { } if !bitIsSet() { - panic("IRA bit should have been set") + PanicOn("IRA bit should have been set") } queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) { query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID) res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query}) - panicOn(err) + PanicOn(err) if len(res.Results) == 0 { return 0 @@ -599,7 +614,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index) if bal != acct0bal { - panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal)) + PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal)) } // clear the bit @@ -608,10 +623,10 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { if err := m0api.Import(ctx, qcx, ir0); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) if bitIsSet() { - panic("IRA bit should have been cleared") + PanicOn("IRA bit should have been cleared") } // clear the BSI @@ -620,11 +635,11 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil { t.Fatal(err) } - panicOn(qcx.Finish()) + PanicOn(qcx.Finish()) bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index) if bal != 0 { - panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0)) + PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0)) } } diff --git a/attr.go b/attr.go index 3502688d1..66667f806 100644 --- a/attr.go +++ b/attr.go @@ -19,7 +19,7 @@ import ( "sort" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/v2/internal" + "github.com/pilosa/pilosa/v2/pb" ) // Attribute data type enum. @@ -119,21 +119,21 @@ func (a attrBlocks) Diff(other []AttrBlock) []uint64 { } } -func encodeAttrs(m map[string]interface{}) []*internal.Attr { +func encodeAttrs(m map[string]interface{}) []*pb.Attr { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } sort.Strings(keys) - a := make([]*internal.Attr, len(keys)) + a := make([]*pb.Attr, len(keys)) for i := range keys { a[i] = encodeAttr(keys[i], m[keys[i]]) } return a } -func decodeAttrs(pb []*internal.Attr) map[string]interface{} { +func decodeAttrs(pb []*pb.Attr) map[string]interface{} { m := make(map[string]interface{}, len(pb)) for i := range pb { key, value := decodeAttr(pb[i]) @@ -142,9 +142,9 @@ func decodeAttrs(pb []*internal.Attr) map[string]interface{} { return m } -// encodeAttr converts a key/value pair into an Attr internal representation. -func encodeAttr(key string, value interface{}) *internal.Attr { - pb := &internal.Attr{Key: key} +// encodeAttr converts a key/value pair into an Attr pb.representation. +func encodeAttr(key string, value interface{}) *pb.Attr { + pb := &pb.Attr{Key: key} switch value := value.(type) { case string: pb.Type = attrTypeString @@ -165,8 +165,8 @@ func encodeAttr(key string, value interface{}) *internal.Attr { return pb } -// decodeAttr converts from an Attr internal representation to a key/value pair. -func decodeAttr(attr *internal.Attr) (key string, value interface{}) { +// decodeAttr converts from an Attr pb.representation to a key/value pair. +func decodeAttr(attr *pb.Attr) (key string, value interface{}) { switch attr.Type { case attrTypeString: return attr.Key, attr.StringValue @@ -192,12 +192,12 @@ func cloneAttrs(m map[string]interface{}) map[string]interface{} { // EncodeAttrs encodes an attribute map into a byte slice. func EncodeAttrs(attr map[string]interface{}) ([]byte, error) { - return proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)}) + return proto.Marshal(&pb.AttrMap{Attrs: encodeAttrs(attr)}) } // DecodeAttrs decodes a byte slice into an attribute map. func DecodeAttrs(v []byte) (map[string]interface{}, error) { - var pb internal.AttrMap + var pb pb.AttrMap if err := proto.Unmarshal(v, &pb); err != nil { return nil, err } diff --git a/bluegreentx.go b/bluegreentx.go index e0d9ddee9..666389315 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -23,7 +23,7 @@ import ( "github.com/pilosa/pilosa/v2/roaring" txkey "github.com/pilosa/pilosa/v2/short_txkey" - //txkey "github.com/pilosa/pilosa/v2/txkey" + . "github.com/pilosa/pilosa/v2/vprint" ) // blueGreenTx runs two Tx together and notices differences in their output. @@ -129,7 +129,7 @@ func (b *blueGreenRegistry) Close() { b.mu.Lock() defer b.mu.Unlock() if len(b.m) > 0 { - panic(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) + PanicOn(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) //AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m) } } @@ -191,7 +191,7 @@ func (c *blueGreenTx) Readonly() bool { a := c.a.Readonly() b := c.b.Readonly() if a != b { - panic(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) + PanicOn(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) } return b } @@ -240,41 +240,41 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { if aFound != bFound { c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, Stack())) } if aErr != nil || bErr != nil { if aErr != nil && bErr != nil { c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, Stack())) } if aErr != nil { c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, Stack())) } if bErr != nil { c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, Stack())) } } for aIter.Next() { aKey, aValue := aIter.Value() if !bIter.Next() { - AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, stack()) + AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, Stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, Stack()) c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, Stack())) } bKey, bValue := bIter.Value() if bKey != aKey { AlwaysPrintf("problem in caller %v", Caller(2)) c.Dump(c.short, shard) - panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, Stack())) } if err := aValue.BitwiseCompare(bValue); err != nil { c.Dump(c.short, shard) - //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()) - panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())) + //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack()) + PanicOn(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack())) } //vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey) } @@ -283,7 +283,7 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn()) c.Dump(c.short, shard) bKey, _ := bIter.Value() - panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), stack())) + PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), Stack())) } //vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) } @@ -341,7 +341,7 @@ func (c *blueGreenTx) Rollback() { c.mu.Lock() defer c.mu.Unlock() if c.rollbackOrCommitDone { - return // avoid using discarded tx for Dump, which will panic. + return // avoid using discarded tx for Dump, which will PanicOn. } c.rollbackOrCommitDone = true @@ -350,8 +350,8 @@ func (c *blueGreenTx) Rollback() { } defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() //vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as) @@ -379,8 +379,8 @@ func (c *blueGreenTx) Commit() error { } defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() errA := c.a.Commit() @@ -396,8 +396,8 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() a, errA := c.a.RoaringBitmap(index, field, view, shard) @@ -409,7 +409,7 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r slcA := a.Slice() slcB := b.Slice() if !reflect.DeepEqual(slcA, slcB) { - panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") + PanicOn("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") } } return b, errB @@ -419,8 +419,8 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() a, errA := c.a.Container(index, field, view, shard, key) @@ -429,7 +429,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin if !c.o.blueGreenOff { compareErrors(errA, errB) err = a.BitwiseCompare(b) - panicOn(err) + PanicOn(err) } return b, errB } @@ -438,8 +438,8 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() errA := c.a.PutContainer(index, field, view, shard, key, rc) @@ -463,14 +463,14 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, // ================== end save comments. defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() // remember where the iterator started, so we can replay it a second time. rit2 := rit.Clone() - panicOn(err) + PanicOn(err) changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) @@ -482,18 +482,18 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will // be inaccurate. if changedA != changedB { - panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) + PanicOn(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) } if len(rowSetA) != len(rowSetB) { - panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) + PanicOn(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) } for k, va := range rowSetA { vb, ok := rowSetB[k] if !ok { - panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) + PanicOn(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) } if va != vb { - panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) + PanicOn(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) } } } @@ -507,8 +507,8 @@ func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, k c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() errA := c.a.RemoveContainer(index, field, view, shard, key) @@ -531,9 +531,9 @@ var _ = (&blueGreenTx{}).isIn // happy linter func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) { r = make([]bool, 2) inA, errA := c.a.Contains(index, field, view, shard, ckey) - panicOn(errA) + PanicOn(errA) inB, errB := c.b.Contains(index, field, view, shard, ckey) - panicOn(errB) + PanicOn(errB) r[0] = inA r[1] = inB return @@ -544,8 +544,8 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, //vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack()) - panic(r) + AlwaysPrintf("see Add() PanicOn '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, Stack()) + PanicOn(r) } }() @@ -562,7 +562,7 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, if !c.o.blueGreenOff { if ach != bch { - panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) + PanicOn(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) } compareErrors(errA, errB) } @@ -575,14 +575,14 @@ func compareErrors(errA, errB error) { case errA == nil && errB == nil: // OK case errA == nil: - panic(fmt.Sprintf("errA is nil, but errB = %#v", errB)) + PanicOn(fmt.Sprintf("errA is nil, but errB = %#v", errB)) case errB == nil: - panic(fmt.Sprintf("errB is nil, but errA = %#v", errA)) + PanicOn(fmt.Sprintf("errB is nil, but errA = %#v", errA)) default: ae := errA.Error() be := errB.Error() if ae != be { - panic(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be)) + PanicOn(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be)) } } } @@ -591,8 +591,8 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6 c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() ach, errA := c.a.Remove(index, field, view, shard, a...) @@ -609,8 +609,8 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() ax, errA := c.a.Contains(index, field, view, shard, key) @@ -627,8 +627,8 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() @@ -690,7 +690,7 @@ func (bgi *blueGreenIterator) Next() bool { na := bgi.ait.Next() nb := bgi.bit.Next() if na != nb { - panic(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) + PanicOn(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) } return nb } @@ -701,10 +701,10 @@ func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { if !bgi.tx.o.blueGreenOff { if ka != kb { - panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) + PanicOn(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) } err := ca.BitwiseCompare(cb) - panicOn(err) + PanicOn(err) } return kb, cb } @@ -718,8 +718,8 @@ func (bgi *blueGreenIterator) Close() { func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.ForEach(index, field, view, shard, fn) @@ -733,8 +733,8 @@ func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, star defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() @@ -748,8 +748,8 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() a, errA := c.a.Count(index, field, view, shard) @@ -767,8 +767,8 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() a, errA := c.a.Max(index, field, view, shard) @@ -786,8 +786,8 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() amin, afound, errA := c.a.Min(index, field, view, shard) @@ -805,8 +805,8 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() errA := c.a.UnionInPlace(index, field, view, shard, others...) @@ -822,8 +822,8 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, defer func() { if r := recover(); r != nil { c.Dump(c.short, shard) - AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() a, errA := c.a.CountRange(index, field, view, shard, start, end) @@ -831,7 +831,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, if !c.o.blueGreenOff { if a != b { - panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) + PanicOn(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) } compareErrors(errA, errB) @@ -843,8 +843,8 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() on _sn_ %v, panic '%v' at '%v'", c.Sn(), r, stack()) - panic(r) + AlwaysPrintf("see OffsetRange() on _sn_ %v, PanicOn '%v' at '%v'", c.Sn(), r, Stack()) + PanicOn(r) } }() a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end) @@ -855,7 +855,7 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star err = roaringBitmapDiff(a, b) if err != nil { c.Dump(false, shard) - panicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) + PanicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) } compareErrors(errA, errB) } @@ -868,8 +868,8 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6 defer func() { if r := recover(); r != nil { c.Dump(c.short, shard) - AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() @@ -891,7 +891,7 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6 } if sizeMustMatch { if szA != szB { - panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) + PanicOn(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) } return &MultiReaderB{a: rcA, b: rcB}, szB, errB } else { @@ -955,14 +955,14 @@ func (m *MultiReaderB) Read(p []byte) (nB int, errB error) { if !m.allowSizeVariation { if errA == io.ErrUnexpectedEOF { - panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) + PanicOn(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) } if nA != nB { - panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) + PanicOn(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) } cmp := bytes.Compare(p[:nB], p2[:nB]) if cmp != 0 { - panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) + PanicOn(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) } } return diff --git a/bluegreentx_test.go b/bluegreentx_test.go index 7455f2c6d..a614e3501 100644 --- a/bluegreentx_test.go +++ b/bluegreentx_test.go @@ -24,6 +24,8 @@ import ( "testing" cryrand "crypto/rand" + + . "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck ) var _ = context.Background @@ -42,7 +44,7 @@ func TestMultiReaderB(t *testing.T) { nr := 0 for nr < n { na, err := src.Read(a) - panicOn(err) + PanicOn(err) nr += na } if nr != n { @@ -62,7 +64,7 @@ func TestMultiReaderB(t *testing.T) { // should not trigger the internal panic of MultiReadB ncp, err := io.Copy(ioutil.Discard, m) - panicOn(err) + PanicOn(err) if ncp != int64(n) { panic("short copy") } diff --git a/bolt.go b/bolt.go index 5d6899f97..5a942dcb0 100644 --- a/bolt.go +++ b/bolt.go @@ -31,6 +31,7 @@ import ( "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/storage" + . "github.com/pilosa/pilosa/v2/vprint" // On Bolt only, we still use the long txkey, because // this allows Max() to work readily. @@ -132,7 +133,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag dir := filepath.Dir(path) if !DirExists(path) { - panicOn(os.MkdirAll(dir, 0755)) + PanicOn(os.MkdirAll(dir, 0755)) } db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) @@ -522,7 +523,7 @@ func (tx *BoltTx) Commit() error { defer tx.mu.Unlock() err := tx.tx.Commit() - panicOn(err) + PanicOn(err) tx.o.dbs.Cleanup(tx) return err @@ -657,15 +658,15 @@ func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, batched, // not first time through, write what we got. if remove && (rc == nil || rc.N() == 0) { err = tx.RemoveContainer(index, field, view, shard, lastHi) - panicOn(err) + PanicOn(err) } else { err = tx.PutContainer(index, field, view, shard, lastHi, rc) - panicOn(err) + PanicOn(err) } } // get the next container rc, err = tx.Container(index, field, view, shard, hi) - panicOn(err) + PanicOn(err) } // else same container, keep adding bits to rct. chng := false // rc can be nil before, and nil after, in both Remove/Add below. @@ -684,17 +685,17 @@ func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, batched, if remove { if rc == nil || rc.N() == 0 { err = tx.RemoveContainer(index, field, view, shard, hi) - panicOn(err) + PanicOn(err) } else { err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) + PanicOn(err) } } else { if rc == nil || rc.N() == 0 { panic("there should be no way to have an empty bitmap AFTER an Add() operation") } err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) + PanicOn(err) } return } @@ -955,7 +956,7 @@ type boltFinder struct { // FindIterator lets boltFinder implement the roaring.FindIterator interface. func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - panicOn(err) + PanicOn(err) bf.needClose = append(bf.needClose, a) return a, found } @@ -1015,7 +1016,7 @@ func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, e func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) { a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - panicOn(err) + PanicOn(err) defer a.Close() if !found { return 0, nil @@ -1101,7 +1102,7 @@ func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, er func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) + PanicOn(err) rbm.UnionInPlace(others...) // iterate over the containers that changed within rbm, and write them back to disk. @@ -1115,7 +1116,7 @@ func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others . // TODO: only write the changed ones back, as optimization? // Compare to ImportRoaringBits. err := tx.PutContainer(index, field, view, shard, containerKey, rc) - panicOn(err) + PanicOn(err) } return nil } @@ -1133,7 +1134,7 @@ func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) _ = found - panicOn(err) + PanicOn(err) defer citer.Close() @@ -1265,7 +1266,7 @@ func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr // INVAR: nsynth > 0 oldC, err = tx.Container(index, field, view, shard, itrKey) - panicOn(err) + PanicOn(err) if err != nil { return } @@ -1346,7 +1347,7 @@ func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr err = tx.PutContainer(index, field, view, shard, itrKey, newC) if err != nil { - panicOn(err) + PanicOn(err) return } continue @@ -1567,7 +1568,7 @@ func (w *BoltWrapper) DeletePrefix(prefix []byte) error { w.muDb.Unlock() err := tx.Commit() - panicOn(err) + PanicOn(err) return nil } diff --git a/bolt_test.go b/bolt_test.go index 9dac07cca..13485161c 100644 --- a/bolt_test.go +++ b/bolt_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/pilosa/pilosa/v2/roaring" + . "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck ) // helpers, each runs their own new txn, and commits if a change/delete @@ -31,7 +32,7 @@ func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) } @@ -44,7 +45,7 @@ func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, sha tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if exists { panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) } @@ -59,36 +60,36 @@ func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard u if changed != 1 { panic("should have 1 bit changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if !exists { panic("ARG putme was NOT SET!!!") } - panicOn(tx.Commit()) + PanicOn(tx.Commit()) } func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) hi := highbits(putme) - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) - panicOn(tx.Commit()) + PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) + PanicOn(tx.Commit()) } func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) _, err := tx.Remove(index, field, view, shard, putme) - panicOn(err) - panicOn(tx.Commit()) + PanicOn(err) + PanicOn(tx.Commit()) } func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { var err error fn := path - panicOn(os.RemoveAll(fn)) + PanicOn(os.RemoveAll(fn)) ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil) - panicOn(err) + PanicOn(err) w = ww.(*BoltWrapper) // verify it is empty @@ -99,7 +100,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { return w, func() { w.Close() - panicOn(os.RemoveAll(fn)) + PanicOn(os.RemoveAll(fn)) } } @@ -126,28 +127,28 @@ func TestBolt_DeleteFragment(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) } } for _, view := range views { for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } } } err := tx.Commit() - panicOn(err) + PanicOn(err) // end of setup victim := "v1" survivor := "v2" err = dbwrap.DeleteFragment(index, field, victim, shard, nil) - panicOn(err) + PanicOn(err) tx, _ = dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() @@ -155,7 +156,7 @@ func TestBolt_DeleteFragment(t *testing.T) { for _, view := range views { for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if view == survivor { if !exists { panic(fmt.Sprintf("ARG survivor died : bit %v", v)) @@ -209,7 +210,7 @@ func TestBolt_Max_on_many_containers(t *testing.T) { for _, shard := range shards { max, err := tx.Max(index, field, view, uint64(shard)) - panicOn(err) + PanicOn(err) if max != uint64(shard) { panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) } @@ -217,12 +218,12 @@ func TestBolt_Max_on_many_containers(t *testing.T) { // check for not found max, err := tx.Max(index, field, view, uint64(200)) - panicOn(err) + PanicOn(err) if max != 0 { panic("expected not found to give 0 max back with nil err") } max, err = tx.Max(index, field, view, uint64(400)) - panicOn(err) + PanicOn(err) if max != 0 { panic("expected not found to give 0 max back with nil err") } @@ -242,16 +243,16 @@ func TestBolt_SetBitmap(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } err = tx.Commit() - panicOn(err) + PanicOn(err) // // commited, so should be visible outside the txn @@ -259,13 +260,13 @@ func TestBolt_SetBitmap(t *testing.T) { tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) exists, err = tx2.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!! on tx2") } n, err := tx2.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if n != 1 { panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) } @@ -284,28 +285,28 @@ func TestBolt_OffsetRange(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) bitvalue2 := uint64(1<<20 + 1) changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2) if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } exists, err = tx.Contains(index, field, view, shard, bitvalue2) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue2 was NOT SET!!!") } err = tx.Commit() - panicOn(err) + PanicOn(err) offset := uint64(0 << 20) start := uint64(0 << 16) @@ -313,7 +314,7 @@ func TestBolt_OffsetRange(t *testing.T) { tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) + PanicOn(err) tx2.Rollback() // should see our 1M value @@ -327,7 +328,7 @@ func TestBolt_OffsetRange(t *testing.T) { offset = uint64(2 << 20) tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) + PanicOn(err) tx3.Rollback() //expect to see 3M == 3145728 @@ -357,7 +358,7 @@ func TestBolt_Count_on_many_containers(t *testing.T) { defer tx.Rollback() n, err := tx.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if int(n) != len(putmeValues) { panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) } @@ -374,7 +375,7 @@ func TestBolt_Count_dense_containers(t *testing.T) { expected := 0 for i := uint64(0); i < (1<<16)+2; i += 2 { changed, err := tx.Add(index, field, view, shard, doBatched, i) - panicOn(err) + PanicOn(err) if changed <= 0 { panic("wat? should have changed") } @@ -383,7 +384,7 @@ func TestBolt_Count_dense_containers(t *testing.T) { defer tx.Rollback() n, err := tx.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if int(n) != expected { panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) } @@ -399,12 +400,12 @@ func TestBolt_ContainerIterator_on_empty(t *testing.T) { defer tx.Rollback() bitvalue := uint64(0) citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) defer citer.Close() if found { panic("should not have found anything") } - panicOn(err) + PanicOn(err) } func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { @@ -423,10 +424,10 @@ func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } @@ -437,7 +438,7 @@ func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { if !found { panic("ContainerIterator did not find the 42 bit") } - panicOn(err) + PanicOn(err) defer citer.Close() loopCount := 0 @@ -481,10 +482,10 @@ func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if !exists { panic("ARG putme was NOT SET!!!") } @@ -517,7 +518,7 @@ func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { break } } - panicOn(err) + PanicOn(err) } func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { @@ -536,10 +537,10 @@ func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if !exists { panic("ARG putme was NOT SET!!!") } @@ -547,7 +548,7 @@ func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { // same Tx, continues in use. citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - panicOn(err) + PanicOn(err) if found { panic("ContainerIterator found the searchme, when it should not have") } @@ -585,10 +586,10 @@ func TestBolt_ForEach_on_one_bit(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } @@ -602,7 +603,7 @@ func TestBolt_ForEach_on_one_bit(t *testing.T) { count += 1 return nil }) - panicOn(err) + PanicOn(err) if count != 1 { panic(fmt.Sprintf("Expected single iteration got %v ", count)) } @@ -637,7 +638,7 @@ func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { // delete, but rollback instead of commit tx, _ := dbwrap.NewTx(writable, index, Txo{}) hi := highbits(putme) - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) + PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) tx.Rollback() // verify that the rollback undid the deletion. @@ -648,15 +649,15 @@ func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { hi = highbits(putme) exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) } - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) + PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) exists, err = tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if exists { panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) } @@ -700,7 +701,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) { hi, lo := highbits(putme), lowbits(putme) _, _ = hi, lo _, err := tx.Remove(index, field, view, shard, hi) - panicOn(err) + PanicOn(err) tx.Rollback() // verify that the rollback undid the deletion. @@ -710,7 +711,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) { tx, _ = dbwrap.NewTx(writable, index, Txo{}) exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) } @@ -718,7 +719,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) { mustRemove(tx.Remove(index, field, view, shard, putme)) exists, err = tx.Contains(index, field, view, shard, putme) - panicOn(err) + PanicOn(err) if exists { panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) } @@ -742,7 +743,7 @@ func TestBolt_Min_on_many_containers(t *testing.T) { tx, _ := dbwrap.NewTx(!writable, index, Txo{}) min, containersExist, err := tx.Min(index, field, view, shard) _ = min - panicOn(err) + PanicOn(err) if containersExist { panic("no containers should exist") } @@ -760,7 +761,7 @@ func TestBolt_Min_on_many_containers(t *testing.T) { defer tx.Rollback() min, containersExist, err = tx.Min(index, field, view, shard) - panicOn(err) + PanicOn(err) if !containersExist { panic("containers should exist") } @@ -779,7 +780,7 @@ func TestBolt_CountRange_on_many_containers(t *testing.T) { // verify no containers flag works tx, _ := dbwrap.NewTx(!writable, index, Txo{}) n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - panicOn(err) + PanicOn(err) if n != 0 { panic("no containers should exist") } @@ -797,7 +798,7 @@ func TestBolt_CountRange_on_many_containers(t *testing.T) { defer tx.Rollback() n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - panicOn(err) + PanicOn(err) if n == 0 { panic("containers should exist") } @@ -826,7 +827,7 @@ func TestBolt_CountRange_middle_container(t *testing.T) { // pick out just the middle container with the 1 bit set on it. n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) - panicOn(err) + PanicOn(err) if n != 1 { panic("middle 1 bit container should exist") } @@ -851,7 +852,7 @@ func TestBolt_CountRange_many_middle_container(t *testing.T) { // get them all n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) - panicOn(err) + PanicOn(err) if n != 3 { panic("count should have been all 3 bits") } @@ -879,7 +880,7 @@ func TestBolt_UnionInPlace(t *testing.T) { tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) n, err := tx2.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if n != 2 { panic("should have 2 bits set") } @@ -895,11 +896,11 @@ func TestBolt_UnionInPlace(t *testing.T) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) defer tx.Rollback() err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) - panicOn(err) + PanicOn(err) // end game, check we got the union. rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) + PanicOn(err) n = rbm.Count() if n != 7 { panic("should have a total 3 + 3 +1 = 7 bits set on the containers") @@ -921,7 +922,7 @@ func TestBolt_RoaringBitmap(t *testing.T) { defer tx.Rollback() rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) + PanicOn(err) slc := rbm.Slice() if slc[0] != uint64(expected) { @@ -947,7 +948,7 @@ func TestBolt_ImportRoaringBits(t *testing.T) { bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} data := getTestBitmapAsRawRoaring(bits...) itr, err := roaring.NewRoaringIterator(data) - panicOn(err) + PanicOn(err) clear := false logme := false @@ -956,11 +957,11 @@ func TestBolt_ImportRoaringBits(t *testing.T) { if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) } - panicOn(err) + PanicOn(err) for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) } @@ -973,11 +974,11 @@ func TestBolt_ImportRoaringBits(t *testing.T) { if changed != 0 { panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) } - panicOn(err) + PanicOn(err) for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) } @@ -990,17 +991,17 @@ func TestBolt_ImportRoaringBits(t *testing.T) { // clear 1 bit at a time data := getTestBitmapAsRawRoaring(v) itr, err := roaring.NewRoaringIterator(data) - panicOn(err) + PanicOn(err) changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != 1 { panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) } - panicOn(err) + PanicOn(err) } n, err := tx.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if n != 0 { panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) } @@ -1027,12 +1028,12 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} data := getTestBitmapAsRawRoaring(bits...) itr, err := roaring.NewRoaringIterator(data) - panicOn(err) + PanicOn(err) bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} data2 := getTestBitmapAsRawRoaring(bits2...) itr2, err := roaring.NewRoaringIterator(data2) - panicOn(err) + PanicOn(err) clear := false logme := false @@ -1042,11 +1043,11 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) } - panicOn(err) + PanicOn(err) for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) } @@ -1059,7 +1060,7 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { if changed != 4 { panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) } - panicOn(err) + PanicOn(err) } func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { @@ -1077,12 +1078,12 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} data := getTestBitmapAsRawRoaring(bits...) itr, err := roaring.NewRoaringIterator(data) - panicOn(err) + PanicOn(err) bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} data2 := getTestBitmapAsRawRoaring(bits2...) itr2, err := roaring.NewRoaringIterator(data2) - panicOn(err) + PanicOn(err) clear := false logme := false @@ -1092,11 +1093,11 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) } - panicOn(err) + PanicOn(err) for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) } @@ -1110,10 +1111,10 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { if changed != 2 { panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) } - panicOn(err) + PanicOn(err) n, err := tx.Count(index, field, view, shard) - panicOn(err) + PanicOn(err) if n != 2 { // just the 0 and the 1<<16 bits should be left set. panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) } @@ -1135,7 +1136,7 @@ func TestBolt_DeleteIndex(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) } index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' @@ -1143,38 +1144,38 @@ func TestBolt_DeleteIndex(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) for _, v := range bits { exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!!") } } exists, err := tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic("ARG bitvalue was NOT SET!!! on index2") } err = tx.Commit() - panicOn(err) + PanicOn(err) // end of setup err = dbwrap.DeleteIndex(index) - panicOn(err) + PanicOn(err) tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) defer tx.Rollback() exists, err = tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) } for _, v := range bits { exists, err = tx.Contains(index, field, view, shard, v) - panicOn(err) + PanicOn(err) if exists { allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) @@ -1199,9 +1200,9 @@ func TestBolt_DeleteIndex_over100k(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) if v%100000 == 0 { - panicOn(tx.Commit()) + PanicOn(tx.Commit()) tx, _ = dbwrap.NewTx(writable, index, Txo{}) } } @@ -1211,25 +1212,25 @@ func TestBolt_DeleteIndex_over100k(t *testing.T) { if changed <= 0 { panic("should have changed") } - panicOn(err) + PanicOn(err) err = tx.Commit() - panicOn(err) + PanicOn(err) // end of setup err = dbwrap.DeleteIndex(index) - panicOn(err) + PanicOn(err) tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) defer tx.Rollback() exists, err := tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) + PanicOn(err) if !exists { panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) } for v := uint64(0); v < limit; v++ { exists, err = tx.Contains(index, field, view, shard, v<<16) - panicOn(err) + PanicOn(err) if exists { allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) diff --git a/catcher.go b/catcher.go index 09f24ce93..983499fef 100644 --- a/catcher.go +++ b/catcher.go @@ -20,13 +20,13 @@ import ( "github.com/pilosa/pilosa/v2/roaring" txkey "github.com/pilosa/pilosa/v2/short_txkey" - //txkey "github.com/pilosa/pilosa/v2/txkey" + . "github.com/pilosa/pilosa/v2/vprint" ) // catcher is useful to report error locations with a -// stack dump before the complexity +// Stack dump before the complexity // of the executor_test swallows up -// the location of a panic. +// the location of a PanicOn. type catcherTx struct { b Tx } @@ -53,8 +53,8 @@ func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roar func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) @@ -67,8 +67,8 @@ func (c *catcherTx) Dump(short bool, shard uint64) { func (c *catcherTx) Readonly() bool { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Readonly() @@ -81,8 +81,8 @@ func (tx *catcherTx) Pointer() string { func (c *catcherTx) Rollback() { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() c.b.Rollback() @@ -92,8 +92,8 @@ func (c *catcherTx) Commit() error { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Commit() @@ -103,8 +103,8 @@ func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roa defer func() { if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.RoaringBitmap(index, field, view, shard) @@ -114,8 +114,8 @@ func (c *catcherTx) Container(index, field, view string, shard uint64, key uint6 defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Container(index, field, view, shard, key) @@ -125,8 +125,8 @@ func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key ui defer func() { if r := recover(); r != nil { - AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.PutContainer(index, field, view, shard, key, rc) @@ -136,8 +136,8 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key defer func() { if r := recover(); r != nil { - AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.RemoveContainer(index, field, view, shard, key) @@ -155,8 +155,8 @@ func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Add(index, field, view, shard, batched, a...) @@ -166,8 +166,8 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Remove(index, field, view, shard, a...) @@ -177,8 +177,8 @@ func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64 defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Contains(index, field, view, shard, key) @@ -188,8 +188,8 @@ func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, f defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) @@ -199,8 +199,8 @@ func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i u defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.ForEach(index, field, view, shard, fn) @@ -210,8 +210,8 @@ func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, defer func() { if r := recover(); r != nil { - AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.ForEachRange(index, field, view, shard, start, end, fn) @@ -221,8 +221,8 @@ func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, erro defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Count(index, field, view, shard) @@ -232,8 +232,8 @@ func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Max(index, field, view, shard) @@ -243,8 +243,8 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, defer func() { if r := recover(); r != nil { - AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.Min(index, field, view, shard) @@ -254,8 +254,8 @@ func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others defer func() { if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.UnionInPlace(index, field, view, shard, others...) @@ -265,8 +265,8 @@ func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, e defer func() { if r := recover(); r != nil { - AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.CountRange(index, field, view, shard, start, end) @@ -276,8 +276,8 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, defer func() { if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.OffsetRange(index, field, view, shard, offset, start, end) @@ -286,8 +286,8 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { defer func() { if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack()) - panic(r) + AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) + PanicOn(r) } }() return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) diff --git a/client.go b/client.go index fd2bf45e0..7da57a9f7 100644 --- a/client.go +++ b/client.go @@ -47,7 +47,8 @@ type FieldValue struct { // something hasn't been architected correctly. // While I understand that putting the entire Client behind an interface might require this many methods, // I don't want to let it go unquestioned. -// Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client. +// Another note from Travis: I think we eventually want to unify `InternalClient` with +// the `github.com/pilosa/pilosa/v2/client` client. // Doing that may obviate the need to refactor this. type InternalClient interface { InternalQueryClient diff --git a/client/README.md b/client/README.md new file mode 100644 index 000000000..9af16df33 --- /dev/null +++ b/client/README.md @@ -0,0 +1,85 @@ +# Go Client for Pilosa + +Go client for Pilosa high performance distributed index. + +## Usage + +If you have the pilosa repo in your `GOPATH`, +you can import the library in your code using: + +```go +import "github.com/pilosa/pilosa/v2/client" +``` + + +### Quick overview + +Assuming [Pilosa](https://github.com/pilosa/pilosa) server is running at `localhost:10101` (the default): + +```go +package main + +import ( + "fmt" + + "github.com/pilosa/pilosa/v2/client" +) + +func main() { + // Create the default client + cli := client.DefaultClient() + + // Retrieve the schema + schema, err := cli.Schema() + + // Create an Index object + myindex := schema.Index("myindex") + + // Create a Field object + myfield := myindex.Field("myfield") + + // make sure the index and the field exists on the server + err := cli.SyncSchema(schema) + + // Send a Set query. If err is non-nil, response will be nil. + response, err := cli.Query(myfield.Set(5, 42)) + + // Send a Row query. If err is non-nil, response will be nil. + response, err = cli.Query(myfield.Row(5)) + + // Get the result + result := response.Result() + // Act on the result + if result != nil { + columns := result.Row().Columns + fmt.Println("Got columns: ", columns) + } + + // You can batch queries to improve throughput + response, err = cli.Query(myindex.BatchQuery( + myfield.Row(5), + myfield.Row(10))) + if err != nil { + fmt.Println(err) + } + + for _, result := range response.Results() { + // Act on the result + fmt.Println(result.Row().Columns) + } +} +``` + +## Documentation + +### Data Model and Queries + +See: [Data Model and Queries](docs/data-model-queries.md) + +### Executing Queries + +See: [Server Interaction](docs/server-interaction.md) + +### Other Documentation + +* [Tracing](docs/tracing.md) \ No newline at end of file diff --git a/client/batch.go b/client/batch.go new file mode 100644 index 000000000..753ee8b4d --- /dev/null +++ b/client/batch.go @@ -0,0 +1,1404 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "time" + + "github.com/pilosa/pilosa/v2/client/egpool" + "github.com/pilosa/pilosa/v2/logger" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" +) + +// Batch defaults. +const ( + DefaultKeyTranslateBatchSize = 100000 +) + +// TODO if using column translation, column ids might get way out of +// order. Could be worth sorting everything after translation (as an +// option?). Instead of sorting all simultaneously, it might be faster +// (more cache friendly) to sort ids and save the swap ops to apply to +// everything else that needs to be sorted. + +// TODO support clearing values? nil values in records are ignored, +// but perhaps we could have a special type indicating that a bit or +// value should explicitly be cleared? + +// RecordBatch is a Pilosa ingest interface designed to allow for +// maximum throughput on common workloads. Users should call Add() +// with a Row object until it returns ErrBatchNowFull, at which time +// they should call Import(), and then repeat. +// +// Add will not modify or otherwise retain the Row once it returns, so +// it is recommended that callers reuse the same Row with repeated +// calls to Add, just modifying its values appropriately in between +// calls. This avoids allocating a new slice of Values for each +// inserted Row. +// +// The supported types of the values in Row.Values are implementation +// defined. Similarly, the supported types for Row.ID are +// implementation defined. +type RecordBatch interface { + Add(Row) error + + // Import does translation, creates the fragment files, and then, + // if we're not using split batch mode, imports everything to + // Pilosa. It then resets internal data structures for the next + // batch. If we are using split batch mode, it saves the fragment + // data to the batch, resets all other internal structures, and + // continues. + // Split batch mode DOES NOT CURRENTLY SUPPORT MUTEX OR INT FIELDS! + Import() error + + // Len reports the number of records which have been added to the + // batch since the last call to Import (or since it was created). + Len() int + + // Flush is only applicable in split batch mode where it actually + // imports the stored data to Pilosa. Otherwise it simply returns + // nil. + Flush() error +} + +// agedTranslation combines a translation with a recording of when it was last used. +type agedTranslation struct { + id uint64 + lastUsed uint64 +} + +// Batch implements RecordBatch. +// +// It supports Values of type string, uint64, int64, or nil. The +// following table describes what Pilosa field each type of value must +// map to. Fields are set up when calling "NewBatch". +// +// | type | pilosa field type | options | +// |--------+-------------------+-----------| +// | string | set | keys=true | +// | uint64 | set | any | +// | int64 | int | any | +// | float64| decimal | scale | +// | nil | any | | +// +// nil values are ignored. +type Batch struct { + client *Client + index *Index + header []*Field + headerMap map[string]*Field + + // prevDuration records the time that each doImport() takes. This + // is used to set the timeout for transactions to a reasonable + // value based on the last import. It starts with a conservative + // default set in NewBatch. + prevDuration time.Duration + + // ids is a slice of length batchSize of record IDs + ids []uint64 + + // rowIDs is a map of field index (in the header) to slices of + // length batchSize which contain row IDs. + rowIDs map[int][]uint64 + // clearRowIDs is a map[fieldIndex][idsIndex]rowID we don't expect + // clears to happen very often, so we store the idIndex/value + // mapping in a map rather than a slice as we do for rowIDs. This + // is a potentially temporary workaround to allow packed boolean + // fields to clear "false" values. Packed fields may be more + // completely supported by Pilosa in future. + clearRowIDs map[int]map[int]uint64 + + // rowIDSets is a map from field name to a batchSize slice of + // slices of row IDs. When a given record can have more than one + // value for a field, rowIDSets stores that information. + rowIDSets map[string][][]uint64 + + // values holds the values for each record of an int field + values map[string][]int64 + + // times holds a time for each record. (if any of the fields are time fields) + times []QuantizedTime + + // nullIndices holds a slice of indices into b.ids for each + // integer field which has nil values. + nullIndices map[string][]uint64 + + // TODO support bool fields. + + // for each field, keep a map of key to which record indexes that key mapped to + toTranslate map[int]map[string][]int + toTranslateClear map[int]map[string][]int + + // toTranslateSets is a map from field name to a map of string + // keys that need to be translated to sets of record indexes which + // those keys map to. + toTranslateSets map[string]map[string][]int + + // toTranslateID maps each string key to a record index - this + // will get translated into Batch.rowIDs + toTranslateID []string + + colTranslations map[string]agedTranslation + rowTranslations map[string]map[string]agedTranslation + cycle uint64 + maxAge uint64 + + // staleTime tracks the time the first record of the batch was inserted + // plus the maxStaleness, in order to raise ErrBatchNowStale if the + // maxStaleness has elapsed + staleTime time.Time + maxStaleness time.Duration + + // Maximum number of keys to translate at one time. + keyTranslateBatchSize int + + log logger.Logger + + // experimental — only used by FlushToFragments which is an + // alternative to Import which just builds the bitmap data for a + // batch without actually importing it. + splitBatchMode bool + frags fragments + clearFrags fragments +} + +func (b *Batch) Len() int { return len(b.ids) } + +// BatchOption is a functional option for Batch objects. +type BatchOption func(b *Batch) error + +func OptLogger(l logger.Logger) BatchOption { + return func(b *Batch) error { + b.log = l + return nil + } +} + +func OptSplitBatchMode(on bool) BatchOption { + return func(b *Batch) error { + b.splitBatchMode = on + return nil + } +} + +func OptCacheMaxAge(age uint64) BatchOption { + return func(b *Batch) error { + b.maxAge = age + return nil + } +} + +func OptMaxStaleness(t time.Duration) BatchOption { + return func(b *Batch) error { + b.maxStaleness = t + return nil + } +} + +func OptKeyTranslateBatchSize(v int) BatchOption { + return func(b *Batch) error { + b.keyTranslateBatchSize = v + return nil + } +} + +// NewBatch initializes a new Batch object which will use the given +// Pilosa client, index, set of fields, and will take "size" records +// before returning ErrBatchNowFull. The positions of the Fields in +// 'fields' correspond to the positions of values in the Row's Values +// passed to Batch.Add(). +func NewBatch(client *Client, size int, index *Index, fields []*Field, opts ...BatchOption) (*Batch, error) { + if len(fields) == 0 || size == 0 { + return nil, errors.New("can't batch with no fields or batch size") + } + headerMap := make(map[string]*Field, len(fields)) + rowIDs := make(map[int][]uint64, len(fields)) + values := make(map[string][]int64) + tt := make(map[int]map[string][]int, len(fields)) + ttSets := make(map[string]map[string][]int) + hasTime := false + for i, field := range fields { + headerMap[field.Name()] = field + opts := field.Opts() + switch typ := opts.Type(); typ { + case FieldTypeDefault, FieldTypeSet, FieldTypeTime: + if opts.Keys() { + tt[i] = make(map[string][]int) + ttSets[field.Name()] = make(map[string][]int) + } + hasTime = typ == FieldTypeTime || hasTime + case FieldTypeInt, FieldTypeDecimal: + // tt line only needed if int field is string foreign key + tt[i] = make(map[string][]int) + values[field.Name()] = make([]int64, 0, size) + case FieldTypeMutex: + // similar to set/time fields, but no need to support sets + // of values (hence no ttSets) + if opts.Keys() { + tt[i] = make(map[string][]int) + } + rowIDs[i] = make([]uint64, 0, size) + default: + return nil, errors.Errorf("field type '%s' is not currently supported through Batch", typ) + } + } + + b := &Batch{ + client: client, + header: fields, + headerMap: headerMap, + prevDuration: time.Minute * 11, + index: index, + ids: make([]uint64, 0, size), + rowIDs: rowIDs, + clearRowIDs: make(map[int]map[int]uint64), + rowIDSets: make(map[string][][]uint64), + values: values, + nullIndices: make(map[string][]uint64), + toTranslate: tt, + toTranslateClear: make(map[int]map[string][]int), + toTranslateSets: ttSets, + colTranslations: make(map[string]agedTranslation), + rowTranslations: make(map[string]map[string]agedTranslation), + maxAge: 64, + maxStaleness: time.Duration(0), + keyTranslateBatchSize: DefaultKeyTranslateBatchSize, + + log: logger.NopLogger, + + frags: make(fragments), + clearFrags: make(fragments), + } + if hasTime { + b.times = make([]QuantizedTime, 0, size) + } + for _, opt := range opts { + err := opt(b) + if err != nil { + return nil, errors.Wrap(err, "applying options") + } + } + return b, nil +} + +// Row represents a single record which can be added to a Batch. +type Row struct { + ID interface{} + // Values map to the slice of fields in Batch.header + Values []interface{} + // Clears' int key is an index into Batch.header + Clears map[int]interface{} + // Time applies to all time fields + Time QuantizedTime +} + +// QuantizedTime represents a moment in time down to some granularity +// (year, month, day, or hour). +type QuantizedTime struct { + ymdh [10]byte +} + +// Set sets the Quantized time to the given timestamp (down to hour +// granularity). +func (qt *QuantizedTime) Set(t time.Time) { + copy(qt.ymdh[:], t.Format("2006010215")) +} + +// SetYear sets the quantized time's year, but leaves month, day, and +// hour untouched. +func (qt *QuantizedTime) SetYear(year string) { + copy(qt.ymdh[:4], year) +} + +// SetMonth sets the QuantizedTime's month, but leaves year, day, and +// hour untouched. +func (qt *QuantizedTime) SetMonth(month string) { + copy(qt.ymdh[4:6], month) +} + +// SetDay sets the QuantizedTime's day, but leaves year, month, and +// hour untouched. +func (qt *QuantizedTime) SetDay(day string) { + copy(qt.ymdh[6:8], day) +} + +// SetHour sets the QuantizedTime's hour, but leaves year, month, and +// day untouched. +func (qt *QuantizedTime) SetHour(hour string) { + copy(qt.ymdh[8:10], hour) +} + +// Reset sets the time to the zero value which generates no time views. +func (qt *QuantizedTime) Reset() { + for i := range qt.ymdh { + qt.ymdh[i] = 0 + } +} + +// views builds the list of Pilosa views for this particular time, +// given a quantum. +func (qt *QuantizedTime) views(q TimeQuantum) ([]string, error) { + zero := QuantizedTime{} + if *qt == zero { + return nil, nil + } + views := make([]string, 0, len(q)) + for _, unit := range q { + switch unit { + case 'Y': + if qt.ymdh[0] == 0 { + return nil, errors.New("no data set for year") + } + views = append(views, string(qt.ymdh[:4])) + case 'M': + if qt.ymdh[4] == 0 { + return nil, errors.New("no data set for month") + } + views = append(views, string(qt.ymdh[:6])) + case 'D': + if qt.ymdh[6] == 0 { + return nil, errors.New("no data set for day") + } + views = append(views, string(qt.ymdh[:8])) + case 'H': + if qt.ymdh[8] == 0 { + return nil, errors.New("no data set for hour") + } + views = append(views, string(qt.ymdh[:10])) + } + } + return views, nil +} + +func (b *Batch) getColTranslation(key string) (uint64, bool) { + trans, ok := b.colTranslations[key] + if ok { + trans.lastUsed = b.cycle + b.colTranslations[key] = trans + } + return trans.id, ok +} + +func (b *Batch) getRowTranslation(field, key string) (uint64, bool) { + trans, ok := b.rowTranslations[field][key] + if ok { + trans.lastUsed = b.cycle + b.rowTranslations[field][key] = trans + } + return trans.id, ok +} + +func (b *Batch) addRowTranslations(fieldName string, keys []string, ids []uint64) { + rowCache := b.rowTranslations[fieldName] + if rowCache == nil { + rowCache = make(map[string]agedTranslation) + b.rowTranslations[fieldName] = rowCache + } + for i, k := range keys { + rowCache[k] = agedTranslation{ + id: ids[i], + lastUsed: b.cycle, + } + } +} + +func (b *Batch) addColTranslations(keys []string, ids []uint64) { + for i, k := range keys { + b.colTranslations[k] = agedTranslation{ + id: ids[i], + lastUsed: b.cycle, + } + } +} + +// Add adds a record to the batch. Performance will be best if record +// IDs are shard-sorted. That is, all records which belong to the same +// Pilosa shard are added adjacent to each other. If the records are +// also in-order within a shard this will likely help as well. Add +// clears rec.Clears when it returns normally (either a nil error or +// BatchNowFull). +func (b *Batch) Add(rec Row) error { + // Clear recValues and rec.Clears upon return. + defer func() { + for i := range rec.Values { + rec.Values[i] = nil + } + for k := range rec.Clears { + delete(rec.Clears, k) + } + }() + + if len(b.ids) == cap(b.ids) { + return ErrBatchAlreadyFull + } + if len(rec.Values) != len(b.header) { + return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.header), len(rec.Values)) + } + + handleStringID := func(rid string) error { + if rid == "" { + return errors.Errorf("record identifier cannot be an empty string") + } + if colID, ok := b.getColTranslation(rid); ok { + b.ids = append(b.ids, colID) + } else { + if b.toTranslateID == nil { + b.toTranslateID = make([]string, cap(b.ids)) + } + b.toTranslateID[len(b.ids)] = rid + b.ids = append(b.ids, 0) + } + return nil + } + var err error + + switch rid := rec.ID.(type) { + case uint64: + b.ids = append(b.ids, rid) + case string: + err := handleStringID(rid) + if err != nil { + return err + } + case []byte: + err = handleStringID(string(rid)) + if err != nil { + return err + } + default: // TODO support nil ID as being auto-allocated. + return errors.Errorf("unsupported id type %T value %v", rid, rid) + } + + // curPos is the current position in b.ids, rowIDs[*], etc. + curPos := len(b.ids) - 1 + + if b.times != nil { + b.times = append(b.times, rec.Time) + } + + for i := 0; i < len(rec.Values); i++ { + field := b.header[i] + switch val := rec.Values[i].(type) { + case string: + if field.Opts().Type() != FieldTypeInt { + // nil-extend + for len(b.rowIDs[i]) < curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + rowIDs := b.rowIDs[i] + // empty string is not a valid value at this point (Pilosa refuses to translate it) + if val == "" { // + b.rowIDs[i] = append(rowIDs, nilSentinel) + + } else if rowID, ok := b.getRowTranslation(field.Name(), val); ok { + b.rowIDs[i] = append(rowIDs, rowID) + } else { + ints, ok := b.toTranslate[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslate[i][val] = ints + b.rowIDs[i] = append(rowIDs, 0) + } + } else if field.Opts().Type() == FieldTypeInt { + if val == "" { + // copied from the `case nil:` section for ints and decimals + b.values[field.Name()] = append(b.values[field.Name()], 0) + nullIndices, ok := b.nullIndices[field.Name()] + if !ok { + nullIndices = make([]uint64, 0) + } + nullIndices = append(nullIndices, uint64(curPos)) + b.nullIndices[field.Name()] = nullIndices + } else if intVal, ok := b.getRowTranslation(field.Name(), val); ok { + b.values[field.Name()] = append(b.values[field.Name()], int64(intVal)) + } else { + ints, ok := b.toTranslate[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslate[i][val] = ints + b.values[field.Name()] = append(b.values[field.Name()], 0) + } + } + case uint64: + // nil-extend + for len(b.rowIDs[i]) < curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + b.rowIDs[i] = append(b.rowIDs[i], val) + case int64: + b.values[field.Name()] = append(b.values[field.Name()], val) + case []string: + if len(val) == 0 { + continue + } + rowIDSets, ok := b.rowIDSets[field.Name()] + if !ok { + rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids)) + b.rowIDSets[field.Name()] = rowIDSets + } + for len(rowIDSets) < len(b.ids)-1 { + rowIDSets = append(rowIDSets, nil) // nil extend + } + + rowIDs := make([]uint64, 0, len(val)) + for _, k := range val { + if k == "" { + continue + } + if rowID, ok := b.getRowTranslation(field.Name(), k); ok { + rowIDs = append(rowIDs, rowID) + } else { + ttsets, ok := b.toTranslateSets[field.Name()] + if !ok { + ttsets = make(map[string][]int) + b.toTranslateSets[field.Name()] = make(map[string][]int) + } + ints, ok := ttsets[k] + if !ok { + ints = make([]int, 0, 1) + } + ints = append(ints, curPos) + b.toTranslateSets[field.Name()][k] = ints + } + } + b.rowIDSets[field.Name()] = append(rowIDSets, rowIDs) + case []uint64: + if len(val) == 0 { + continue + } + rowIDSets, ok := b.rowIDSets[field.Name()] + if !ok { + rowIDSets = make([][]uint64, len(b.ids)-1, cap(b.ids)) + } + for len(rowIDSets) < len(b.ids)-1 { + rowIDSets = append(rowIDSets, nil) // nil extend + } + b.rowIDSets[field.Name()] = append(rowIDSets, val) + case nil: + if field.Opts().Type() == FieldTypeInt || field.Opts().Type() == FieldTypeDecimal { + b.values[field.Name()] = append(b.values[field.Name()], 0) + nullIndices, ok := b.nullIndices[field.Name()] + if !ok { + nullIndices = make([]uint64, 0) + } + nullIndices = append(nullIndices, uint64(curPos)) + b.nullIndices[field.Name()] = nullIndices + + } else { + // only append nil to rowIDs if this field already has + // rowIDs. Otherwise, this could be a []string or + // []uint64 field where we've only seen nil values so + // far. when we see a uint64 or string value, we'll + // "nil-extend" rowIDs to make sure it's the right + // length. + if rowIDs, ok := b.rowIDs[i]; ok { + b.rowIDs[i] = append(rowIDs, nilSentinel) + } + } + default: + return errors.Errorf("Val %v Type %[1]T is not currently supported. Use string, uint64 (row id), or int64 (integer value)", val) + } + } + + for i, uval := range rec.Clears { + field := b.header[i] + if _, ok := b.clearRowIDs[i]; !ok { + b.clearRowIDs[i] = make(map[int]uint64) + } + switch val := uval.(type) { + case string: + clearRows := b.clearRowIDs[i] + // translate val and add to clearRows + if rowID, ok := b.getRowTranslation(field.Name(), val); ok { + clearRows[curPos] = rowID + } else { + _, ok := b.toTranslateClear[i] + if !ok { + b.toTranslateClear[i] = make(map[string][]int) + } + ints, ok := b.toTranslateClear[i][val] + if !ok { + ints = make([]int, 0) + } + ints = append(ints, curPos) + b.toTranslateClear[i][val] = ints + } + case uint64: + b.clearRowIDs[i][curPos] = val + default: + return errors.Errorf("Clearing a value '%v' Type %[1]T is not currently supported (field '%s')", val, field.Name()) + } + // nil extend b.rowIDs so we don't run into a horrible bug + // where we skip doing clears because b.rowIDs doesn't have a + // value for this field + for len(b.rowIDs[i]) <= curPos { + b.rowIDs[i] = append(b.rowIDs[i], nilSentinel) + } + + } + + if len(b.ids) == cap(b.ids) { + return ErrBatchNowFull + } + if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking + if len(b.ids) == 1 { + b.staleTime = time.Now().Add(b.maxStaleness) + } else if time.Now().After(b.staleTime) { + return ErrBatchNowStale + } + } + return nil +} + +// ErrBatchNowFull, similar to io.EOF, is a marker error to notify the +// user of a batch that it is time to call Import. +var ErrBatchNowFull = errors.New("batch is now full - you cannot add any more records (though the one you just added was accepted)") + +// ErrBatchAlreadyFull is a real error saying that Batch.Add did not +// complete because the batch was full. +var ErrBatchAlreadyFull = errors.New("batch was already full, record was rejected") + +// ErrBatchNowStale indicates that the oldest record in the batch is older than +// the maxStaleness value of the batch. Like ErrBatchNowFull, the error does +// not mean the record was rejected. +var ErrBatchNowStale = errors.New("batch is stale and needs to be imported (however, record was accepted)") + +// Import does translation, creates the fragment files, and then, +// if we're not using split batch mode, imports everything to +// Pilosa. It then resets internal data structures for the next +// batch. If we are using split batch mode, it saves the fragment +// data to the batch, resets all other internal structures, and +// continues. split batch mode DOES NOT CURRENTLY SUPPORT MUTEX +// OR INT FIELDS! +func (b *Batch) Import() error { + start := time.Now() + trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour) + if err != nil { + return errors.Wrap(err, "starting transaction") + } + defer func() { + trns, err := b.client.FinishTransaction(trns.ID) + if err != nil { + b.log.Printf("error finishing transaction: %v. trns: %+v", err, trns) + } + b.client.Stats.Timing(MetricBatchImportDurationSeconds, time.Since(start), 1.0) + }() + + size := len(b.ids) + transStart := time.Now() + // first we need to translate the toTranslate, then fill out the missing row IDs + err = b.doTranslation() + if err != nil { + return errors.Wrap(err, "doing Translation") + } + transTime := time.Now() + b.log.Printf("translating batch of %d took: %v", size, transTime.Sub(transStart)) + + frags, clearFrags, err := b.makeFragments(b.frags, b.clearFrags) + if err != nil { + return errors.Wrap(err, "making fragments (flush)") + } + makeTime := time.Now() + b.log.Printf("making fragments for batch of %d took %v", size, makeTime.Sub(transTime)) + + if b.splitBatchMode { + b.frags = frags + b.clearFrags = clearFrags + } else { + b.frags = make(fragments) + b.clearFrags = make(fragments) + // create bitmaps out of each field in b.rowIDs and import. Also + // import int data. + err = b.doImport(frags, clearFrags) + if err != nil { + return errors.Wrap(err, "doing import") + } + b.log.Printf("importing fragments took %v", time.Since(makeTime)) + } + + b.reset() + return nil +} + +// Flush is only applicable in split batch mode where it actually +// imports the stored data to Pilosa. Otherwise it simply returns +// nil. +func (b *Batch) Flush() error { + if !b.splitBatchMode { + return nil + } + start := time.Now() + + trns, err := b.client.StartTransaction("", b.prevDuration*10, false, time.Hour) + if err != nil { + return errors.Wrap(err, "starting transaction") + } + defer func() { + trns, err := b.client.FinishTransaction(trns.ID) + if err != nil { + b.log.Printf("error finishing transaction: %v. trns: %+v", err, trns) + } + b.client.Stats.Timing(MetricBatchFlushDurationSeconds, time.Since(start), 1.0) + }() + + importStart := time.Now() + err = b.doImport(b.frags, b.clearFrags) + if err != nil { + return errors.Wrap(err, "doing import (ImportFragments)") + } + + b.log.Debugf("superbatch import took %v", time.Since(importStart)) + + b.reset() + b.frags = make(fragments) + b.clearFrags = make(fragments) + return nil +} + +func (b *Batch) doTranslation() error { + keys := make([]string, 0) + + // translate column keys if there are any + + // TODO test. Also this implementation (using a set to de-dup + // keys) will likely have much worse performance than the previous + // one (two slices, one of keys one of ids) in the case that most + // of the keys are unique. + keySet := make(map[string]uint64) + for _, key := range b.toTranslateID { + if key != "" { + if _, ok := keySet[key]; ok { + continue + } + keys = append(keys, key) + keySet[key] = 0 + } + } + + if len(keys) > 0 { + start := time.Now() + ids, err := b.translateColumnKeys(b.index, keys) + if err != nil { + return errors.Wrap(err, "translating col keys") + } + if len(ids) != len(keys) { + return errors.Errorf("requested IDs for %d column keys but got %d back", len(keys), len(ids)) + } + b.log.Debugf("translating %d column keys took %v", len(keys), time.Since(start)) + b.addColTranslations(keys, ids) + for j, id := range ids { + keySet[keys[j]] = id + } + for index, ttkey := range b.toTranslateID { + if ttkey != "" { + b.ids[index] = keySet[ttkey] + } + } + } + // translate row keys + for i, tt := range b.toTranslate { + fieldName := b.header[i].Name() + keys = keys[:0] + + // make a slice of keys + for k := range tt { + keys = append(keys, k) + } + // append keys to clear so we can translate them all in one + // request. ttEnd is the index where clearing starts which we + // use later on. + ttEnd := len(keys) + ttc := b.toTranslateClear[i] + for k := range ttc { + keys = append(keys, k) + } + + if len(keys) == 0 { + continue + } + + // translate keys from Pilosa + start := time.Now() + ids, err := b.translateRowKeys(b.headerMap[fieldName], keys) + if err != nil { + return errors.Wrap(err, "translating row keys") + } + if len(ids) != len(keys) { + return errors.Errorf("requested IDs for %d row keys but got %d back", len(keys), len(ids)) + } + b.log.Debugf("translating %d row keys for %s took %v", len(keys), fieldName, time.Since(start)) + b.addRowTranslations(fieldName, keys, ids) + + switch b.header[i].Opts().Type() { + case FieldTypeInt: + // handle foreign key int fields — fill out b.values instead of b.rows + for j := 0; j < ttEnd; j++ { + key := keys[j] + id := ids[j] + for _, recordIdx := range tt[key] { + b.values[fieldName][recordIdx] = int64(id) + } + } + case FieldTypeDecimal: + return errors.Errorf("unexpected field type for translation: decimal") + default: + // fill out missing IDs in local batch records with translated IDs + rows := b.rowIDs[i] + for j := 0; j < ttEnd; j++ { + key := keys[j] + id := ids[j] + for _, recordIdx := range tt[key] { + rows[recordIdx] = id + } + } + // fill out missing IDs in clear lists. + clearRows := b.clearRowIDs[i] + for j := ttEnd; j < len(keys); j++ { + key := keys[j] + id := ids[j] + for _, recordIdx := range ttc[key] { + clearRows[recordIdx] = id + } + } + } + } + + for fieldName, tt := range b.toTranslateSets { + keys = keys[:0] + + for k := range tt { + keys = append(keys, k) + } + + if len(keys) == 0 { + continue + } + // translate keys from Pilosa + start := time.Now() + ids, err := b.translateRowKeys(b.headerMap[fieldName], keys) + if err != nil { + return errors.Wrap(err, "translating row keys (sets)") + } + if len(ids) != len(keys) { + return errors.Errorf("requested IDs for %d row (set) keys but got %d back", len(keys), len(ids)) + } + b.log.Debugf("translating %d row keys(sets) for %s took %v", len(keys), fieldName, time.Since(start)) + b.addRowTranslations(fieldName, keys, ids) + rowIDSets := b.rowIDSets[fieldName] + rowIDSets = rowIDSets[:cap(b.ids)] + b.rowIDSets[fieldName] = rowIDSets + for j, key := range keys { + rowID := ids[j] + for _, recordIdx := range tt[key] { + rowIDSets[recordIdx] = append(rowIDSets[recordIdx], rowID) + } + } + } + + return nil +} + +func (b *Batch) translateColumnKeys(index *Index, keys []string) ([]uint64, error) { + batchSize := b.keyTranslateBatchSize + if batchSize <= 0 { + batchSize = len(keys) + } + + ids := make([]uint64, 0, len(keys)) + for i := 0; i < len(keys); i += batchSize { + keySlice := keys[i:] + if len(keySlice) > batchSize { + keySlice = keySlice[:batchSize] + } + + idSlice, err := b.client.TranslateColumnKeys(b.index, keySlice) + if err != nil { + return nil, err + } else if len(idSlice) != len(keySlice) { + return nil, errors.Errorf("requested IDs slice for %d column keys but got %d back", len(keySlice), len(idSlice)) + } + ids = append(ids, idSlice...) + } + + return ids, nil +} + +func (b *Batch) translateRowKeys(field *Field, keys []string) ([]uint64, error) { + batchSize := b.keyTranslateBatchSize + if batchSize <= 0 { + batchSize = len(keys) + } + + ids := make([]uint64, 0, len(keys)) + for i := 0; i < len(keys); i += batchSize { + keySlice := keys[i:] + if len(keySlice) > batchSize { + keySlice = keySlice[:batchSize] + } + + idSlice, err := b.client.TranslateRowKeys(field, keySlice) + if err != nil { + return nil, err + } else if len(idSlice) != len(keySlice) { + return nil, errors.Errorf("requested IDs slice for %d row keys but got %d back", len(keySlice), len(idSlice)) + } + ids = append(ids, idSlice...) + } + + return ids, nil +} + +func (b *Batch) doImport(frags, clearFrags fragments) error { + + start := time.Now() + eg := egpool.Group{PoolSize: 20} + // TODO, currently this relies on upstream behavior of + // makeFragments to guarantee that any shard/field combination in + // clearFrags also has a shard/field in frags. We're only + // iterating over frags and then checking to see if clearFrags has + // the same keys. If we optimized makeFragments to skip adding + // things to frags which had no set bits (e.g. if we were only + // clearing things), then this code would need to be updated to + // ensure that it looked at the things in clearFrags which were + // *not* in frags. + for fragmentKey, viewMap := range frags { + field := fragmentKey.field + shard := fragmentKey.shard + viewMap := viewMap + + eg.Go(func() error { + clearViewMap := clearFrags.GetViewMap(shard, field) + if len(clearViewMap) > 0 { + start := time.Now() + err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, clearViewMap, true) + if err != nil { + return errors.Wrapf(err, "import clearing clearing data for %s", field) + } + b.log.Debugf("imp-roar-clr %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start)) + } + + start := time.Now() + err := b.client.ImportRoaringBitmap(b.index.Field(field), shard, viewMap, false) + b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(start)) + return errors.Wrapf(err, "importing data for %s", field) + }) + } + eg.Go(func() error { return b.importValueData() }) + eg.Go(func() error { return b.importMutexData() }) + + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + b.prevDuration = time.Since(start) + return nil +} + +func anyCause(cause error, errs ...error) error { + if cause == nil { + return nil + } + + for _, err := range errs { + if errors.Cause(err) == cause { + return err + } + } + return nil +} + +// this is kind of bad as it means we can never import column id +// ^uint64(0) which is a valid column ID. I think it's unlikely to +// matter much in practice (we could maybe special case it somewhere +// if needed though). +var nilSentinel = ^uint64(0) + +func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments, error) { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + emptyClearRows := make(map[int]uint64) + + // create _exists fragments if needed + if b.index.Opts().TrackExistence() { + var curBM *roaring.Bitmap + curShard := ^uint64(0) // impossible sentinel value for shard. + for _, col := range b.ids { + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, "_exists", "") + } + curBM.DirectAdd(col % shardWidth) + } + } + + for i, rowIDs := range b.rowIDs { + if len(rowIDs) == 0 { + continue // this can happen when the values that came in for this field were string slices + } + clearRows := b.clearRowIDs[i] + if clearRows == nil { + clearRows = emptyClearRows + } + field := b.header[i] + opts := field.Opts() + if opts.Type() == FieldTypeMutex { + continue // we handle mutex fields separately — they can't use importRoaring + } + curShard := ^uint64(0) // impossible sentinel value for shard. + var curBM *roaring.Bitmap + var clearBM *roaring.Bitmap + for j := range b.ids { + col, row := b.ids[j], rowIDs[j] + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, field.Name(), "") + clearBM = clearFrags.GetOrCreate(curShard, field.Name(), "") + } + if row != nilSentinel { + // TODO this is super ugly, but we want to avoid setting + // bits on the standard view in the specific case when + // there isn't one. Should probably refactor this whole + // loop to be more general w.r.t. views. Also... tests for + // the NoStandardView case would be great. + if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) { + curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + } + if opts.Type() == FieldTypeTime { + views, err := b.times[j].views(opts.TimeQuantum()) + if err != nil { + return nil, nil, errors.Wrap(err, "calculating views") + } + for _, view := range views { + tbm := frags.GetOrCreate(curShard, field.Name(), view) + tbm.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + } + + clearRow, ok := clearRows[j] + if ok { + clearBM.DirectAddN(clearRow*shardWidth + (col % shardWidth)) + // we're going to execute the clear before the set, so + // we want to make sure that at this point, the "set" + // fragments don't contain the bit that we're clearing + curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth)) + } + } + } + + for fname, rowIDSets := range b.rowIDSets { + if len(rowIDSets) == 0 { + continue + } else if len(rowIDSets) < len(b.ids) { + // rowIDSets is guaranteed to have capacity == to b.ids, + // but if the last record had a nil for this field, it + // might not have the same length, so we re-slice it to + // ensure the lengths are the same. + rowIDSets = rowIDSets[:len(b.ids)] + } + field := b.headerMap[fname] + opts := field.Opts() + curShard := ^uint64(0) // impossible sentinel value for shard. + var curBM *roaring.Bitmap + for j := range b.ids { + col, rowIDs := b.ids[j], rowIDSets[j] + if len(rowIDs) == 0 { + continue + } + if col/shardWidth != curShard { + curShard = col / shardWidth + curBM = frags.GetOrCreate(curShard, fname, "") + } + // TODO this is super ugly, but we want to avoid setting + // bits on the standard view in the specific case when + // there isn't one. Should probably refactor this whole + // loop to be more general w.r.t. views. Also... tests for + // the NoStandardView case would be great. + if !(opts.Type() == FieldTypeTime && opts.NoStandardView()) { + for _, row := range rowIDs { + curBM.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + if opts.Type() == FieldTypeTime { + views, err := b.times[j].views(opts.TimeQuantum()) + if err != nil { + return nil, nil, errors.Wrap(err, "calculating views") + } + for _, view := range views { + tbm := frags.GetOrCreate(curShard, fname, view) + for _, row := range rowIDs { + tbm.DirectAdd(row*shardWidth + (col % shardWidth)) + } + } + } + } + } + return frags, clearFrags, nil +} + +// importValueData imports data for int fields. +func (b *Batch) importValueData() error { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + eg := egpool.Group{PoolSize: 20} + + ids := make([]uint64, len(b.ids)) + for fieldName, bvalues := range b.values { + ids = ids[:len(b.ids)] + + // trim out null values from ids and values. + nullIndices := b.nullIndices[fieldName] + + i, n := uint64(0), 0 + for _, nullIndex := range nullIndices { + copy(ids[n:], b.ids[i:nullIndex]) + n += copy(bvalues[n:], bvalues[i:nullIndex]) + i = nullIndex + 1 + } + + copy(ids[n:], b.ids[i:]) + n += copy(bvalues[n:], bvalues[i:]) + ids, bvalues = ids[:n], bvalues[:n] + + // now do imports by shard + if len(ids) == 0 { + continue // TODO test this "all nil" case + } + curShard := ids[0] / shardWidth + startIdx := 0 + for i := 1; i <= len(ids); i++ { + var recordID uint64 + if i < len(ids) { + recordID = ids[i] + } else { + recordID = (curShard + 2) * shardWidth + } + + if recordID/shardWidth != curShard { + endIdx := i + shard := curShard + field := b.headerMap[fieldName] + path, data, err := b.client.EncodeImportValues(field, shard, bvalues[startIdx:endIdx], ids[startIdx:endIdx], false) + if err != nil { + return errors.Wrap(err, "encoding import values") + } + eg.Go(func() error { + start := time.Now() + err := b.client.DoImportValues(b.index.Name(), shard, path, data) + b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start)) + return errors.Wrapf(err, "importing values for %s", field) + }) + startIdx = i + curShard = recordID / shardWidth + } + } + } + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + return errors.Wrap(err, "importing value data") +} + +// TODO this should work for bools as well - just need to support them +// at batch creation time and when calling Add, I think. +func (b *Batch) importMutexData() error { + shardWidth := b.index.ShardWidth() + if shardWidth == 0 { + shardWidth = DefaultShardWidth + } + + eg := egpool.Group{PoolSize: 20} + ids := make([]uint64, 0, len(b.ids)) + for findex, rowIDs := range b.rowIDs { + field := b.header[findex] + if field.Opts().Type() != FieldTypeMutex { + continue + } + ids = ids[:0] + + // get slice of column ids for non-nil rowIDs and cut nil row + // IDs out of rowIDs. + idsIndex := 0 + for i, id := range b.ids { + rowID := rowIDs[i] + if rowID == nilSentinel { + continue + } + rowIDs[idsIndex] = rowID + ids = append(ids, id) + idsIndex++ + } + rowIDs = rowIDs[:idsIndex] + + if len(ids) == 0 { + continue + } + curShard := ids[0] / shardWidth + startIdx := 0 + for i := 1; i <= len(ids); i++ { + var recordID uint64 + if i < len(ids) { + recordID = ids[i] + } else { + recordID = (curShard + 2) * shardWidth + } + + if recordID/shardWidth != curShard { + endIdx := i + shard := curShard + field := field + path, data, err := b.client.EncodeImport(field, shard, rowIDs[startIdx:endIdx], ids[startIdx:endIdx], false) + if err != nil { + return errors.Wrap(err, "encoding mutex import") + } + eg.Go(func() error { + start := time.Now() + err := b.client.DoImport(b.index.Name(), shard, path, data) + b.log.Debugf("imp-mux %s,shard:%d,data:%d %v", field.Name(), shard, len(data), time.Since(start)) + return errors.Wrapf(err, "importing values for %s", field) + }) + startIdx = i + curShard = recordID / shardWidth + } + } + } + err := eg.Wait() + if err != nil { + if pferr := anyCause(ErrPreconditionFailed, eg.Errors()...); pferr != nil { + return pferr + } + return err + } + return errors.Wrap(err, "importing mutex data") +} + +// reset is called at the end of importing to ready the batch for the +// next round. Where possible it does not re-allocate memory. +func (b *Batch) reset() { + b.ids = b.ids[:0] + b.times = b.times[:0] + for i, rowIDs := range b.rowIDs { + b.rowIDs[i] = rowIDs[:0] + } + for _, tt := range b.toTranslate { + for k := range tt { + delete(tt, k) // TODO pool these slices + } + } + for _, tts := range b.toTranslateSets { + for k := range tts { + delete(tts, k) + } + } + for field, rowIDSet := range b.rowIDSets { + for i := range rowIDSet { + rowIDSet[i] = nil + } + b.rowIDSets[field] = rowIDSet[:0] + } + for _, rowIDs := range b.clearRowIDs { + for k := range rowIDs { + delete(rowIDs, k) + } + } + for _, clearMap := range b.toTranslateClear { + for k := range clearMap { + delete(clearMap, k) + } + } + for i := range b.toTranslateID { + b.toTranslateID[i] = "" + } + for k := range b.values { + delete(b.values, k) // TODO pool these slices + } + for k := range b.nullIndices { + delete(b.nullIndices, k) // TODO pool these slices + } + b.cycle++ + for k, trans := range b.colTranslations { + if trans.lastUsed-b.cycle > b.maxAge { + delete(b.colTranslations, k) + } + } + for field, rowTranslations := range b.rowTranslations { + for k, trans := range rowTranslations { + if trans.lastUsed-b.cycle > b.maxAge { + delete(rowTranslations, k) + } + } + + if len(rowTranslations) == 0 { + delete(b.rowTranslations, field) + } + } +} + +// map[shard][field][view]fragmentData +type fragments map[fragmentKey]map[string]*roaring.Bitmap + +type fragmentKey struct { + shard uint64 + field string +} + +func (f fragments) GetOrCreate(shard uint64, field, view string) *roaring.Bitmap { + key := fragmentKey{shard, field} + viewMap, ok := f[key] + if !ok { + viewMap = make(map[string]*roaring.Bitmap) + f[key] = viewMap + } + bm, ok := viewMap[view] + if !ok { + bm = roaring.NewBTreeBitmap() + viewMap[view] = bm + } + return bm +} + +func (f fragments) GetViewMap(shard uint64, field string) map[string]*roaring.Bitmap { + key := fragmentKey{shard, field} + viewMap, ok := f[key] + if !ok { + return nil + } + // Remove any views which have an empty bitmap. + // TODO: Ideally we would prevent allocating the empty bitmap to begin with, + // but the logic is a bit tricky, and since we don't want to spend too much + // time on it right now, we're leaving that for a future exercise. + for k, v := range viewMap { + if v.Count() == 0 { + delete(viewMap, k) + } + } + return viewMap +} diff --git a/client/batch_test.go b/client/batch_test.go new file mode 100644 index 000000000..69cb6d247 --- /dev/null +++ b/client/batch_test.go @@ -0,0 +1,1311 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//+build integration + +package client + +import ( + "reflect" + "sort" + "strconv" + "testing" + "time" + + "github.com/pkg/errors" +) + +func TestStringSliceCombos(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("test-string-slicecombos") + fields := make([]*Field, 1) + fields[0] = idx.Field("a1", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 5, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + + records := []Row{ + {ID: uint64(0), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(1), Values: []interface{}{[]string{"z"}}}, + {ID: uint64(2), Values: []interface{}{[]string{}}}, + {ID: uint64(3), Values: []interface{}{[]string{"q", "r", "s", "t", "c"}}}, + {ID: uint64(4), Values: []interface{}{nil}}, + {ID: uint64(5), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(6), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(7), Values: []interface{}{[]string{"z"}}}, + {ID: uint64(8), Values: []interface{}{[]string{}}}, + {ID: uint64(9), Values: []interface{}{[]string{"q", "r", "s", "t"}}}, + {ID: uint64(10), Values: []interface{}{nil}}, + {ID: uint64(11), Values: []interface{}{[]string{"a", "b", "c"}}}, + {ID: uint64(12), Values: []interface{}{[]string{}}}, + {ID: uint64(13), Values: []interface{}{[]string{}}}, + } + + err = ingestRecords(records, b) + if err != nil { + t.Fatalf("importing: %v", err) + } + + a1 := fields[0] + + result := tq(t, client, a1.TopN(10)) + rez := sortableCRI(result.CountItems()) + sort.Sort(rez) + exp := sortableCRI{ + {Key: "a", Count: 4}, + {Key: "b", Count: 4}, + {Key: "c", Count: 5}, + {Key: "q", Count: 2}, + {Key: "r", Count: 2}, + {Key: "s", Count: 2}, + {Key: "t", Count: 2}, + {Key: "z", Count: 2}, + } + sort.Sort(exp) + errorIfNotEqual(t, exp, rez) + + result = tq(t, client, a1.Row("a")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11}) + result = tq(t, client, a1.Row("b")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 5, 6, 11}) + result = tq(t, client, a1.Row("c")) + errorIfNotEqual(t, result.Row().Columns, []uint64{0, 3, 5, 6, 11}) + result = tq(t, client, a1.Row("z")) + errorIfNotEqual(t, result.Row().Columns, []uint64{1, 7}) + result = tq(t, client, a1.Row("q")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("r")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("s")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + result = tq(t, client, a1.Row("t")) + errorIfNotEqual(t, result.Row().Columns, []uint64{3, 9}) + + result = tq(t, client, idx.RawQuery("Count(All())")) + errorIfNotEqual(t, result.Count(), int64(14)) +} + +func errorIfNotEqual(t *testing.T, exp, got interface{}) { + t.Helper() + if !reflect.DeepEqual(exp, got) { + t.Errorf("unequal exp/got:\n%v\n%v", exp, got) + } +} + +type sortableCRI []CountResultItem + +func (s sortableCRI) Len() int { return len(s) } +func (s sortableCRI) Less(i, j int) bool { + if s[i].Count != s[j].Count { + return s[i].Count > s[j].Count + } + if s[i].ID != s[j].ID { + return s[i].ID < s[j].ID + } + if s[i].Key != s[j].Key { + return s[i].Key < s[j].Key + } + return true +} +func (s sortableCRI) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func tq(t *testing.T, client *Client, query PQLQuery) QueryResult { + resp, err := client.Query(query) + if err != nil { + t.Fatalf("querying: %v", err) + } + return resp.Results()[0] +} + +func ingestRecords(records []Row, batch *Batch) error { + for _, rec := range records { + err := batch.Add(rec) + if err == ErrBatchNowFull { + err = batch.Import() + if err != nil { + return errors.Wrap(err, "importing batch") + } + } else if err != nil { + return errors.Wrap(err, "while adding record") + } + } + if batch.Len() > 0 { + err := batch.Import() + if err != nil { + return errors.Wrap(err, "importing batch") + } + } + return nil +} + +func TestImportBatchInts(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-blah") + field := idx.Field("anint", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + + b, err := NewBatch(client, 3, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{Values: make([]interface{}, 1)} + + for i := uint64(0); i < 3; i++ { + r.ID = i + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + r.ID = uint64(0) + r.Values[0] = nil + err = b.Add(r) + if err != nil { + t.Fatalf("adding after import: %v", err) + } + r.ID = uint64(1) + r.Values[0] = int64(7) + err = b.Add(r) + if err != nil { + t.Fatalf("adding second after import: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("second import: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(7), field.Equals(2))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + for i, result := range resp.Results() { + if !reflect.DeepEqual(result.Row().Columns, []uint64{uint64(i)}) { + t.Errorf("expected %v for %d, but got %v", []uint64{uint64(i)}, i, result.Row().Columns) + } + } +} + +func TestTrimNull(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-null") + field := idx.Field("empty", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + b, err := NewBatch(client, 3, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + b.nullIndices = make(map[string][]uint64, 1) + b.nullIndices[field.Name()] = []uint64{0, 1, 2} + r := Row{Values: make([]interface{}, 1)} + for i := 0; i < 3; i++ { + r.ID = uint64(i) + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + resp, err := client.Query(idx.BatchQuery(field.Equals(0), field.Equals(1), field.Equals(2))) + if err != nil { + t.Fatalf("querying: %v", err) + } + for i, result := range resp.Results() { + if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) { + t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns) + } + } + + b, err = NewBatch(client, 4, idx, []*Field{field}) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + r = Row{Values: make([]interface{}, 1)} + for i := 10; i < 40; i += 10 { + r.ID = uint64(i) + r.Values[0] = int64(i) + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + } + + r.ID = uint64(40) + r.Values[0] = nil + err = b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err = client.Query(idx.BatchQuery(field.Equals(10), field.Equals(40), field.Equals(20), field.Equals(30))) + if err != nil { + t.Fatalf("querying: %v", err) + } + for i, result := range resp.Results() { + if 1 == i { + if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) { + t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns) + } + } else { + if !reflect.DeepEqual(result.Row().Columns, []uint64{result.Row().Columns[0]}) { + t.Errorf("expected %#v for %d, but got %#v", []uint64{result.Row().Columns[0]}, i, result.Row().Columns) + } + } + } + +} + +func TestStringSliceEmptyAndNil(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("test-string-slice-nil") + fields := make([]*Field, 1) + fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + // first create a batch and test adding a single value with empty + // string - this failed with a translation error at one point, and + // how we catch it and treat it like a nil. + b, err := NewBatch(client, 2, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + r := Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(1) + r.Values[0] = "" + err = b.Add(r) + if err != nil { + t.Fatalf("adding: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + // now create a batch and add a mixture of string slice values + b, err = NewBatch(client, 6, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + r = Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(0) + r.Values[0] = []string{"a"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + + r.ID = uint64(1) + r.Values[0] = nil + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with nil stringslice to r: %v", err) + } + + r.ID = uint64(2) + r.Values[0] = []string{"a", "b", "z"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with idslice to r: %v", err) + } + + r.ID = uint64(3) + r.Values[0] = []string{"b", "c"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with stringslice to r: %v", err) + } + + r.ID = uint64(4) + r.Values[0] = []string{} + err = b.Add(r) + if err != nil { + t.Fatalf("adding batch with stringslice to r: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + rows := []interface{}{"a", "b", "c", "z"} + resp, err := client.Query(idx.BatchQuery(fields[0].Row(rows[0]), fields[0].Row(rows[1]), fields[0].Row(rows[2]), fields[0].Row(rows[3]))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + // TODO test is flaky because we can't guarantee what a,b,c map to + expectations := [][]uint64{{0, 2}, {2, 3}, {3}, {2}} + for i, re := range resp.Results() { + if !reflect.DeepEqual(re.Row().Columns, expectations[i]) { + t.Errorf("expected row %v to have columns %v, but got %v", rows[i], expectations[i], re.Row().Columns) + } + } + +} + +func TestStringSlice(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("test-string-slice") + fields := make([]*Field, 1) + fields[0] = idx.Field("strslice", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, fields) + if err != nil { + t.Fatalf("creating new batch: %v", err) + } + + rowmap := map[string]uint64{ + "c": 9, + "d": 10, + "f": 13, + } + b.rowTranslations["strslice"] = make(map[string]agedTranslation) + for k, id := range rowmap { + b.rowTranslations["strslice"][k] = agedTranslation{ + id: id, + } + } + + r := Row{Values: make([]interface{}, len(fields))} + r.ID = uint64(0) + r.Values[0] = []string{"a"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0}) { + t.Fatalf("expected []int{0}, got: %v", got) + } + + r.ID = uint64(1) + r.Values[0] = []string{"a", "b", "c"} + err = b.Add(r) + if err != nil { + t.Fatalf("adding to batch: %v", err) + } + if got := b.toTranslateSets["strslice"]["a"]; !reflect.DeepEqual(got, []int{0, 1}) { + t.Fatalf("expected []int{0,1}, got: %v", got) + } + if got := b.toTranslateSets["strslice"]["b"]; !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("expected []int{1}, got: %v", got) + } + if got, ok := b.toTranslateSets["strslice"]["c"]; ok { + t.Fatalf("should be nothing at c, got: %v", got) + } + if got := b.rowIDSets["strslice"][1]; !reflect.DeepEqual(got, []uint64{9}) { + t.Fatalf("expected c to map to rowID 9 but got %v", got) + } + + r.ID = uint64(2) + r.Values[0] = []string{"d", "e", "f"} + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + if got, ok := b.toTranslateSets["strslice"]["d"]; ok { + t.Fatalf("should be nothing at d, got: %v", got) + } + if got, ok := b.toTranslateSets["strslice"]["f"]; ok { + t.Fatalf("should be nothing at f, got: %v", got) + } + if got := b.toTranslateSets["strslice"]["e"]; !reflect.DeepEqual(got, []int{2}) { + t.Fatalf("expected []int{2}, got: %v", got) + } + if got := b.rowIDSets["strslice"][2]; !reflect.DeepEqual(got, []uint64{10, 13}) { + t.Fatalf("expected c to map to rowID 9 but got %v", got) + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("translating: %v", err) + } + + if got0 := b.rowIDSets["strslice"][0]; len(got0) != 1 { + t.Errorf("after translation, rec 0, wrong len: %v", got0) + } else if got1 := b.rowIDSets["strslice"][1]; len(got1) != 3 || got1[0] != 9 || (got1[1] != got0[0] && got1[2] != got0[0]) { + t.Errorf("after translation, rec 1: %v, rec 0: %v", got1, got0) + } else if got2 := b.rowIDSets["strslice"][2]; len(got2) != 3 || got2[0] != 10 || got2[1] != 13 || got2[2] == got1[2] || got2[2] == got0[0] { + t.Errorf("after translation, rec 2: %v", got2) + } + + frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + result := resp.Result() + if !reflect.DeepEqual(result.Row().Columns, []uint64{0, 1}) { + t.Fatalf("expected a to be [0,1], got %v", result.Row().Columns) + } +} + +func TestSingleClearBatchRegression(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-blah") + numFields := 1 + fields := make([]*Field, numFields) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + _, err = client.Query(fields[0].Set("row1", 1)) + if err != nil { + t.Fatalf("setting bit: %v", err) + } + + b, err := NewBatch(client, 1, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + r := Row{ID: uint64(1), Values: make([]interface{}, numFields), Clears: make(map[int]interface{})} + r.Values[0] = nil + r.Clears[0] = "row1" + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("wrong error from batch add: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("error importing: %v", err) + } + + resp, err := client.Query(fields[0].Row("row1")) + if err != nil { + t.Fatalf("error querying: %v", err) + } + result := resp.Results()[0].Row().Columns + if len(result) != 0 { + t.Fatalf("unexpected values in row: result %+v", result) + } + +} + +func TestBatches(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-blah") + numFields := 5 + fields := make([]*Field, numFields) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + fields[1] = idx.Field("one", OptFieldKeys(true)) + fields[2] = idx.Field("two", OptFieldKeys(true)) + fields[3] = idx.Field("three", OptFieldTypeInt()) + fields[4] = idx.Field("four", OptFieldTypeTime(TimeQuantumYearMonthDay)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + b, err := NewBatch(client, 10, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + r := Row{Values: make([]interface{}, numFields), Clears: make(map[int]interface{})} + r.Time.Set(time.Date(2019, time.January, 2, 15, 45, 0, 0, time.UTC)) + + for i := 0; i < 9; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Values[3] = int64(99) + r.Values[4] = uint64(1) + r.Time.SetMonth("01") + } else { + r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Values[3] = int64(-10) + r.Values[4] = uint64(1) + r.Time.SetMonth("02") + } + if i == 8 { + r.Values[0] = nil + r.Clears[1] = uint64(97) + r.Clears[2] = "c" + r.Values[3] = nil + r.Values[4] = nil + } + err := b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + } + + if len(b.toTranslate[0]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[0]") + } + for k, ints := range b.toTranslate[0] { + if k == "a" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6}) { + t.Fatalf("wrong ints for key a in field zero: %v", ints) + } + } else if k == "x" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key x in field zero: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + if !reflect.DeepEqual(b.toTranslateClear, map[int]map[string][]int{2: {"c": {8}}}) { + t.Errorf("unexpected toTranslateClear: %+v", b.toTranslateClear) + } + if !reflect.DeepEqual(b.clearRowIDs, map[int]map[int]uint64{1: {8: 97}, 2: {}}) { + t.Errorf("unexpected clearRowIDs: %+v", b.clearRowIDs) + } + + if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0}) { + t.Fatalf("unexpected values: %v", b.values["three"]) + } + if !reflect.DeepEqual(b.nullIndices["three"], []uint64{8}) { + t.Fatalf("unexpected nullIndices: %v", b.nullIndices["three"]) + } + + if len(b.toTranslate[1]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[1]") + } + for k, ints := range b.toTranslate[1] { + if k == "b" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) { + t.Fatalf("wrong ints for key b in field one: %v", ints) + } + } else if k == "y" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key y in field one: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + + if len(b.toTranslate[2]) != 2 { + t.Fatalf("wrong number of keys in toTranslate[2]") + } + for k, ints := range b.toTranslate[2] { + if k == "c" { + if !reflect.DeepEqual(ints, []int{0, 2, 4, 6, 8}) { + t.Fatalf("wrong ints for key c in field two: %v", ints) + } + } else if k == "z" { + if !reflect.DeepEqual(ints, []int{1, 3, 5, 7}) { + t.Fatalf("wrong ints for key z in field two: %v", ints) + } + + } else { + t.Fatalf("unexpected key %s", k) + } + } + + err = b.Add(r) + if err != ErrBatchNowFull { + t.Fatalf("should have gotten full batch error, but got %v", err) + } + + err = b.Add(r) + if err != ErrBatchAlreadyFull { + t.Fatalf("should have gotten already full batch error, but got %v", err) + } + + if !reflect.DeepEqual(b.values["three"], []int64{99, -10, 99, -10, 99, -10, 99, -10, 0, 0}) { + t.Fatalf("unexpected values: %v", b.values["three"]) + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + // we don't know which key will get translated first, but we do know the pattern + if fidx == 0 { + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, nilSentinel, nilSentinel}) && + !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, nilSentinel, nilSentinel}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + + } else if fidx == 4 { + if !reflect.DeepEqual(rowIDs, []uint64{1, 1, 1, 1, 1, 1, 1, 1, nilSentinel, nilSentinel}) { + t.Fatalf("unexpected rowids for time field") + } + } else if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + } else { + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, nilSentinel}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, nilSentinel}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + } + + if !reflect.DeepEqual(b.clearRowIDs[1], map[int]uint64{8: 97}) { + t.Errorf("unexpected clearRowIDs after translation: %+v", b.clearRowIDs[1]) + } + if !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 2}) && !reflect.DeepEqual(b.clearRowIDs[2], map[int]uint64{8: 1}) { + t.Errorf("unexpected clearRowIDs: after translation%+v", b.clearRowIDs[2]) + } + + frags, clearFrags, err := b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + b.reset() + + for i := 9; i < 19; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "a" + r.Values[1] = "b" + r.Values[2] = "c" + r.Values[3] = int64(99) + r.Values[4] = uint64(1) + } else { + r.Values[0] = "x" + r.Values[1] = "y" + r.Values[2] = "z" + r.Values[3] = int64(-10) + r.Values[4] = uint64(2) + } + err := b.Add(r) + if i != 18 && err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + if i == 18 && err != ErrBatchNowFull { + t.Fatalf("unexpected err: %v", err) + } + } + + // should do nothing + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + continue + } + // we don't know which key will get translated first, but we do know the pattern + if !reflect.DeepEqual(rowIDs, []uint64{1, 2, 1, 2, 1, 2, 1, 2, 1, 2}) && !reflect.DeepEqual(rowIDs, []uint64{2, 1, 2, 1, 2, 1, 2, 1, 2, 1}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + + b.reset() + + for i := 19; i < 29; i++ { + r.ID = uint64(i) + if i%2 == 0 { + r.Values[0] = "d" + r.Values[1] = "e" + r.Values[2] = "f" + r.Values[3] = int64(100) + r.Values[4] = uint64(3) + } else { + r.Values[0] = "u" + r.Values[1] = "v" + r.Values[2] = "w" + r.Values[3] = int64(0) + r.Values[4] = uint64(4) + } + err := b.Add(r) + if i != 28 && err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + if i == 28 && err != ErrBatchNowFull { + t.Fatalf("unexpected err: %v", err) + } + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("doing translation: %v", err) + } + + frags, clearFrags, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Errorf("making fragments: %v", err) + } + + err = b.doImport(frags, clearFrags) + if err != nil { + t.Fatalf("doing import: %v", err) + } + + for fidx, rowIDs := range b.rowIDs { + // we don't know which key will get translated first, but we do know the pattern + if fidx == 3 { + if len(rowIDs) != 0 { + t.Fatalf("expected no rowIDs for int field, but got: %v", rowIDs) + } + continue + } + if !reflect.DeepEqual(rowIDs, []uint64{3, 4, 3, 4, 3, 4, 3, 4, 3, 4}) && !reflect.DeepEqual(rowIDs, []uint64{4, 3, 4, 3, 4, 3, 4, 3, 4, 3}) { + t.Fatalf("unexpected row ids for field %d: %v", fidx, rowIDs) + } + } + + frags, _, err = b.makeFragments(make(fragments), make(fragments)) + if err != nil { + t.Fatalf("making fragments: %v", err) + } + + var n int + for key := range frags { + if key.shard == 0 { + n++ + } + } + if n != 5 { // zero, one, two, four (three is an int field so not in fragments) + _exists + t.Fatalf("there should be 5 views, but have %d", n) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), + fields[1].Row("b"), + fields[2].Row("c"), + fields[3].Equals(99))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results := resp.Results() + for _, j := range []int{0, 2, 3} { + cols := results[j].Row().Columns + if !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18}) { + t.Fatalf("unexpected columns for a: %v", cols) + } + } + res := results[1] + cols := res.Row().Columns + if !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 8, 10, 12, 14, 16, 18}) { + t.Fatalf("unexpected columns for field 1 row b: %v", cols) + } + + resp, err = client.Query(idx.BatchQuery(fields[0].Row("d"), + fields[1].Row("e"), + fields[2].Row("f"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results = resp.Results() + for _, res := range results { + cols := res.Row().Columns + if !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + + resp, err = client.Query(idx.BatchQuery(fields[3].GT(-11), + fields[3].Equals(0), + fields[3].Equals(100), + fields[4].Range(1, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.January, 29, 0, 0, 0, 0, time.UTC)), + fields[4].Range(1, time.Date(2019, time.February, 1, 0, 0, 0, 0, time.UTC), time.Date(2019, time.February, 29, 0, 0, 0, 0, time.UTC)))) + if err != nil { + t.Fatalf("querying: %v", err) + } + results = resp.Results() + cols = results[0].Row().Columns + if !reflect.DeepEqual(cols, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) { + t.Fatalf("all columns (but 8) should be greater than -11, but got: %v", cols) + } + cols = results[1].Row().Columns + if !reflect.DeepEqual(cols, []uint64{19, 21, 23, 25, 27}) { + t.Fatalf("wrong cols for ==0: %v", cols) + } + cols = results[2].Row().Columns + if !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) { + t.Fatalf("wrong cols for ==100: %v", cols) + } + cols = results[3].Row().Columns + exp := []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18} + if !reflect.DeepEqual(cols, exp) { + t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp) + } + cols = results[4].Row().Columns + exp = []uint64{1, 3, 5, 7} + if !reflect.DeepEqual(cols, exp) { + t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp) + } + + b.reset() + r.ID = uint64(0) + r.Values[0] = "x" + r.Values[1] = "b" + r.Clears[0] = "a" + r.Clears[1] = "b" // b should get cleared + err = b.Add(r) + if err != nil { + t.Fatalf("adding with clears: %v", err) + } + err = b.Import() + if err != nil { + t.Fatalf("importing w/clears: %v", err) + } + resp, err = client.Query(idx.BatchQuery( + fields[0].Row("a"), + fields[0].Row("x"), + fields[1].Row("b"), + )) + if err != nil { + t.Fatalf("querying after clears: %v", err) + } + if arow := resp.Results()[0].Row().Columns; arow[0] == 0 { + t.Errorf("shouldn't have id 0 in row a after clearing! %v", arow) + } + if xrow := resp.Results()[1].Row().Columns; xrow[0] != 0 { + t.Errorf("should have id 0 in row x after setting %v", xrow) + } + if brow := resp.Results()[2].Row().Columns; brow[0] == 0 { + t.Errorf("shouldn't have id 0 in row b after clearing! %v", brow) + } + + // TODO test importing across multiple shards +} + +func TestBatchesStringIDs(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-blah", OptIndexKeys(true)) + fields := make([]*Field, 1) + fields[0] = idx.Field("zero", OptFieldKeys(true)) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, fields) + if err != nil { + t.Fatalf("getting new batch: %v", err) + } + + r := Row{Values: make([]interface{}, 1)} + + for i := 0; i < 3; i++ { + r.ID = strconv.Itoa(i) + if i%2 == 0 { + r.Values[0] = "a" + } else { + r.Values[0] = "x" + } + err := b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("unexpected err adding record: %v", err) + } + } + + if len(b.toTranslateID) != 3 { + t.Fatalf("id translation table unexpected size: %v", b.toTranslateID) + } + for i, k := range b.toTranslateID { + if ik, err := strconv.Atoi(k); err != nil || ik != i { + t.Errorf("unexpected toTranslateID key %s at index %d", k, i) + } + } + + err = b.doTranslation() + if err != nil { + t.Fatalf("translating: %v", err) + } + + // the ids are based off what the strings hash to, and are at the + // very beginning of a few different shards. this could change if + // Pilosa's hashing algorithm changes. + if err := isPermutationOfInt(b.ids, []uint64{44040193, 45088769, 41943041}); err != nil { + t.Fatalf("wrong ids: %v. exp/got:\n%v\n%v", err, []uint64{44040193, 45088769, 41943041}, b.ids) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err := client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("x"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results := resp.Results() + for i, res := range results { + cols := res.Row().Keys + if i == 0 && !reflect.DeepEqual(cols, []string{"0", "2"}) && !reflect.DeepEqual(cols, []string{"2", "0"}) { + t.Fatalf("unexpected columns: %v", cols) + } + if i == 1 && !reflect.DeepEqual(cols, []string{"1"}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + + b.reset() + + r.ID = "1" + r.Values[0] = "a" + err = b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + r.ID = "3" + r.Values[0] = "z" + err = b.Add(r) + if err != nil { + t.Fatalf("unexpected err adding record: %v", err) + } + + err = b.Import() + if err != nil { + t.Fatalf("importing: %v", err) + } + + resp, err = client.Query(idx.BatchQuery(fields[0].Row("a"), fields[0].Row("z"))) + if err != nil { + t.Fatalf("querying: %v", err) + } + + results = resp.Results() + for i, res := range results { + cols := res.Row().Keys + if err := isPermutationOf(cols, []string{"0", "1", "2"}); i == 0 && err != nil { + t.Fatalf("unexpected columns: %v: %v", cols, err) + } + if i == 1 && !reflect.DeepEqual(cols, []string{"3"}) { + t.Fatalf("unexpected columns: %v", cols) + } + } + +} + +func isPermutationOf(one, two []string) error { + if len(one) != len(two) { + return errors.Errorf("different lengths %d and %d", len(one), len(two)) + } +outer: + for _, vOne := range one { + for j, vTwo := range two { + if vOne == vTwo { + two = append(two[:j], two[j+1:]...) + continue outer + } + } + return errors.Errorf("%s in one but not two", vOne) + } + if len(two) != 0 { + return errors.Errorf("vals in two but not one: %v", two) + } + return nil +} + +func isPermutationOfInt(one, two []uint64) error { + if len(one) != len(two) { + return errors.Errorf("different lengths %d and %d", len(one), len(two)) + } +outer: + for _, vOne := range one { + for j, vTwo := range two { + if vOne == vTwo { + two = append(two[:j], two[j+1:]...) + continue outer + } + } + return errors.Errorf("%d in one but not two", vOne) + } + if len(two) != 0 { + return errors.Errorf("vals in two but not one: %v", two) + } + return nil +} + +func TestQuantizedTime(t *testing.T) { + cases := []struct { + name string + time time.Time + year string + month string + day string + hour string + quantum TimeQuantum + reset bool + exp []string + expErr string + }{ + { + name: "no time quantum", + expErr: "", + }, + { + name: "no time quantum with data", + year: "2017", + exp: []string{}, + expErr: "", + }, + { + name: "no data", + quantum: TimeQuantumYear, + exp: nil, + expErr: "", + }, + { + name: "timestamp", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YMDH", + exp: []string{"2013", "201310", "20131016", "2013101617"}, + }, + { + name: "timestamp-less-granular", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YM", + exp: []string{"2013", "201310"}, + }, + { + name: "timestamp-mid-granular", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "MD", + exp: []string{"201310", "20131016"}, + }, + { + name: "justyear", + year: "2013", + quantum: "Y", + exp: []string{"2013"}, + }, + { + name: "justyear-wantmonth", + year: "2013", + quantum: "YM", + expErr: "no data set for month", + }, + { + name: "timestamp-changeyear", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + year: "2019", + quantum: "YMDH", + exp: []string{"2019", "201910", "20191016", "2019101617"}, + }, + { + name: "yearmonthdayhour", + year: "2013", + month: "10", + day: "16", + hour: "17", + quantum: "YMDH", + exp: []string{"2013", "201310", "20131016", "2013101617"}, + }, + { + name: "timestamp-changehour", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + hour: "05", + quantum: "MDH", + exp: []string{"201310", "20131016", "2013101605"}, + }, + { + name: "timestamp", + time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)), + quantum: "YMDH", + reset: true, + exp: nil, + }, + } + + for i, test := range cases { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + tq := QuantizedTime{} + var zt time.Time + if zt != test.time { + tq.Set(test.time) + } + if test.year != "" { + tq.SetYear(test.year) + } + if test.month != "" { + tq.SetMonth(test.month) + } + if test.day != "" { + tq.SetDay(test.day) + } + if test.hour != "" { + tq.SetHour(test.hour) + } + if test.reset { + tq.Reset() + } + + views, err := tq.views(test.quantum) + if !reflect.DeepEqual(views, test.exp) { + t.Errorf("unexpected views, got/want:\n%v\n%v\n", views, test.exp) + } + if (err != nil && err.Error() != test.expErr) || (err == nil && test.expErr != "") { + t.Errorf("unexpected error, got/want:\n%v\n%s\n", err, test.expErr) + } + }) + } + +} + +func TestBatchStaleness(t *testing.T) { + client := DefaultClient() + schema := NewSchema() + idx := schema.Index("gopilosatest-blah") + field := idx.Field("anint", OptFieldTypeInt()) + err := client.SyncSchema(schema) + if err != nil { + t.Fatalf("syncing schema: %v", err) + } + defer func() { + err := client.DeleteIndex(idx) + if err != nil { + t.Logf("problem cleaning up from test: %v", err) + } + }() + + b, err := NewBatch(client, 3, idx, []*Field{field}, OptMaxStaleness(time.Millisecond)) + if err != nil { + t.Fatalf("getting batch: %v", err) + } + + r := Row{ID: uint64(0), Values: []interface{}{int64(0)}} + err = b.Add(r) + if err != nil && err != ErrBatchNowFull { + t.Fatalf("adding to batch: %v", err) + } + + // sleep so batch becomes stale + time.Sleep(time.Millisecond) + + r = Row{ID: uint64(1), Values: []interface{}{int64(0)}} + err = b.Add(r) + if err != ErrBatchNowStale { + t.Fatal("batch expected to be stale") + } +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 000000000..405af53b2 --- /dev/null +++ b/client/client.go @@ -0,0 +1,1797 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. +package client + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "hash/fnv" + "io" + "log" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/golang/protobuf/proto" //nolint:staticcheck + "github.com/opentracing/opentracing-go" + "github.com/pilosa/pilosa/v2" + pnet "github.com/pilosa/pilosa/v2/net" + "github.com/pilosa/pilosa/v2/pb" + "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/stats" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +// PQLVersion is the version of PQL expected by the client +const PQLVersion = "1.0" + +// DefaultShardWidth is used if an index doesn't have it defined. +const DefaultShardWidth = 1 << 20 + +const maxHosts = 10 + +// Client is the HTTP client for Pilosa server. +type Client struct { + cluster *Cluster + client *http.Client + logger *log.Logger + primaryURI *pnet.URI + primaryLock *sync.RWMutex + manualFragmentNode *fragmentNode + manualServerURI *pnet.URI + tracer opentracing.Tracer + Stats stats.StatsClient + // An exponential backoff algorithm retries requests exponentially (if an HTTP request fails), + // increasing the waiting time between retries up to a maximum backoff time. + maxBackoff time.Duration + maxRetries int + + nat map[pnet.URI]pnet.URI + + shardNodes shardNodes + tick *time.Ticker + done chan struct{} + + // TODO — remove this! Pilosa should have API which gives us the node->partition mapping + Hasher Hasher +} + +func (c *Client) getURIsForShard(index string, shard uint64) ([]*pnet.URI, error) { + uris, ok := c.shardNodes.Get(index, shard) + if ok { + return uris, nil + } + fragmentNodes, err := c.fetchFragmentNodes(index, shard) + if err != nil { + return nil, errors.Wrap(err, "trying to look up nodes for shard") + } + uris = make([]*pnet.URI, 0, len(fragmentNodes)) + for _, fn := range fragmentNodes { + uris = append(uris, fn.URI()) + } + c.shardNodes.Put(index, shard, uris) + return uris, nil +} + +func (c *Client) runChangeDetection() { + for { + select { + case <-c.tick.C: + c.detectClusterChanges() + case <-c.done: + return + } + } +} + +func (c *Client) Close() error { + c.tick.Stop() + close(c.done) + return nil +} + +// detectClusterChanges chooses a random index and shard from the +// shardNodes cache and deletes it. It then looks it up from Pilosa to +// see if it still matches, and if not it drops the whole cache. +func (c *Client) detectClusterChanges() { + c.shardNodes.mu.Lock() + needsUnlock := true + // we rely on Go's random map iteration order to get a random + // element. If it doesn't end up being random, it shouldn't + // actually matter. + for index, shardMap := range c.shardNodes.data { + for shard, uris := range shardMap { + delete(shardMap, shard) + c.shardNodes.data[index] = shardMap + c.shardNodes.mu.Unlock() + needsUnlock = false + newURIs, err := c.getURIsForShard(index, shard) // refetch URIs from server. + if err != nil { + c.logger.Printf("problem invalidating shard node cache: %v", err) + return + } + if len(uris) != len(newURIs) { + c.logger.Printf("invalidating shard node cache old: %v, new: %v", uris, newURIs) + c.shardNodes.Invalidate() + return + } + for i := range uris { + u1, u2 := uris[i], newURIs[i] + if *u1 != *u2 { + c.logger.Printf("invalidating shard node cache, uri mismatch at %d old: %v, new: %v", i, uris, newURIs) + c.shardNodes.Invalidate() + return + } + } + break + } + break + } + if needsUnlock { + c.shardNodes.mu.Unlock() + } +} + +// DefaultClient creates a client with the default address and options. +func DefaultClient() *Client { + return newClientWithCluster(NewClusterWithHost(pnet.DefaultURI()), nil) +} + +func newClientFromAddresses(addresses []string, options *ClientOptions) (*Client, error) { + uris := make([]*pnet.URI, len(addresses)) + for i, address := range addresses { + uri, err := pnet.NewURIFromAddress(address) + if err != nil { + return nil, err + } + uris[i] = uri + } + cluster := NewClusterWithHost(uris...) + client := newClientWithCluster(cluster, options) + return client, nil +} + +func newClientWithCluster(cluster *Cluster, options *ClientOptions) *Client { + client := newClientWithOptions(options) + client.cluster = cluster + return client +} + +func newClientWithURI(uri *pnet.URI, options *ClientOptions) *Client { + client := newClientWithOptions(options) + if options.manualServerAddress { + fragmentNode := newFragmentNodeFromURI(uri) + client.manualFragmentNode = &fragmentNode + client.manualServerURI = uri + client.cluster = NewClusterWithHost() + } + client.cluster = NewClusterWithHost(uri) + return client +} + +func newClientWithOptions(options *ClientOptions) *Client { + if options == nil { + options = &ClientOptions{} + } + options = options.withDefaults() + + c := &Client{ + client: newHTTPClient(options.withDefaults()), + logger: log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds), + primaryLock: &sync.RWMutex{}, + + shardNodes: newShardNodes(), + tick: time.NewTicker(time.Minute), + done: make(chan struct{}), + + nat: options.nat, + + // TODO get rid of this. Pilosa should have api to expose node->partition mapping. + Hasher: &jmphasher{}, + } + + if options.tracer == nil { + c.tracer = NoopTracer{} + } else { + c.tracer = options.tracer + } + if options.stats == nil { + c.Stats = stats.NopStatsClient + } else { + c.Stats = options.stats + } + + c.maxRetries = *options.retries + c.maxBackoff = 2 * time.Minute + go c.runChangeDetection() + return c + +} + +// NewClient creates a client with the given address, URI, or cluster and options. +func NewClient(addrURIOrCluster interface{}, options ...ClientOption) (*Client, error) { + var cluster *Cluster + clientOptions := &ClientOptions{ + nat: make(map[pnet.URI]pnet.URI), + } + err := clientOptions.addOptions(options...) + if err != nil { + return nil, err + } + + switch u := addrURIOrCluster.(type) { + case string: + uri, err := pnet.NewURIFromAddress(u) + if err != nil { + return nil, err + } + return newClientWithURI(uri, clientOptions), nil + case []string: + if len(u) == 1 { + uri, err := pnet.NewURIFromAddress(u[0]) + if err != nil { + return nil, err + } + return newClientWithURI(uri, clientOptions), nil + } else if clientOptions.manualServerAddress { + return nil, ErrSingleServerAddressRequired + } + return newClientFromAddresses(u, clientOptions) + case *pnet.URI: + uriCopy := *u + return newClientWithURI(&uriCopy, clientOptions), nil + case []*pnet.URI: + if len(u) == 1 { + uriCopy := *u[0] + return newClientWithURI(&uriCopy, clientOptions), nil + } else if clientOptions.manualServerAddress { + return nil, ErrSingleServerAddressRequired + } + cluster = NewClusterWithHost(u...) + case *Cluster: + cluster = u + case nil: + cluster = NewClusterWithHost() + default: + return nil, ErrAddrURIClusterExpected + } + + return newClientWithCluster(cluster, clientOptions), nil +} + +// Query runs the given query against the server with the given options. +// Pass nil for default options. +func (c *Client) Query(query PQLQuery, options ...interface{}) (*QueryResponse, error) { + span := c.tracer.StartSpan("Client.Query") + defer span.Finish() + + if err := query.Error(); err != nil { + return nil, err + } + queryOptions := &QueryOptions{} + err := queryOptions.addOptions(options...) + if err != nil { + return nil, err + } + serializedQuery := query.Serialize() + reqData, err := makeRequestData(serializedQuery.String(), queryOptions) + if err != nil { + return nil, errors.Wrap(err, "making request data") + } + path := fmt.Sprintf("/index/%s/query", query.Index().name) + _, respData, err := c.HTTPRequest("POST", path, reqData, defaultProtobufHeaders()) + if err != nil { + return nil, err + } + iqr := &pb.QueryResponse{} + err = proto.Unmarshal(respData, iqr) + if err != nil { + return nil, err + } + queryResponse, err := newQueryResponseFromInternal(iqr) + if err != nil { + return nil, err + } + return queryResponse, nil +} + +// CreateIndex creates an index on the server using the given Index struct. +func (c *Client) CreateIndex(index *Index) error { + span := c.tracer.StartSpan("Client.CreateIndex") + defer span.Finish() + + data := []byte(index.options.String()) + path := fmt.Sprintf("/index/%s", index.name) + status, body, err := c.HTTPRequest("POST", path, data, nil) + if err != nil { + return errors.Wrapf(err, "creating index: %s", index.name) + } + var resp struct { + CreatedAt int64 `json:"createdAt,omitempty"` + } + if err := json.Unmarshal(body, &resp); err == nil && resp.CreatedAt != 0 { + index.createdAt = resp.CreatedAt + } + + if status == http.StatusConflict { + return ErrIndexExists + } + return nil +} + +// CreateField creates a field on the server using the given Field struct. +func (c *Client) CreateField(field *Field) error { + span := c.tracer.StartSpan("Client.CreateField") + defer span.Finish() + + data := []byte(field.options.String()) + path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) + status, body, err := c.HTTPRequest("POST", path, data, nil) + if err != nil { + return errors.Wrapf(err, "creating field: %s in index: %s", field.name, field.index.name) + } + var resp struct { + CreatedAt int64 `json:"createdAt,omitempty"` + } + if err := json.Unmarshal(body, &resp); err == nil && resp.CreatedAt != 0 { + field.createdAt = resp.CreatedAt + } + + if status == http.StatusConflict { + return ErrFieldExists + } + return nil +} + +// EnsureIndex creates an index on the server if it does not exist. +func (c *Client) EnsureIndex(index *Index) error { + err := c.CreateIndex(index) + if err == ErrIndexExists { + return nil + } + return errors.Wrap(err, "creating index") +} + +func (c *Client) SyncIndex(index *Index) error { + err := c.EnsureIndex(index) + if err != nil { + return errors.Wrapf(err, "ensuring index exists") + } + + for name, field := range index.fields { + if name == "_exists" { + continue + } + err = c.EnsureField(field) + if err != nil { + return errors.Wrapf(err, "ensuring field") + } + } + + return nil +} + +// EnsureField creates a field on the server if it doesn't exists. +func (c *Client) EnsureField(field *Field) error { + err := c.CreateField(field) + if err == ErrFieldExists { + return nil + } + return err +} + +// DeleteIndex deletes an index on the server. +func (c *Client) DeleteIndex(index *Index) error { + if index != nil { + return c.DeleteIndexByName(index.Name()) + } + return nil +} + +// DeleteIndexByName deletes the named index on the server. +func (c *Client) DeleteIndexByName(index string) error { + span := c.tracer.StartSpan("Client.DeleteIndex") + defer span.Finish() + + path := fmt.Sprintf("/index/%s", index) + _, _, err := c.HTTPRequest("DELETE", path, nil, nil) + return err +} + +// DeleteField deletes a field on the server. +func (c *Client) DeleteField(field *Field) error { + span := c.tracer.StartSpan("Client.DeleteField") + defer span.Finish() + + path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) + _, _, err := c.HTTPRequest("DELETE", path, nil, nil) + return err +} + +// SyncSchema updates a schema with the indexes and fields on the server and +// creates the indexes and fields in the schema on the server side. +// This function does not delete indexes and the fields on the server side nor in the schema. +func (c *Client) SyncSchema(schema *Schema) error { + span := c.tracer.StartSpan("Client.SyncSchema") + defer span.Finish() + schema.mu.Lock() + defer schema.mu.Unlock() + serverSchema, err := c.Schema() + if err != nil { + return err + } + serverSchema.mu.RLock() + defer serverSchema.mu.RUnlock() + + return c.syncSchema(schema, serverSchema) +} + +func (c *Client) syncSchema(schema *Schema, serverSchema *Schema) error { + var err error + + // find out local - remote schema + diffSchema := schema.diff(serverSchema) + // create the indexes and fields which doesn't exist on the server side + for indexName, index := range diffSchema.indexes { + if _, ok := serverSchema.indexes[indexName]; !ok { + err = c.EnsureIndex(index) + if err != nil { + return errors.Wrap(err, "ensuring index") + } + } + for name, field := range index.fields { + if name == "_exists" { + continue + } + err = c.EnsureField(field) + if err != nil { + return errors.Wrapf(err, "ensuring field") + } + } + } + + // find out remote - local schema + diffSchema = serverSchema.diff(schema) + for indexName, index := range diffSchema.indexes { + if localIndex, ok := schema.indexes[indexName]; !ok { + schema.indexes[indexName] = index + } else { + for fieldName, field := range index.fields { + localIndex.fields[fieldName] = field + } + } + } + + return nil +} + +// Schema returns the indexes and fields on the server. +func (c *Client) Schema() (*Schema, error) { + span := c.tracer.StartSpan("Client.Schema") + defer span.Finish() + + var indexes []SchemaIndex + indexes, err := c.readSchema() + if err != nil { + return nil, err + } + schema := NewSchema() + for _, indexInfo := range indexes { + index := schema.indexWithOptions(indexInfo.Name, indexInfo.CreatedAt, indexInfo.ShardWidth, indexInfo.Options.asIndexOptions()) + + for _, fieldInfo := range indexInfo.Fields { + index.fieldWithOptions(fieldInfo.Name, fieldInfo.CreatedAt, fieldInfo.Options.asFieldOptions()) + } + } + return schema, nil +} + +// Import imports data for a single shard using the regular import +// endpoint rather than import-roaring. This is good for e.g. mutex or +// bool fields where import-roaring is not supported. +func (c *Client) Import(field *Field, shard uint64, vals, ids []uint64, clear bool) error { + path, data, err := c.EncodeImport(field, shard, vals, ids, clear) + if err != nil { + return errors.Wrap(err, "encoding import request") + } + err = c.DoImport(field.index.Name(), shard, path, data) + return errors.Wrap(err, "doing import") +} + +// EncodeImport computes the HTTP path and payload for an import +// request. It is typically followed by a call to DoImport. +func (c *Client) EncodeImport(field *Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) { + msg := &pb.ImportRequest{ + Index: field.index.Name(), + IndexCreatedAt: field.index.CreatedAt(), + Field: field.Name(), + FieldCreatedAt: field.CreatedAt(), + Shard: shard, + RowIDs: vals, + ColumnIDs: ids, + } + data, err = proto.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling Import to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.index.Name(), field.Name(), strconv.FormatBool(clear)) + return path, data, nil +} + +// DoImport takes a path and data payload (normally from EncodeImport +// or EncodeImportValues), logs the import, finds all nodes which own +// this shard, and concurrently imports to those nodes. +func (c *Client) DoImport(index string, shard uint64, path string, data []byte) error { + // Unlike ImportRoaring, Pilosa does not forward requests to the + // .../import endpoint to all replicas, so we must do that + // here. Yes this is odd. To make it worse, if a ../import request + // was made with keys that needed to be translated server side, + // Pilosa would handle sending the translated data to all the + // appropriate nodes and replicas. + + uris, err := c.getURIsForShard(index, shard) + if err != nil { + return errors.Wrap(err, "getting uris") + } + + eg := errgroup.Group{} + for _, uri := range uris { + uri := uri + eg.Go(func() error { + return c.importData(uri, path, data) + }) + } + return errors.Wrap(eg.Wait(), "importing to nodes") +} + +// EncodeImportValues computes the HTTP path and payload for an +// import-values request. It is typically followed by a call to +// DoImportValues. +func (c *Client) EncodeImportValues(field *Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) { + msg := &pb.ImportValueRequest{ + Index: field.index.Name(), + IndexCreatedAt: field.index.CreatedAt(), + Field: field.Name(), + FieldCreatedAt: field.CreatedAt(), + Shard: shard, + ColumnIDs: ids, + Values: vals, + } + data, err = proto.Marshal(msg) + if err != nil { + return "", nil, errors.Wrap(err, "marshaling ImportValue to protobuf") + } + path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.index.Name(), field.Name(), strconv.FormatBool(clear)) + return path, data, nil +} + +// ImportValues takes the given integer values and column ids (which +// must all be in the given shard) and imports them into the given +// index,field,shard on all nodes which should hold that shard. It +// assumes that the ids have been translated from keys if necessary +// and so tells Pilosa to ignore checking if the index uses column +// keys. ImportValues wraps EncodeImportValues and DoImportValues — +// these are broken out and exported so that performance conscious +// users can re-use the same vals and ids byte buffers for local +// encoding, while performing the imports concurrently. +func (c *Client) ImportValues(field *Field, shard uint64, vals []int64, ids []uint64, clear bool) error { + path, data, err := c.EncodeImportValues(field, shard, vals, ids, clear) + if err != nil { + return errors.Wrap(err, "encoding import-values request") + } + err = c.DoImportValues(field.index.Name(), shard, path, data) + return errors.Wrap(err, "doing import values") +} + +// DoImportValues is deprecated. Use DoImport. +func (c *Client) DoImportValues(index string, shard uint64, path string, data []byte) error { + return c.DoImport(index, shard, path, data) +} + +func (c *Client) fetchFragmentNodes(indexName string, shard uint64) ([]fragmentNode, error) { + if c.manualFragmentNode != nil { + return []fragmentNode{*c.manualFragmentNode}, nil + } + path := fmt.Sprintf("/internal/fragment/nodes?shard=%d&index=%s", shard, indexName) + _, body, err := c.HTTPRequest("GET", path, []byte{}, nil) + if err != nil { + return nil, err + } + fragmentNodes := []fragmentNode{} + var fragmentNodeURIs []fragmentNodeRoot + err = json.Unmarshal(body, &fragmentNodeURIs) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling fragment node URIs") + } + for _, nodeURI := range fragmentNodeURIs { + fragmentNodes = append(fragmentNodes, nodeURI.URI) + } + return fragmentNodes, nil +} + +func (c *Client) fetchPrimaryNode() (fragmentNode, error) { + if c.manualFragmentNode != nil { + return *c.manualFragmentNode, nil + } + status, err := c.Status() + if err != nil { + return fragmentNode{}, err + } + for _, node := range status.Nodes { + if node.IsPrimary { + nodeURI := node.URI.URI().Translate(c.nat) + return fragmentNode{ //nolint:gosimple + Scheme: nodeURI.Scheme, + Host: nodeURI.Host, + Port: nodeURI.Port, + }, nil + } + } + return fragmentNode{}, errors.New("Primary node not found") +} + +func (c *Client) importData(uri *pnet.URI, path string, data []byte) error { + if status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data); err != nil { + return errors.Wrapf(err, "import to %s", uri.HostPort()) + } else if status == http.StatusPreconditionFailed { + return ErrPreconditionFailed + } + + return nil +} + +// ImportRoaringBitmap can import pre-made bitmaps for a number of +// different views into the given field/shard. If the view name in the +// map is an empty string, the standard view will be used. +func (c *Client) ImportRoaringBitmap(field *Field, shard uint64, views map[string]*roaring.Bitmap, clear bool) error { + uris, err := c.getURIsForShard(field.index.Name(), shard) + if err != nil { + return errors.Wrap(err, "getting URIs for import") + } + err = c.importRoaringBitmap(uris[0], field, shard, views, &ImportOptions{clear: clear}) + return errors.Wrap(err, "importing bitmap") +} + +func (c *Client) importRoaringBitmap(uri *pnet.URI, field *Field, shard uint64, views viewImports, options *ImportOptions) error { + protoViews := []*pb.ImportRoaringRequestView{} + for name, bmp := range views { + buf := &bytes.Buffer{} + _, err := bmp.WriteTo(buf) + if err != nil { + return errors.Wrap(err, "marshalling bitmap") + } + protoViews = append(protoViews, &pb.ImportRoaringRequestView{ + Name: name, + Data: buf.Bytes(), + }) + } + params := url.Values{} + params.Add("clear", strconv.FormatBool(options.clear)) + path := makeRoaringImportPath(field, shard, params) + req := &pb.ImportRoaringRequest{ + Clear: options.clear, + Views: protoViews, + IndexCreatedAt: field.index.CreatedAt(), + FieldCreatedAt: field.CreatedAt(), + } + data, err := proto.Marshal(req) + if err != nil { + return err + } + + status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data) + if err != nil { + return errors.Wrapf(err, "roaring import to %s, status: %d", uri.HostPort(), status) + } + if status == http.StatusPreconditionFailed { + return ErrPreconditionFailed + } + + return nil +} + +// ExportField exports columns for a field. +func (c *Client) ExportField(field *Field) (io.Reader, error) { + span := c.tracer.StartSpan("Client.ExportField") + defer span.Finish() + + var shardsMax map[string]uint64 + var err error + + status, err := c.Status() + if err != nil { + return nil, err + } + shardsMax, err = c.shardsMax() + if err != nil { + return nil, err + } + status.indexMaxShard = shardsMax + shardURIs, err := c.statusToNodeShardsForIndex(status, field.index.Name()) + if err != nil { + return nil, err + } + + return newExportReader(c, shardURIs, field), nil +} + +// Info returns the server's configuration/host information. +func (c *Client) Info() (Info, error) { + span := c.tracer.StartSpan("Client.Info") + defer span.Finish() + + _, data, err := c.HTTPRequest("GET", "/info", nil, nil) + if err != nil { + return Info{}, errors.Wrap(err, "requesting /info") + } + info := Info{} + err = json.Unmarshal(data, &info) + if err != nil { + return Info{}, errors.Wrap(err, "unmarshaling /info data") + } + return info, nil +} + +// Status returns the server's status. +func (c *Client) Status() (Status, error) { + span := c.tracer.StartSpan("Client.Status") + defer span.Finish() + + _, data, err := c.HTTPRequest("GET", "/status", nil, nil) + if err != nil { + return Status{}, errors.Wrap(err, "requesting /status") + } + status := Status{} + err = json.Unmarshal(data, &status) + if err != nil { + return Status{}, errors.Wrap(err, "unmarshaling /status data") + } + return status, nil +} + +func (c *Client) readSchema() ([]SchemaIndex, error) { + _, data, err := c.HTTPRequest("GET", "/schema", nil, nil) + if err != nil { + return nil, errors.Wrap(err, "requesting /schema") + } + schemaInfo := SchemaInfo{} + err = json.Unmarshal(data, &schemaInfo) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling /schema data") + } + return schemaInfo.Indexes, nil +} + +func (c *Client) shardsMax() (map[string]uint64, error) { + _, data, err := c.HTTPRequest("GET", "/internal/shards/max", nil, nil) + if err != nil { + return nil, errors.Wrap(err, "requesting /internal/shards/max") + } + m := map[string]map[string]uint64{} + err = json.Unmarshal(data, &m) + if err != nil { + return nil, errors.Wrap(err, "unmarshaling /internal/shards/max data") + } + return m["standard"], nil +} + +// HTTPRequest sends an HTTP request to the Pilosa server (used by idk) +// nolint: deadcode +func (c *Client) HTTPRequest(method string, path string, data []byte, headers map[string]string) (status int, body []byte, err error) { + span := c.tracer.StartSpan("Client.HTTPRequest") + + status, body, err = c.httpRequest(method, path, data, headers, false) + span.Finish() + return +} + +// httpRequest makes a request to the cluster - use this when you want the +// client to choose a host, and it doesn't matter if the request goes to a +// specific host +func (c *Client) httpRequest(method string, path string, data []byte, headers map[string]string, usePrimary bool) (int, []byte, error) { + if data == nil { + data = []byte{} + } + + var ( + status int + body []byte + err error + ) + // try at most maxHosts non-failed hosts; protect against broken cluster.removeHost + for i := 0; i < maxHosts; i++ { + host, herr := c.host(usePrimary) + if herr != nil { + return status, nil, errors.Wrapf(herr, "getting host, previous err: %v", err) + } + // doRequest implements expotential backoff + status, body, err = c.doRequest(host, method, path, c.augmentHeaders(headers), data) + if err == nil { + break + } + if c.manualServerURI == nil { + if usePrimary { + c.primaryLock.Lock() + c.primaryURI = nil + c.primaryLock.Unlock() + } else { + c.logger.Printf("removing host (%s) due to '%v'\n", host.Normalize(), err) + c.cluster.RemoveHost(host) + } + } + } + + if err != nil { + err = errors.Wrap(err, ErrTriedMaxHosts.Error()) + } + + return status, body, err +} + +// host returns the first URI that applies, in this order: +// - a non-nil manualServerURI +// - primary URI (if usePrimary = true) +// - the next host from the node list (round-robin) +func (c *Client) host(usePrimary bool) (*pnet.URI, error) { + if c.manualServerURI != nil { + return c.manualServerURI, nil + } + var host *pnet.URI + if usePrimary { + c.primaryLock.RLock() + host = c.primaryURI + c.primaryLock.RUnlock() + if host == nil { + c.primaryLock.Lock() + if c.primaryURI == nil { + node, err := c.fetchPrimaryNode() + if err != nil { + c.primaryLock.Unlock() + return nil, errors.Wrap(err, "fetching primary node") + } + if host, err = pnet.NewURIFromAddress(fmt.Sprintf("%s://%s:%d", node.Scheme, node.Host, node.Port)); err != nil { + return nil, errors.Wrap(err, "parsing primary node URL") + } + } else { + host = c.primaryURI + } + c.primaryURI = host + c.primaryLock.Unlock() + } + } else { + // get a host from the cluster + host = c.cluster.Host() + if host == nil { + return nil, ErrEmptyCluster + } + } + return host, nil +} + +// doRequest creates and performs an http request. +func (c *Client) doRequest(host *pnet.URI, method, path string, headers map[string]string, data []byte) (int, []byte, error) { + var ( + req *http.Request + resp *http.Response + err error + sleepTime time.Duration + rand = rand.New(rand.NewSource(time.Now().UnixNano())) + ) + + for retry := 0; ; { + if req, err = buildRequest(host, method, path, headers, data); err != nil { + return 0, nil, errors.Wrap(err, "building request") + } + if resp, err = c.client.Do(req); err != nil { + return 0, nil, errors.Wrap(err, "sending request") + } + if warning := resp.Header.Get("warning"); warning != "" { + c.logger.Println(warning) + } + + buf := bytes.NewBuffer(make([]byte, 0, 1+resp.ContentLength)) + _, err = buf.ReadFrom(resp.Body) + _ = resp.Body.Close() + if err != nil { + return resp.StatusCode, nil, errors.Wrap(err, "reading response body") + } + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + // [200, 300): OK + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 409: + // 409 Conflict + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 412: + // 412 Precondition Failed + return resp.StatusCode, buf.Bytes(), nil + + case resp.StatusCode == 429: + // 429 Too Many Requests + // A Retry-After header might be included to this response indicating how long to wait before making a new request. + if ms, _ := strconv.Atoi(resp.Header.Get("Retry-After")); ms > 0 { + sleepTime = time.Duration(ms) * time.Millisecond + } else { + sleepTime = time.Duration(1< 400 && resp.StatusCode < 500: + // Pilosa nodes sometimes return 400, we retry in that case. + // (400, 500): No need to retry in other 4xx cases. + return resp.StatusCode, nil, errors.New(strings.TrimSpace(buf.String())) + + case resp.StatusCode == 503: + // This indicates that Pilosa is not ready to service this request, + // typically during startup. In this case, it's ok to give Pilosa + // some time and try again. + sleepTime = time.Duration(1<= c.maxRetries { + // If the error here is nil, we still want to return an error because + // we've hit the max retries limit. If an error exists, wrap it. + errMsg := fmt.Sprintf("max retries (%d) exceeded", c.maxRetries) + if err == nil { + return resp.StatusCode, nil, errors.New(errMsg) + } + return resp.StatusCode, nil, errors.Wrap(err, errMsg) + } + // The client can continue retrying after it has reached the maxBackoff time. + if sleepTime > c.maxBackoff { + return resp.StatusCode, nil, errors.Wrapf(err, "max backoff (%s) time exceeded", c.maxBackoff) + } + retry++ + c.logger.Printf("request failed with: '%v' status: %d, retrying %d after %v ", err, resp.StatusCode, retry, sleepTime) + time.Sleep(sleepTime) + } + // Unreachable code +} + +// statusToNodeShardsForIndex finds the hosts which contains shards for the given index +func (c *Client) statusToNodeShardsForIndex(status Status, indexName string) (map[uint64]*pnet.URI, error) { + result := make(map[uint64]*pnet.URI) + if maxShard, ok := status.indexMaxShard[indexName]; ok { + for shard := 0; shard <= int(maxShard); shard++ { + fragmentNodes, err := c.fetchFragmentNodes(indexName, uint64(shard)) + if err != nil { + return nil, err + } + if len(fragmentNodes) == 0 { + return nil, ErrNoFragmentNodes + } + node := fragmentNodes[0] + uri := &pnet.URI{ + Host: node.Host, + Port: node.Port, + Scheme: node.Scheme, + } + + result[uint64(shard)] = uri + } + } else { + return nil, ErrNoShard + } + return result, nil +} + +func (c *Client) augmentHeaders(headers map[string]string) map[string]string { + if headers == nil { + headers = map[string]string{} + } + + // TODO: move the following block to NewClient once cluster-resize support branch is merged. + version := strings.TrimPrefix(Version, "v") + + headers["User-Agent"] = fmt.Sprintf("pilosa/client/%s", version) + return headers +} + +func (c *Client) TranslateRowKeys(field *Field, keys []string) ([]uint64, error) { + req := &pb.TranslateKeysRequest{ + Index: field.index.name, + Field: field.name, + Keys: keys, + } + return c.translateKeys(req) +} + +func (c *Client) TranslateColumnKeys(index *Index, keys []string) ([]uint64, error) { + // If a manual server URI override has been provided, there's no + // point in partitioning the translation request on the client + // because every request is going to be sent to the manual URI. + if c.manualServerURI != nil { + req := &pb.TranslateKeysRequest{ + Index: index.name, + Keys: keys, + } + return c.translateKeys(req) + } + + // Get the list of hosts from the server. + // TODO: it's not ideal to request the list of nodes from the server + // on every call to TranslateColumnKeys(), but if we cache that list + // on the client, we risk calculating the partition distribution based + // on a stale node list. This TODO is here to indicate that we may, + // in the future, want to remove the overhead of this status request. + status, err := c.Status() + if err != nil { + return nil, errors.Wrap(err, "getting cluster status") + } + + hosts := make([]pnet.URI, len(status.Nodes)) + for i, node := range status.Nodes { + hosts[i] = node.URI.URI() + } + + keysByNode := make(map[pnet.URI][]string, len(hosts)) + for _, key := range keys { + // TODO 256 is DefaultPartitionN in Pilosa. Eventually this will likely be an index configuration parameter. + partitionID := keyPartition(index.Name(), key, 256) + uri := c.partitionOwner(partitionID, hosts) + keysByNode[uri] = append(keysByNode[uri], key) + } + + eg := errgroup.Group{} + idsByNode := make(map[pnet.URI][]uint64, len(keysByNode)) + ibnLock := &sync.Mutex{} + for uri, keys := range keysByNode { + uri := uri + keys := keys + eg.Go(func() error { + req := &pb.TranslateKeysRequest{ + Index: index.name, + Keys: keys, + } + ids, err := c.translateKeys(req, uri) + if err != nil { + return errors.Wrapf(err, "translating column keys at %v", uri) + } + ibnLock.Lock() + idsByNode[uri] = ids + ibnLock.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + + finalIDs := make([]uint64, len(keys)) + // put the ids back together into one slice + for uri, uriIds := range idsByNode { + uriKeys := keysByNode[uri] + kidx := 0 + for i, key := range uriKeys { + for ; keys[kidx] != key; kidx++ { + } + finalIDs[kidx] = uriIds[i] + kidx++ + } + } + return finalIDs, nil +} + +func (c *Client) partitionOwner(partitionID int, hosts []pnet.URI) pnet.URI { + nodeIndex := c.Hasher.Hash(uint64(partitionID), len(hosts)) + return hosts[nodeIndex] +} + +// Hasher represents an interface to hash integers into buckets. +type Hasher interface { + // Hashes the key into a number between [0,N). + Hash(key uint64, n int) int +} + +// jmphasher represents an implementation of jmphash. Implements Hasher. +type jmphasher struct{} + +// Hash returns the integer hash for the given key. +func (h *jmphasher) Hash(key uint64, n int) int { + b, j := int64(-1), int64(0) + for j < int64(n) { + b = j + key = key*uint64(2862933555777941757) + 1 + j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) + } + return int(b) +} + +func keyPartition(index, key string, partitionN int) int { + // Hash the bytes and mod by partition count. + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write([]byte(key)) + return int(h.Sum64() % uint64(partitionN)) +} + +func (c *Client) translateKeys(req *pb.TranslateKeysRequest, uris ...pnet.URI) ([]uint64, error) { + if len(req.Keys) == 0 { + return []uint64{}, nil + } + reqData, err := proto.Marshal(req) + if err != nil { + return nil, errors.Wrap(err, "marshalling translate keys request") + } + + var respData []byte + if len(uris) == 0 { + if _, respData, err = c.httpRequest("POST", "/internal/translate/keys", reqData, defaultProtobufHeaders(), true); err != nil { + return nil, err + } + } else { + if _, respData, err = c.doRequest(&uris[0], "POST", "/internal/translate/keys", defaultProtobufHeaders(), reqData); err != nil { + return nil, errors.Wrapf(err, "reading response body of /internal/translate/keys request to %v", uris[0]) + } + } + + idsResp := &pb.TranslateKeysResponse{} + err = proto.Unmarshal(respData, idsResp) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling traslate keys response") + } + return idsResp.IDs, nil +} + +type TransactionResponse struct { + Transaction *pilosa.Transaction `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` +} + +// StartTransaction tries to start a new transaction in Pilosa. It +// will continue trying until at least requestTimeout time has +// passed. If it fails due to an exclusive transaction already +// existing, it will return that transaction along with a non-nil +// error. +func (c *Client) StartTransaction(id string, timeout time.Duration, exclusive bool, requestTimeout time.Duration) (*pilosa.Transaction, error) { + return c.startTransaction(id, timeout, exclusive, time.Now().Add(requestTimeout)) +} + +func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bool, deadline time.Time) (*pilosa.Transaction, error) { + trns := pilosa.Transaction{ + ID: id, + Timeout: timeout, + Exclusive: exclusive, + } + bod, err := json.Marshal(&trns) + if err != nil { + return nil, errors.Wrap(err, "marshalling transaction") + } + + status, data, err := c.httpRequest("POST", "/transaction", bod, defaultJSONHeaders(), true) + if status == http.StatusConflict && time.Now().Before(deadline) { + // if we're getting StatusConflict after all the usual timeouts/retries, keep retrying until the deadline + time.Sleep(time.Second) + return c.startTransaction(id, timeout, exclusive, deadline) + } + if err != nil { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) { + _, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, defaultJSONHeaders(), true) + if err != nil && len(data) == 0 { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) { + _, respData, err := c.httpRequest("GET", "/transactions", nil, defaultJSONHeaders(), true) + if err != nil { + return nil, errors.Wrap(err, "getting transactions") + } + + trnsMap := make(map[string]*pilosa.Transaction) + err = json.Unmarshal(respData, &trnsMap) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling transactions") + } + return trnsMap, nil +} + +func (c *Client) GetTransaction(id string) (*pilosa.Transaction, error) { + _, data, err := c.httpRequest("GET", "/transaction/"+id, nil, defaultJSONHeaders(), true) + if err != nil { + return nil, err + } + + tr := &TransactionResponse{} + uerr := json.Unmarshal(data, &tr) + if uerr != nil { + if err != nil { + return nil, errors.Wrap(err, "unmarshal failed after") + } + return nil, errors.Wrap(uerr, "couldn't decode body") + } + + if tr.Error != "" { + err = errors.New(tr.Error) + } + + return tr.Transaction, err +} + +func defaultProtobufHeaders() map[string]string { + return map[string]string{ + "Content-Type": "application/x-protobuf", + "Accept": "application/x-protobuf", + "PQL-Version": PQLVersion, + } +} + +func defaultJSONHeaders() map[string]string { + return map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + "PQL-Version": PQLVersion, + } +} + +func buildRequest(host *pnet.URI, method, path string, headers map[string]string, data []byte) (*http.Request, error) { + request, err := http.NewRequest(method, host.Normalize()+path, bytes.NewReader(data)) + if err != nil { + return nil, err + } + + for k, v := range headers { + request.Header.Set(k, v) + } + + return request, nil +} + +func newHTTPClient(options *ClientOptions) *http.Client { + transport := &http.Transport{ + Dial: (&net.Dialer{ + Timeout: options.ConnectTimeout, + }).Dial, + TLSClientConfig: options.TLSConfig, + MaxIdleConnsPerHost: options.PoolSizePerRoute, + MaxIdleConns: options.TotalPoolSize, + } + return &http.Client{ + Transport: transport, + Timeout: options.SocketTimeout, + } +} + +func makeRequestData(query string, options *QueryOptions) ([]byte, error) { + request := &pb.QueryRequest{ + Query: query, + Shards: options.Shards, + ColumnAttrs: options.ColumnAttrs, + ExcludeRowAttrs: options.ExcludeRowAttrs, + ExcludeColumns: options.ExcludeColumns, + } + r, err := proto.Marshal(request) + if err != nil { + return nil, errors.Wrap(err, "marshaling request to protobuf") + } + return r, nil +} + +func makeRoaringImportPath(field *Field, shard uint64, params url.Values) string { + return fmt.Sprintf("/index/%s/field/%s/import-roaring/%d?%s", + field.index.name, field.name, shard, params.Encode()) +} + +type viewImports map[string]*roaring.Bitmap + +// ClientOptions control the properties of client connection to the server. +type ClientOptions struct { + SocketTimeout time.Duration + ConnectTimeout time.Duration + PoolSizePerRoute int + TotalPoolSize int + TLSConfig *tls.Config + manualServerAddress bool + tracer opentracing.Tracer + retries *int + stats stats.StatsClient + nat map[pnet.URI]pnet.URI +} + +func (co *ClientOptions) addOptions(options ...ClientOption) error { + for _, option := range options { + err := option(co) + if err != nil { + return err + } + } + return nil +} + +// ClientOption is used when creating a PilosaClient struct. +type ClientOption func(options *ClientOptions) error + +// OptClientSocketTimeout is the maximum idle socket time in nanoseconds +func OptClientSocketTimeout(timeout time.Duration) ClientOption { + return func(options *ClientOptions) error { + options.SocketTimeout = timeout + return nil + } +} + +// OptClientConnectTimeout is the maximum time to connect in nanoseconds. +func OptClientConnectTimeout(timeout time.Duration) ClientOption { + return func(options *ClientOptions) error { + options.ConnectTimeout = timeout + return nil + } +} + +// OptClientPoolSizePerRoute is the maximum number of active connections in the pool to a host. +func OptClientPoolSizePerRoute(size int) ClientOption { + return func(options *ClientOptions) error { + options.PoolSizePerRoute = size + return nil + } +} + +// OptClientTotalPoolSize is the maximum number of connections in the pool. +func OptClientTotalPoolSize(size int) ClientOption { + return func(options *ClientOptions) error { + options.TotalPoolSize = size + return nil + } +} + +// OptClientTLSConfig contains the TLS configuration. +func OptClientTLSConfig(config *tls.Config) ClientOption { + return func(options *ClientOptions) error { + options.TLSConfig = config + return nil + } +} + +// OptClientManualServerAddress forces the client use only the manual server address +func OptClientManualServerAddress(enabled bool) ClientOption { + return func(options *ClientOptions) error { + options.manualServerAddress = enabled + return nil + } +} + +// OptClientTracer sets the Open Tracing tracer +// See: https://opentracing.io +func OptClientTracer(tracer opentracing.Tracer) ClientOption { + return func(options *ClientOptions) error { + options.tracer = tracer + return nil + } +} + +// OptClientRetries sets the number of retries on HTTP request failures. +func OptClientRetries(retries int) ClientOption { + return func(options *ClientOptions) error { + if retries < 0 { + return errors.New("retries must be non-negative") + } + options.retries = &retries + return nil + } +} + +// OptClientStatsClient sets a stats client, such as Prometheus +func OptClientStatsClient(stats stats.StatsClient) ClientOption { + return func(options *ClientOptions) error { + options.stats = stats + return nil + } +} + +// OptClientNAT sets a NAT map used to translate the advertised URI to something +// else (for example, when accessing pilosa running in docker). +func OptClientNAT(nat map[string]string) ClientOption { + return func(options *ClientOptions) error { + // covert the strings to URIs + m := make(map[pnet.URI]pnet.URI) + for k, v := range nat { + if kuri, err := pnet.NewURIFromAddress(k); err != nil { + return errors.Wrapf(err, "converting string to URI: %s", k) + } else if vuri, err := pnet.NewURIFromAddress(v); err != nil { + return errors.Wrapf(err, "converting string to URI: %s", v) + } else { + m[*kuri] = *vuri + } + } + options.nat = m + return nil + } +} + +func (co *ClientOptions) withDefaults() (updated *ClientOptions) { + // copy options so the original is not updated + updated = &ClientOptions{} + *updated = *co + // impose defaults + if updated.SocketTimeout <= 0 { + updated.SocketTimeout = time.Second * 300 + } + if updated.ConnectTimeout <= 0 { + updated.ConnectTimeout = time.Second * 60 + } + if updated.PoolSizePerRoute <= 0 { + updated.PoolSizePerRoute = 50 + } + if updated.TotalPoolSize <= 0 { + updated.TotalPoolSize = 500 + } + if updated.TLSConfig == nil { + updated.TLSConfig = &tls.Config{} + } + if updated.retries == nil { + retries := 2 + updated.retries = &retries + } + return +} + +// QueryOptions contains options to customize the Query function. +type QueryOptions struct { + // Shards restricts query to a subset of shards. Queries all shards if nil. + Shards []uint64 + // ColumnAttrs enables returning columns in the query response. + ColumnAttrs bool + // ExcludeRowAttrs inhibits returning attributes + ExcludeRowAttrs bool + // ExcludeColumns inhibits returning columns + ExcludeColumns bool +} + +func (qo *QueryOptions) addOptions(options ...interface{}) error { + for i, option := range options { + switch o := option.(type) { + case nil: + if i != 0 { + return ErrInvalidQueryOption + } + continue + case *QueryOptions: + if i != 0 { + return ErrInvalidQueryOption + } + *qo = *o + case QueryOption: + err := o(qo) + if err != nil { + return err + } + default: + return ErrInvalidQueryOption + } + } + return nil +} + +// QueryOption is used when using options with a client.Query, +type QueryOption func(options *QueryOptions) error + +// OptQueryColumnAttrs enables returning column attributes in the result. +func OptQueryColumnAttrs(enable bool) QueryOption { + return func(options *QueryOptions) error { + options.ColumnAttrs = enable + return nil + } +} + +// OptQueryShards restricts the set of shards on which a query operates. +func OptQueryShards(shards ...uint64) QueryOption { + return func(options *QueryOptions) error { + options.Shards = append(options.Shards, shards...) + return nil + } +} + +// OptQueryExcludeAttrs enables discarding attributes from a result, +func OptQueryExcludeAttrs(enable bool) QueryOption { + return func(options *QueryOptions) error { + options.ExcludeRowAttrs = enable + return nil + } +} + +// OptQueryExcludeColumns enables discarding columns from a result, +func OptQueryExcludeColumns(enable bool) QueryOption { + return func(options *QueryOptions) error { + options.ExcludeColumns = enable + return nil + } +} + +// ImportOptions are the options for controlling the importer +type ImportOptions struct { + threadCount int + batchSize int + wantRoaring *bool + clear bool + skipSort bool +} + +// ImportOption is used when running imports. +type ImportOption func(options *ImportOptions) error + +// OptImportThreadCount is the number of goroutines allocated for import. +func OptImportThreadCount(count int) ImportOption { + return func(options *ImportOptions) error { + options.threadCount = count + return nil + } +} + +// OptImportBatchSize is the number of records read before importing them. +func OptImportBatchSize(batchSize int) ImportOption { + return func(options *ImportOptions) error { + options.batchSize = batchSize + return nil + } +} + +// OptImportClear sets clear import, which clears bits instead of setting them. +func OptImportClear(clear bool) ImportOption { + return func(options *ImportOptions) error { + options.clear = clear + return nil + } +} + +// OptImportRoaring enables importing using roaring bitmaps which is more performant. +func OptImportRoaring(enable bool) ImportOption { + return func(options *ImportOptions) error { + options.wantRoaring = &enable + return nil + } +} + +// OptImportSort tells the importer whether or not to sort batches of records, on +// by default. Sorting imposes some performance cost, especially on data that's +// already sorted, but dramatically improves performance in pathological +// cases. It is enabled by default because the pathological cases are awful, +// and the performance hit is comparatively small, but the performance cost can +// be significant if you know your data is sorted. +func OptImportSort(sorting bool) ImportOption { + return func(options *ImportOptions) error { + // skipSort is expressed negatively because we want to + // keep sorting enabled by default, so the zero value should + // be that default behavior. The client option expresses it + // positively because that's easier for API users. + options.skipSort = !sorting + return nil + } +} + +type fragmentNodeRoot struct { + URI fragmentNode `json:"uri"` +} + +type fragmentNode struct { + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` +} + +func newFragmentNodeFromURI(uri *pnet.URI) fragmentNode { + return fragmentNode{ + Scheme: uri.Scheme, + Host: uri.Host, + Port: uri.Port, + } +} + +func (node fragmentNode) URI() *pnet.URI { + return &pnet.URI{ + Scheme: node.Scheme, + Host: node.Host, + Port: node.Port, + } +} + +// Info contains the configuration/host information from a Pilosa server. +type Info struct { + ShardWidth uint64 `json:"shardWidth"` // width of each shard + Memory uint64 `json:"memory"` // approximate host physical memory + CPUType string `json:"cpuType"` // "brand name string" from cpuid + CPUPhysicalCores int `json:"CPUPhysicalCores"` // physical cores (cpuid) + CPULogicalCores int `json:"CPULogicalCores"` // logical cores cpuid + CPUMHz uint64 `json:"CPUMHz"` // estimated clock speed +} + +// Status contains the status information from a Pilosa server. +type Status struct { + Nodes []StatusNode `json:"nodes"` + State string `json:"state"` + LocalID string `json:"localID"` + indexMaxShard map[string]uint64 +} + +// StatusNode contains information about a node in the cluster. +type StatusNode struct { + ID string `json:"id"` + URI StatusURI `json:"uri"` + IsPrimary bool `json:"isPrimary"` +} + +// StatusURI contains node information. +type StatusURI struct { + Scheme string `json:"scheme"` + Host string `json:"host"` + Port uint16 `json:"port"` +} + +// URI returns the StatusURI as a URI. +func (s StatusURI) URI() pnet.URI { + return pnet.URI{ + Scheme: s.Scheme, + Host: s.Host, + Port: s.Port, + } +} + +// SchemaInfo contains the indexes. +type SchemaInfo struct { + Indexes []SchemaIndex `json:"indexes"` +} + +// SchemaIndex contains index information. +type SchemaIndex struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options SchemaOptions `json:"options"` + Fields []SchemaField `json:"fields"` + Shards []uint64 `json:"shards"` + ShardWidth uint64 `json:"shardWidth"` +} + +// SchemaField contains field information. +type SchemaField struct { + Name string `json:"name"` + CreatedAt int64 `json:"createdAt,omitempty"` + Options SchemaOptions `json:"options"` +} + +// SchemaOptions contains options for a field or an index. +type SchemaOptions struct { + FieldType FieldType `json:"type"` + CacheType string `json:"cacheType"` + CacheSize uint `json:"cacheSize"` + TimeQuantum string `json:"timeQuantum"` + Min pql.Decimal `json:"min"` + Max pql.Decimal `json:"max"` + Scale int64 `json:"scale"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + TrackExistence bool `json:"trackExistence"` +} + +func (so SchemaOptions) asIndexOptions() *IndexOptions { + return &IndexOptions{ + keys: so.Keys, + keysSet: true, + trackExistence: so.TrackExistence, + trackExistenceSet: true, + } +} + +func (so SchemaOptions) asFieldOptions() *FieldOptions { + return &FieldOptions{ + fieldType: so.FieldType, + cacheSize: int(so.CacheSize), + cacheType: CacheType(so.CacheType), + timeQuantum: TimeQuantum(so.TimeQuantum), + min: so.Min, + max: so.Max, + scale: so.Scale, + keys: so.Keys, + noStandardView: so.NoStandardView, + } +} + +type exportReader struct { + client *Client + shardURIs map[uint64]*pnet.URI + field *Field + body []byte + bodyIndex int + currentShard uint64 + shardCount uint64 +} + +func newExportReader(client *Client, shardURIs map[uint64]*pnet.URI, field *Field) *exportReader { + return &exportReader{ + client: client, + shardURIs: shardURIs, + field: field, + shardCount: uint64(len(shardURIs)), + } +} + +// Read updates the passed array with the exported CSV data and returns the number of bytes read +func (r *exportReader) Read(p []byte) (n int, err error) { + if r.currentShard >= r.shardCount { + err = io.EOF + return + } + if r.body == nil { + uri := r.shardURIs[r.currentShard] + headers := map[string]string{ + "Accept": "text/csv", + } + path := fmt.Sprintf("/export?index=%s&field=%s&shard=%d", + r.field.index.Name(), r.field.Name(), r.currentShard) + _, respData, err := r.client.doRequest(uri, "GET", path, headers, nil) + if err != nil { + return 0, errors.Wrap(err, "doing export request") + } + r.body = respData + r.bodyIndex = 0 + } + n = copy(p, r.body[r.bodyIndex:]) + r.bodyIndex += n + if n >= len(r.body) { + r.body = nil + r.currentShard++ + } + return +} diff --git a/client/client_it_test.go b/client/client_it_test.go new file mode 100644 index 000000000..5fda05def --- /dev/null +++ b/client/client_it_test.go @@ -0,0 +1,816 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/pilosa/pilosa/v2/disco" + pnet "github.com/pilosa/pilosa/v2/net" + "github.com/pilosa/pilosa/v2/shardwidth" + "github.com/pilosa/pilosa/v2/test" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +var ( + testIndex *Index + testIndexWithKeys *Index + testIndexWithKeysNoTrack *Index + testIndexAtomicRecord *Index + + testField *Field + testFieldTimeQuantum *Field + testFieldInt0 *Field + testFieldInt1 *Field +) + +func setup(t *testing.T, r *require.Assertions, cli *Client) { + t.Helper() + + testSchema := NewSchema() + testIndex = testSchema.Index("test-index") + testIndexWithKeys = testSchema.Index("test-index-keys", OptIndexKeys(true)) + testIndexWithKeysNoTrack = testSchema.Index("test-index-keys-notrack", + OptIndexKeys(true), + OptIndexTrackExistence(false), + ) + testField = testIndex.Field("test-field") + testFieldTimeQuantum = testIndex.Field("test-field-timequantum", OptFieldTypeTime(TimeQuantumYear)) + + testIndexAtomicRecord = testSchema.Index("test-index-atomic-record") + testFieldInt0 = testIndexAtomicRecord.Field("test-field-int0", OptFieldTypeInt(-1000, 1000)) + testFieldInt1 = testIndexAtomicRecord.Field("test-field-int1", OptFieldTypeInt(-1000, 1000)) + + r.NoErrorf(cli.SyncSchema(testSchema), "SyncSchema") +} + +func tearDown(t *testing.T, r *require.Assertions, cli *Client) { + t.Helper() + + for _, i := range []*Index{testIndex, testIndexWithKeys, testIndexWithKeysNoTrack, testIndexAtomicRecord} { + r.NoErrorf(cli.DeleteIndex(i), "DeleteIndex(%s)", i.name) + } +} + +func TestClientAgainstCluster(t *testing.T) { + require := require.New(t) + + for size, replicaN := 3, 1; replicaN <= 2; replicaN++ { + testName := fmt.Sprintf("%d.%d", size, replicaN) + t.Run(testName, func(t *testing.T) { + + // Start size.replicaN cluster + c := test.MustNewCluster(t, size) + for _, n := range c.Nodes { + n.Config.Cluster.ReplicaN = replicaN + } + err := c.Start() + require.NoError(err, "Start cluster "+testName) + + urls := make([]string, len(c.Nodes)) + for i, n := range c.Nodes { + urls[i] = n.URL() + } + defer c.Close() + + // Create a new client for the cluster + cli, err := newClientFromAddresses(urls, &ClientOptions{}) + require.NoErrorf(err, "newClientFromAddresses(%v): %v", urls, err) + defer cli.Close() + + t.Run("GetStatus", func(t *testing.T) { + status, err := cli.Status() + require.NoErrorf(err, "GET /status") + + require.Equalf(disco.ClusterStateNormal, disco.ClusterState(status.State), "GET /status") + }) + + t.Run("QueryRow", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + resp, err := cli.Query(testField.Row(1)) + require.NoErrorf(err, "Query Row") + require.NotNil(resp, "Response should not be nil") + }) + + t.Run("QueryWithShards", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + shardWidth := uint64(1 << shardwidth.Exponent) + + _, err := cli.Query(testField.Set(1, 1)) + require.NoErrorf(err, "Set(1, %d)", 1) + + _, err = cli.Query(testField.Set(1, shardWidth)) + require.NoErrorf(err, "Set(1, %d)", shardWidth) + + _, err = cli.Query(testField.Set(1, shardWidth*3)) + require.NoErrorf(err, "Set(1, %d)", shardWidth*3) + + resp, err := cli.Query(testField.Row(1), OptQueryShards(0, 3)) + require.NoErrorf(err, "Row(1) OptQueryShards(0, 3)") + + cols := resp.Result().Row().Columns + require.Equalf([]uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols) + }) + + t.Run("QueryWithColumns", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + targetAttrs := map[string]interface{}{ + "name": "some string", + "age": int64(95), + "registered": true, + "height": 1.83, + } + _, err := cli.Query(testField.Set(1, 100)) + require.NoErrorf(err, "Set(1, 100)") + + resp, err := cli.Query(testIndex.SetColumnAttrs(100, targetAttrs)) + require.NoErrorf(err, "SetColumnAttrs(100, %v)", targetAttrs) + require.Equalf(ColumnItem{}, resp.Column(), "No columns should be returned if it wasn't explicitly requested") + + resp, err = cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true}) + require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}") + require.Equalf(1, len(resp.ColumnAttrs()), "ColumnAttrs count should be == 1") + + cols := resp.Columns() + require.Equalf(1, len(cols), "Column count") + require.Equalf(uint64(100), cols[0].ID, "Column ID") + + require.Equalf(targetAttrs, cols[0].Attributes, "Column attrs.") + + require.Equalf(cols[0], resp.Column(), "Column() should be equivalent to first column in the response") + }) + + t.Run("SetRowAttrs", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + targetAttrs := map[string]interface{}{ + "name": "some string", + "age": int64(95), + "registered": true, + "height": 1.83, + } + + _, err := cli.Query(testField.Set(1, 100)) + require.NoErrorf(err, "Set(1, 100)") + + _, err = cli.Query(testField.SetRowAttrs(1, targetAttrs)) + require.NoErrorf(err, "SetRowAttrs(1, %v)", targetAttrs) + + resp, err := cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true}) + require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}") + + require.Equalf(targetAttrs, resp.Result().Row().Attributes, "Row attributes should be set") + }) + + t.Run("OrmCount", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldCount := testIndex.Field("test-field-count") + err := cli.EnsureField(testFieldCount) + require.NoError(err) + + qry := testIndex.BatchQuery( + testFieldCount.Set(10, 20), + testFieldCount.Set(10, 21), + testFieldCount.Set(15, 25), + ) + _, err = cli.Query(qry) + require.NoErrorf(err, "BatchQuery") + + resp, err := cli.Query(testIndex.Count(testFieldCount.Row(10))) + require.NoErrorf(err, "Count") + require.Equalf(int64(2), resp.Result().Count(), "Count") + }) + + t.Run("DecimalField", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldDec := testIndex.Field("test-field-dec", OptFieldTypeDecimal(3)) + err := cli.EnsureField(testFieldDec) + require.NoError(err) + + sch, err := cli.Schema() + require.NoErrorf(err, "Schema") + + idx := sch.indexes[testIndex.name] + opts := idx.Field(testFieldDec.name).Options() + require.Equalf(int64(3), opts.scale, "%s scale", testFieldDec.name) + }) + + t.Run("IntersectReturns", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldSegments := testIndex.Field("test-field-segments") + err := cli.EnsureField(testFieldSegments) + require.NoError(err) + + qry1 := testIndex.BatchQuery( + testFieldSegments.Set(2, 10), + testFieldSegments.Set(2, 15), + testFieldSegments.Set(3, 10), + testFieldSegments.Set(3, 20), + ) + _, err = cli.Query(qry1) + require.NoErrorf(err, "BatchQuery") + + qry2 := testIndex.Intersect(testFieldSegments.Row(2), testFieldSegments.Row(3)) + resp, err := cli.Query(qry2) + require.NoErrorf(err, "Intersect") + + require.Equalf(1, len(resp.Results()), "Intersect number of results") + require.Equalf([]uint64{10}, resp.Result().Row().Columns, "Intersect columns results") + }) + + t.Run("TopNReturns", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldTopN := testIndex.Field("test-field-topn") + err := cli.EnsureField(testFieldTopN) + require.NoError(err) + + qry := testIndex.BatchQuery( + testFieldTopN.Set(10, 5), + testFieldTopN.Set(10, 10), + testFieldTopN.Set(10, 15), + testFieldTopN.Set(20, 5), + testFieldTopN.Set(30, 5), + ) + _, err = cli.Query(qry) + require.NoErrorf(err, "BatchQuery") + + // XXX: The following is required to make this test pass. See: https://github.com/pilosa/pilosa/issues/625 + _, _, err = cli.HTTPRequest("POST", "/recalculate-caches", nil, nil) + require.NoErrorf(err, "POST /recalculate-caches") + + resp, err := cli.Query(testFieldTopN.TopN(2)) + require.NoErrorf(err, "TopN(2)") + + items := resp.Result().CountItems() + require.Equalf(2, len(items), "TopN result CountItems") + + item := items[0] + require.Equalf(uint64(10), item.ID, "TopN result item[0].ID") + require.Equalf(uint64(3), item.Count, "TopN result item[0].Count") + + _, err = cli.Query(testFieldTopN.SetRowAttrs(10, map[string]interface{}{"foo": "bar"})) + require.NoErrorf(err, "SetRowAttrs(10)") + + resp, err = cli.Query(testFieldTopN.FilterAttrTopN(5, nil, "foo", "bar")) + require.NoErrorf(err, `FilterAttrTopN(5, nil, "foo", "bar")`) + + items = resp.Result().CountItems() + require.Equalf(1, len(items), "FilterAttrTopN result CountItems") + + item = items[0] + require.Equalf(uint64(10), item.ID, "FilterAttrTopN result item[0].ID") + require.Equalf(uint64(3), item.Count, "FilterAttrTopN result item[0].Count") + }) + + t.Run("MinMaxRow", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldMinMax := testIndex.Field("test-field-minmax") + err := cli.EnsureField(testFieldMinMax) + require.NoError(err) + + qry := testIndex.BatchQuery( + testFieldMinMax.Set(10, 5), + testFieldMinMax.Set(10, 10), + testFieldMinMax.Set(10, 15), + testFieldMinMax.Set(20, 5), + testFieldMinMax.Set(30, 5), + ) + _, err = cli.Query(qry) + require.NoErrorf(err, "Setting bits") + + resp, err := cli.Query(testFieldMinMax.MinRow()) + require.NoErrorf(err, "MinRow") + + min := resp.Result().CountItem().ID + require.Equalf(uint64(10), min, "Min") + + resp, err = cli.Query(testFieldMinMax.MaxRow()) + require.NoErrorf(err, "MaxRow") + + max := resp.Result().CountItem().ID + require.Equalf(uint64(30), max, "Max") + }) + + t.Run("SetMutexField", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldMutex := testIndex.Field("test-field-mutex", OptFieldTypeMutex(CacheTypeDefault, 0)) + err := cli.EnsureField(testFieldMutex) + require.NoError(err) + + // can set mutex + _, err = cli.Query(testFieldMutex.Set(1, 100)) + require.NoErrorf(err, "Set(1, 100)") + + resp, err := cli.Query(testFieldMutex.Row(1)) + require.NoErrorf(err, "Row(1)") + + target := []uint64{100} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + + // setting another row removes the previous + _, err = cli.Query(testFieldMutex.Set(42, 100)) + require.NoErrorf(err, "Set(42, 100)") + + resp, err = cli.Query(testIndex.BatchQuery( + testFieldMutex.Row(1), + testFieldMutex.Row(42), + )) + require.NoErrorf(err, "BatchQuery") + + target1 := []uint64(nil) + target42 := []uint64{100} + require.Equalf(target1, resp.Results()[0].Row().Columns, "Row Results[0] Columns") + require.Equalf(target42, resp.Results()[1].Row().Columns, "Row Results[1] Columns") + }) + + t.Run("SetBoolField", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldBool := testIndex.Field("test-field-bool", OptFieldTypeBool()) + err := cli.EnsureField(testFieldBool) + require.NoError(err) + + // can set bool + _, err = cli.Query(testFieldBool.Set(true, 100)) + require.NoErrorf(err, "Set(true, 100)") + + resp, err := cli.Query(testFieldBool.Row(true)) + require.NoErrorf(err, "Row(true)") + + target := []uint64{100} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("ClearRowQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldClear := testIndex.Field("test-field-clear") + err := cli.EnsureField(testFieldClear) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldClear.Set(1, 100), + testFieldClear.Set(1, 200), + )) + require.NoErrorf(err, "Set(1, 100) Set(1, 200)") + + resp, err := cli.Query(testFieldClear.Row(1)) + require.NoErrorf(err, "Row(1)") + + target := []uint64{100, 200} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + + _, err = cli.Query(testFieldClear.ClearRow(1)) + require.NoErrorf(err, "ClearRow(1)") + + resp, err = cli.Query(testFieldClear.Row(1)) + require.NoErrorf(err, "Row(1)") + + target = []uint64(nil) + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("RowsQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldRows := testIndex.Field("test-field-rows") + err := cli.EnsureField(testFieldRows) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRows.Set(1, 100), + testFieldRows.Set(1, 200), + testFieldRows.Set(2, 200), + )) + require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testFieldRows.Rows()) + require.NoErrorf(err, "Rows") + + target := RowIdentifiersResult{ + IDs: []uint64{1, 2}, + } + require.Equalf(target, resp.Result().RowIdentifiers(), "RowIdentifiers Result") + }) + + t.Run("UnionRowsQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldRows := testIndex.Field("test-field-rows") + err := cli.EnsureField(testFieldRows) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRows.Set(1, 100), + testFieldRows.Set(1, 200), + testFieldRows.Set(2, 200), + )) + require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testFieldRows.Rows().Union()) + require.NoErrorf(err, "Rows Union") + + target := []uint64{100, 200} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("LikeQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldLike := testIndex.Field("test-field-like", OptFieldKeys(true)) + err := cli.EnsureField(testFieldLike) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldLike.Set("a", 100), + testFieldLike.Set("b", 200), + testFieldLike.Set("bc", 200), + )) + require.NoErrorf(err, "Set(a, 100) Set(b, 200) Set(bc, 200)") + + resp, err := cli.Query(testFieldLike.Like("b%")) + require.NoErrorf(err, `Like(b%)`) + + target := RowIdentifiersResult{ + Keys: []string{"b", "bc"}, + } + require.Equalf(target, resp.Result().RowIdentifiers(), "RowIdentifiers Result") + }) + + t.Run("GroupByQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldGroupBy := testIndex.Field("test-field-group-by") + err := cli.EnsureField(testFieldGroupBy) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldGroupBy.Set(1, 100), + testFieldGroupBy.Set(1, 200), + testFieldGroupBy.Set(2, 200), + )) + require.NoErrorf(err, "Set(1, 100) Set(1, 200) Set(2, 200)") + + resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows())) + require.NoErrorf(err, `Like(b%)`) + + target := []GroupCount{ + {Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "test-field-group-by", RowID: 2}}, Count: 1}, + } + + assertGroupBy(t, require, target, resp.Result().GroupCounts()) + }) + + t.Run("GroupByQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldGroupBy := testIndex.Field("test-field-group-by-int", OptFieldTypeInt(-10, 10)) + err := cli.EnsureField(testFieldGroupBy) + require.NoError(err) + + _, err = cli.Query(testIndex.RawQuery(` + Set(0, test-field-group-by-int=1) + Set(1, test-field-group-by-int=2) + + Set(2, test-field-group-by-int=-2) + Set(3, test-field-group-by-int=-1) + + Set(4, test-field-group-by-int=4) + + Set(10, test-field-group-by-int=0) + Set(100, test-field-group-by-int=0) + Set(1000, test-field-group-by-int=0) + Set(10000, test-field-group-by-int=0) + Set(100000, test-field-group-by-int=0) + `)) + require.NoError(err, "Set(0..100000)") + + resp, err := cli.Query(testIndex.GroupBy(testFieldGroupBy.Rows())) + require.NoErrorf(err, `GroupBy(Rows)`) + + var a, b, c, d, e, f int64 = -2, -1, 0, 1, 2, 4 + target := []GroupCount{ + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &a}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &b}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &c}}, Count: 5}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &d}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &e}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "test-field-group-by-int", Value: &f}}, Count: 1}, + } + assertGroupBy(t, require, target, resp.Result().GroupCounts()) + }) + + t.Run("CreateDeleteIndexField", func(t *testing.T) { + tmpIndex := NewIndex("tmp-index") + tmpField := tmpIndex.Field("tmp-field") + + err := cli.CreateIndex(tmpIndex) + require.NoError(err) + + err = cli.CreateField(tmpField) + require.NoError(err) + + err = cli.DeleteField(tmpField) + require.NoError(err) + + err = cli.DeleteIndex(tmpIndex) + require.NoError(err) + }) + + t.Run("ErrorCreatingIndexField", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + require.ErrorIs(cli.CreateIndex(testIndex), ErrIndexExists) + require.ErrorIs(cli.CreateField(testField), ErrFieldExists) + }) + + t.Run("Failover", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar") + tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0)) + + attrs := map[string]interface{}{"a": 1} + _, err := tmpcli.Query(testIndex.SetColumnAttrs(0, attrs)) + require.Error(err, ErrTriedMaxHosts) + }) + + t.Run("InvalidQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + _, _, err := cli.HTTPRequest("INVALID METHOD", "/foo", nil, nil) + require.Error(err) + + _, err = cli.Query(testIndex.RawQuery("Invalid query")) + require.Error(err) + }) + + t.Run("Sync", func(t *testing.T) { + testIndexRemote := NewIndex("test-index-remote") + err := cli.EnsureIndex(testIndexRemote) + require.NoError(err) + + testFieldRemote := testIndexRemote.Field("test-field-remote") + err = cli.EnsureField(testFieldRemote) + require.NoError(err) + + schema := NewSchema() + idx1 := schema.Index("index-1") + idx1.Field("field-1-1") + idx1.Field("field-1-2") + + idx2 := schema.Index("index-2") + idx2.Field("field-2-1") + schema.Index(testIndexRemote.Name()) + + err = cli.SyncSchema(schema) + require.NoError(err) + + err = cli.DeleteIndex(testIndexRemote) + require.NoError(err) + + err = cli.DeleteIndex(idx1) + require.NoError(err) + + err = cli.DeleteIndex(idx2) + require.NoError(err) + }) + + t.Run("FetchFragmentNodes", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + nodes, err := cli.fetchFragmentNodes(testIndex.Name(), 0) + require.NoErrorf(err, "fetchFragmentNodes(%s, 0)", testIndex.name) + require.Equalf(replicaN, len(nodes), "len(nodes)") + + // running the same for coverage + nodes, err = cli.fetchFragmentNodes(testIndex.Name(), 0) + require.NoErrorf(err, "fetchFragmentNodes(%s, 0)", testIndex.name) + require.Equalf(replicaN, len(nodes), "len(nodes)") + }) + + t.Run("RowRangeQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldRange := testIndex.Field("test-field-range", OptFieldTypeTime(TimeQuantumMonthDayHour)) + err := cli.EnsureField(testFieldRange) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldRange.SetTimestamp(10, 100, time.Date(2017, time.January, 1, 0, 0, 0, 0, time.UTC)), + testFieldRange.SetTimestamp(10, 100, time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)), + testFieldRange.SetTimestamp(10, 100, time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)), + )) + require.NoErrorf(err, "BatchQuery SetTimestamp") + + start := time.Date(2017, time.January, 5, 0, 0, 0, 0, time.UTC) + end := time.Date(2018, time.January, 5, 0, 0, 0, 0, time.UTC) + resp, err := cli.Query(testFieldRange.RowRange(10, start, end)) + require.NoErrorf(err, "RowRange(10, %v, %v)", start, end) + + target := []uint64{100} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("StoreQuery", func(t *testing.T) { + schema := NewSchema() + testIndexStore := schema.Index("test-index-store") + testFieldFrom := testIndexStore.Field("test-field-from") + testFieldTo := testIndexStore.Field("test-field-to") + err := cli.SyncSchema(schema) + require.NoError(err) + + defer func() { + cerr := cli.DeleteIndex(testIndexStore) + require.NoErrorf(cerr, "failed to delete index: %v", testIndexStore.name) + }() + + _, err = cli.Query(testIndexStore.BatchQuery( + testFieldFrom.Set(10, 100), + testFieldFrom.Set(10, 200), + testFieldTo.Store(testFieldFrom.Row(10), 1), + )) + require.NoErrorf(err, "Set(10, 100) Set(10, 200) Store(Row(10), 1)") + + resp, err := cli.Query(testFieldTo.Row(1)) + require.NoErrorf(err, "Row(1)") + + target := []uint64{100, 200} + require.Equalf(target, resp.Result().Row().Columns, "Row Result Columns") + }) + + t.Run("MultipleClientKeyQuery", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldMultiClient := testIndexWithKeys.Field("test-field-multiclient") + err := cli.EnsureField(testFieldMultiClient) + require.NoError(err) + + eg := &errgroup.Group{} + for i := 0; i < 10; i++ { + rowID := uint64(i) + eg.Go(func() error { + _, e := cli.Query(testFieldMultiClient.Set(rowID, "col")) + return e + }) + } + require.NoError(eg.Wait()) + }) + + t.Run("ExportRowIDColumnID", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldExport := testIndex.Field("test-field-export") + err := cli.EnsureField(testFieldExport) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldExport.Set(1, 1), + testFieldExport.Set(1, 10), + testFieldExport.Set(2, 1048577), + ), nil) + require.NoErrorf(err, "Set(1, 1) Set(1, 10) Set(2, 1048577)") + + r, err := cli.ExportField(testFieldExport) + require.NoErrorf(err, "ExportField") + + b, err := ioutil.ReadAll(r) + require.NoError(err) + + target := "1,1\n1,10\n2,1048577\n" + require.Equalf(target, string(b), "Export Field Response") + }) + + t.Run("ExportRowIDColumnKey", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldExport := testIndexWithKeys.Field("test-field-export") + err := cli.EnsureField(testFieldExport) + require.NoError(err) + + _, err = cli.Query(testIndexWithKeys.BatchQuery( + testFieldExport.Set(1, "one"), + testFieldExport.Set(1, "ten"), + testFieldExport.Set(2, "big-number"), + ), nil) + require.NoErrorf(err, "Set(1, one) Set(1, ten) Set(2, big-number)") + + r, err := cli.ExportField(testFieldExport) + require.NoErrorf(err, "ExportField") + + b, err := ioutil.ReadAll(r) + require.NoError(err) + + target := "1,one\n1,ten\n2,big-number\n" + require.Equalf(target, string(b), "Export Field Response") + }) + + t.Run("TranslateRowKeys", func(t *testing.T) { + setup(t, require, cli) + defer tearDown(t, require, cli) + + testFieldTranslate := testIndex.Field("test-field-translate-rowkeys", OptFieldKeys(true)) + err := cli.EnsureField(testFieldTranslate) + require.NoError(err) + + _, err = cli.Query(testIndex.BatchQuery( + testFieldTranslate.Set("key1", 10), + testFieldTranslate.Set("key2", 1000), + )) + require.NoErrorf(err, "Set(key1, 10) Set(key2, 1000)") + + rowIDs, err := cli.TranslateRowKeys(testFieldTranslate, []string{"key1", "key2"}) + require.NoErrorf(err, "TranslateRowKeys") + + target := []uint64{1, 2} + require.Equalf(target, rowIDs, "TranslateRowKeys") + }) + + t.Run("TranslateColKeys", func(t *testing.T) { + trns, err := cli.StartTransaction("blah", time.Minute, false, time.Minute) + require.NoErrorf(err, "StartTransaction(blah)") + require.Equalf("blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(trns.Active, "TranslateColumnKeys Active") + + trnsMap, err := cli.Transactions() + require.NoErrorf(err, "Transactions") + require.Equalf(1, len(trnsMap), "Transactions len") + require.Truef(trnsMap["blah"].Active, "Transactions Active") + + trns, err = cli.GetTransaction("blah") + require.NoErrorf(err, "GetTransaction(blah)") + require.Equalf("blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(trns.Active, "TranslateColumnKeys Active") + + trns, err = cli.FinishTransaction("blah") + require.NoErrorf(err, "FinishTransaction(blah)") + require.Equalf("blah", trns.ID, "TranslateColumnKeys ID") + require.Equalf(time.Minute, trns.Timeout, "TranslateColumnKeys Timeout") + require.Truef(trns.Active, "TranslateColumnKeys Active") + }) + }) + } +} + +func assertGroupBy(t *testing.T, r *require.Assertions, expected, results []GroupCount) { + t.Helper() + + r.Equalf(len(expected), len(results), "number of groupings mismatch") + + for i, result := range results { + r.Equalf(expected[i], result, "unexpected result at %d", i) + } +} diff --git a/client/client_test.go b/client/client_test.go new file mode 100644 index 000000000..0833b7739 --- /dev/null +++ b/client/client_test.go @@ -0,0 +1,293 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "crypto/tls" + "errors" + "reflect" + "testing" + + pnet "github.com/pilosa/pilosa/v2/net" +) + +func TestQueryWithError(t *testing.T) { + var err error + client := DefaultClient() + index := NewIndex("foo") + field := index.Field("foo") + invalid := field.FilterAttrTopN(12, field.Row(7), "$invalid$", 80, 81) + _, err = client.Query(invalid, nil) + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestClientOptions(t *testing.T) { + targets := []*ClientOptions{ + {SocketTimeout: 10}, + {ConnectTimeout: 5}, + {PoolSizePerRoute: 7}, + {TotalPoolSize: 17}, + {TLSConfig: &tls.Config{InsecureSkipVerify: true}}, + } + optionsList := [][]ClientOption{ + {OptClientSocketTimeout(10)}, + {OptClientConnectTimeout(5)}, + {OptClientPoolSizePerRoute(7)}, + {OptClientTotalPoolSize(17)}, + {OptClientTLSConfig(&tls.Config{InsecureSkipVerify: true})}, + } + + for i := 0; i < len(targets); i++ { + options := &ClientOptions{} + err := options.addOptions(optionsList[i]...) + if err != nil { + t.Fatal(err) + } + target := targets[i] + if !reflect.DeepEqual(target, options) { + t.Fatalf("%v != %v", target, options) + } + } +} + +func TestNewClientWithErrorredOption(t *testing.T) { + _, err := NewClient(":8888", ClientOptionErr(0)) + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestNewClient(t *testing.T) { + client, err := NewClient(":9999", OptClientManualServerAddress(true)) + if err != nil { + t.Fatal(err) + } + targetURI, err := pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(targetURI, client.manualServerURI) { + t.Fatalf("%v != %v", targetURI, client.manualServerURI) + } + targetFragmentNode := &fragmentNode{ + Scheme: "http", + Host: "localhost", + Port: 9999, + } + if !reflect.DeepEqual(targetFragmentNode, client.manualFragmentNode) { + t.Fatalf("%v != %v", targetFragmentNode, client.manualFragmentNode) + } + client, err = NewClient(":9999") + if err != nil { + t.Fatal(err) + } + + targetURI, err = pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + + target := []*pnet.URI{targetURI} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + client, err = NewClient([]string{":9999"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + targetURI1, err := pnet.NewURIFromAddress(":8888") + if err != nil { + t.Fatal(err) + } + targetURI2, err := pnet.NewURIFromAddress(":9999") + if err != nil { + t.Fatal(err) + } + + client, err = NewClient([]*pnet.URI{targetURI1, targetURI2}) + if err != nil { + t.Fatal(err) + } + + target = []*pnet.URI{targetURI1, targetURI2} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + client, err = NewClient([]*pnet.URI{targetURI}) + if err != nil { + t.Fatal(err) + } + target = []*pnet.URI{targetURI} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } + + client, err = NewClient(DefaultCluster()) + if err != nil { + t.Fatal(err) + } + target = []*pnet.URI{} + if !reflect.DeepEqual(target, client.cluster.hosts) { + t.Fatalf("%v != %v", target, client.cluster.hosts) + } +} + +func TestNewClientWithInvalidAddr(t *testing.T) { + _, err := NewClient(10) + if err != ErrAddrURIClusterExpected { + t.Fatalf("%v != %v", ErrAddrURIClusterExpected, err) + } + _, err = NewClient(":invalid") + if err == nil { + t.Fatalf("should have failed: %+v", err) + } + _, err = NewClient([]string{"valid:8000", ":invalid"}) + if err != pnet.ErrInvalidAddress { + t.Fatalf("Should have failed '%v, got '%v'", pnet.ErrInvalidAddress, err) + } +} + +func TestNewClientManualAddressWithNoURIs(t *testing.T) { + _, err := NewClient([]string{}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } + _, err = NewClient([]*pnet.URI{}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } +} + +func TestNewClientManualAddressWithMultipleURIs(t *testing.T) { + _, err := NewClient([]string{":9000", ":5000"}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } + + targetURI1, err := pnet.NewURIFromAddress(":9000") + if err != nil { + t.Fatal(err) + } + targetURI2, err := pnet.NewURIFromAddress(":5000") + if err != nil { + t.Fatal(err) + } + + _, err = NewClient([]*pnet.URI{targetURI1, targetURI2}, OptClientManualServerAddress(true)) + if err != ErrSingleServerAddressRequired { + t.Fatalf("%v != %v", ErrSingleServerAddressRequired, err) + } +} + +func ClientOptionErr(int) ClientOption { + return func(*ClientOptions) error { + return errors.New("Some error") + } +} + +func TestQueryOptions(t *testing.T) { + targets := []*QueryOptions{ + {ColumnAttrs: true}, + {ColumnAttrs: false}, + {ExcludeRowAttrs: true}, + {ExcludeRowAttrs: false}, + {ExcludeColumns: true}, + {ExcludeColumns: false}, + } + + optionsList := [][]interface{}{ + {OptQueryColumnAttrs(true)}, + {OptQueryColumnAttrs(false)}, + {OptQueryExcludeAttrs(true)}, + {OptQueryExcludeAttrs(false)}, + {OptQueryExcludeColumns(true)}, + {OptQueryExcludeColumns(false)}, + } + + for i := 0; i < len(targets); i++ { + options := &QueryOptions{} + err := options.addOptions(optionsList[i]...) + if err != nil { + t.Fatal(err) + } + target := targets[i] + if !reflect.DeepEqual(target, options) { + t.Fatalf("%v != %v", target, options) + } + } + + target := &QueryOptions{ + ColumnAttrs: true, + ExcludeRowAttrs: true, + ExcludeColumns: true, + } + options := &QueryOptions{} + err := options.addOptions(&QueryOptions{ + ColumnAttrs: true, + ExcludeRowAttrs: true, + ExcludeColumns: true, + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, options) { + t.Fatalf("%v != %v", target, options) + } +} + +func TestQueryOptionsWithError(t *testing.T) { + options := &QueryOptions{} + err := options.addOptions(1) + if err == nil { + t.Fatalf("should have failed") + } + err = options.addOptions(OptQueryColumnAttrs(true), nil) + if err == nil { + t.Fatalf("should have failed") + } + err = options.addOptions(OptQueryColumnAttrs(true), &QueryOptions{}) + if err == nil { + t.Fatalf("should have failed") + } + err = options.addOptions(QueryOptionErr(0)) + if err == nil { + t.Fatalf("should have failed") + } +} + +func TestQueryOptionsError(t *testing.T) { + client := DefaultClient() + index := NewIndex("foo") + _, err := client.Query(index.RawQuery(""), QueryOptionErr(0)) + if err == nil { + t.Fatalf("should have failed") + } +} + +func QueryOptionErr(int) QueryOption { + return func(*QueryOptions) error { + return errors.New("Some error") + } +} diff --git a/client/cluster.go b/client/cluster.go new file mode 100644 index 000000000..4c0df25db --- /dev/null +++ b/client/cluster.go @@ -0,0 +1,112 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "sync" + + pnet "github.com/pilosa/pilosa/v2/net" +) + +// Cluster contains hosts in a Pilosa cluster. +type Cluster struct { + hosts []*pnet.URI + okList []bool + mutex *sync.RWMutex + lastHostIdx int +} + +// DefaultCluster returns the default Cluster. +func DefaultCluster() *Cluster { + return &Cluster{ + hosts: make([]*pnet.URI, 0), + okList: make([]bool, 0), + mutex: &sync.RWMutex{}, + } +} + +// NewClusterWithHost returns a cluster with the given URIs. +func NewClusterWithHost(hosts ...*pnet.URI) *Cluster { + cluster := DefaultCluster() + for _, host := range hosts { + cluster.AddHost(host) + } + return cluster +} + +// AddHost adds a host to the cluster. +func (c *Cluster) AddHost(address *pnet.URI) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.hosts = append(c.hosts, address) + c.okList = append(c.okList, true) +} + +// Host returns a host in the cluster. +func (c *Cluster) Host() *pnet.URI { + c.mutex.Lock() + var host *pnet.URI + for i := range c.okList { + idx := (i + c.lastHostIdx) % len(c.okList) + ok := c.okList[idx] + if ok { + host = c.hosts[idx] + break + } + } + c.lastHostIdx++ + c.mutex.Unlock() + if host != nil { + return host + } + c.reset() + return host +} + +// RemoveHost black lists the host with the given pnet.URI from the cluster. +func (c *Cluster) RemoveHost(address *pnet.URI) { + c.mutex.Lock() + defer c.mutex.Unlock() + for i, uri := range c.hosts { + if uri.Equals(address) { + c.okList[i] = false + break + } + } +} + +// Hosts returns all available hosts in the cluster. +func (c *Cluster) Hosts() []pnet.URI { + c.mutex.RLock() + defer c.mutex.RUnlock() + hosts := make([]pnet.URI, 0, len(c.hosts)) + for i, host := range c.hosts { + if c.okList[i] { + hosts = append(hosts, *host) + } + } + return hosts +} + +func (c *Cluster) reset() { + c.mutex.Lock() + defer c.mutex.Unlock() + for i := range c.okList { + c.okList[i] = true + } +} diff --git a/client/cluster_test.go b/client/cluster_test.go new file mode 100644 index 000000000..caf1284f2 --- /dev/null +++ b/client/cluster_test.go @@ -0,0 +1,83 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "testing" + + pnet "github.com/pilosa/pilosa/v2/net" +) + +func TestNewClusterWithHost(t *testing.T) { + c := NewClusterWithHost(pnet.DefaultURI()) + hosts := c.Hosts() + if len(hosts) != 1 || !hosts[0].Equals(pnet.DefaultURI()) { + t.Fail() + } +} + +func TestAddHost(t *testing.T) { + const addr = "http://localhost:3000" + c := DefaultCluster() + if c.Hosts() == nil { + t.Fatalf("Hosts should not be nil") + } + uri, err := pnet.NewURIFromAddress(addr) + if err != nil { + t.Fatalf("Cannot parse address") + } + target, err := pnet.NewURIFromAddress(addr) + if err != nil { + t.Fatalf("Cannot parse address") + } + c.AddHost(uri) + hosts := c.Hosts() + if len(hosts) != 1 || !hosts[0].Equals(target) { + t.Fail() + } +} + +func TestHosts(t *testing.T) { + c := DefaultCluster() + if c.Host() != nil { + t.Fatalf("Hosts with empty cluster should return nil") + } + c = NewClusterWithHost(pnet.DefaultURI()) + if !c.Host().Equals(pnet.DefaultURI()) { + t.Fatalf("Host should return a value if there are hosts in the cluster") + } +} + +func TestRemoveHost(t *testing.T) { + uri, err := pnet.NewURIFromAddress("index1.pilosa.com:9999") + if err != nil { + t.Fatal(err) + } + c := NewClusterWithHost(uri) + if len(c.hosts) != 1 { + t.Fatalf("The cluster should contain the host") + } + uri, err = pnet.NewURIFromAddress("index1.pilosa.com:9999") + if err != nil { + t.Fatal(err) + } + c.RemoveHost(uri) + if len(c.Hosts()) != 0 { + t.Fatalf("The cluster should not contain the host") + } +} diff --git a/client/csv/csv.go b/client/csv/csv.go new file mode 100644 index 000000000..9c30a622e --- /dev/null +++ b/client/csv/csv.go @@ -0,0 +1,194 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package csv + +import ( + "bufio" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/pilosa/pilosa/v2/client" +) + +// Format is the format of the data in the CSV file. +type Format uint + +const ( + // RowIDColumnID formatted data is ROW_ID,COLUMN_ID. + RowIDColumnID Format = iota + // RowIDColumnKey formatted data is ROW_ID,COLUMN_KEY. + RowIDColumnKey + // RowKeyColumnID formatted data is ROW_KEY,COLUMN_ID. + RowKeyColumnID + // RowKeyColumnKey formatted data is ROW_KEY,COLUMN_ID. + RowKeyColumnKey + // ColumnID formatted data is COLUMN_ID. Valid only for value import. + ColumnID + // ColumnKey formatted data is COLUMN_KEY. Valud only for value import. + ColumnKey +) + +// ColumnUnmarshaller creates a RecordUnmarshaller for importing columns with the given format. +func ColumnUnmarshaller(format Format) RecordUnmarshaller { + return ColumnUnmarshallerWithTimestamp(format, "") +} + +// ColumnUnmarshallerWithTimestamp creates a RecordUnmarshaller for importing columns with the given format and timestamp format. +func ColumnUnmarshallerWithTimestamp(format Format, timestampFormat string) RecordUnmarshaller { + return func(text string) (client.Record, error) { + var err error + column := client.Column{} + parts := strings.Split(text, ",") + if len(parts) < 2 { + return nil, errors.New("Invalid CSV line") + } + + hasRowKey := format == RowKeyColumnID || format == RowKeyColumnKey + hasColumnKey := format == RowIDColumnKey || format == RowKeyColumnKey + + if hasRowKey { + column.RowKey = parts[0] + } else { + column.RowID, err = strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return nil, errors.New("Invalid row ID") + } + } + + if hasColumnKey { + column.ColumnKey = parts[1] + } else { + column.ColumnID, err = strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return nil, errors.New("Invalid column ID") + } + } + + timestamp := int64(0) + if len(parts) == 3 { + if timestampFormat == "" { + if tsInt, err := strconv.Atoi(parts[2]); err != nil { + return nil, err + } else { + timestamp = int64(tsInt) + } + } else { + t, err := time.Parse(timestampFormat, parts[2]) + if err != nil { + return nil, err + } + timestamp = t.Unix() * int64(time.Second) // Casting a duration to int64 gives the number of nanoseconds in that duration. + } + } + column.Timestamp = timestamp + + return column, nil + } +} + +// RecordUnmarshaller is a function which creates a Record from a CSV file line with column data. +type RecordUnmarshaller func(text string) (client.Record, error) + +// Iterator reads records from a Reader. +// Each line should contain a single record in the following form: +// field1,field2,... +type Iterator struct { + reader io.Reader + line int + scanner *bufio.Scanner + unmarshaller RecordUnmarshaller +} + +// NewIterator creates a CSVIterator from a Reader. +func NewIterator(reader io.Reader, unmarshaller RecordUnmarshaller) *Iterator { + return &Iterator{ + reader: reader, + line: 0, + scanner: bufio.NewScanner(reader), + unmarshaller: unmarshaller, + } +} + +// NewColumnIterator creates a new iterator for column data. +func NewColumnIterator(format Format, reader io.Reader) *Iterator { + return NewIterator(reader, ColumnUnmarshaller(format)) +} + +// NewColumnIteratorWithTimestampFormat creates a new iterator for column data with timestamp. +func NewColumnIteratorWithTimestampFormat(format Format, reader io.Reader, timestampFormat string) *Iterator { + return NewIterator(reader, ColumnUnmarshallerWithTimestamp(format, timestampFormat)) +} + +// NewValueIterator creates a new iterator for value data. +func NewValueIterator(format Format, reader io.Reader) *Iterator { + return NewIterator(reader, FieldValueUnmarshaller(format)) +} + +// NextRecord iterates on lines of a Reader. +// Returns io.EOF on end of iteration. +func (c *Iterator) NextRecord() (client.Record, error) { + if ok := c.scanner.Scan(); ok { + c.line++ + text := strings.TrimSpace(c.scanner.Text()) + if text != "" { + rc, err := c.unmarshaller(text) + if err != nil { + return nil, fmt.Errorf("%s at line: %d", err.Error(), c.line) + } + return rc, nil + } + } + err := c.scanner.Err() + if err != nil { + return nil, err + } + return nil, io.EOF +} + +// FieldValueUnmarshaller is a function which creates a Record from a CSV file line with value data. +func FieldValueUnmarshaller(format Format) RecordUnmarshaller { + return func(text string) (client.Record, error) { + parts := strings.Split(text, ",") + if len(parts) < 2 { + return nil, errors.New("Invalid CSV") + } + value, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, errors.New("Invalid value") + } + switch format { + case ColumnID: + columnID, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return nil, errors.New("Invalid column ID at line: %d") + } + return client.FieldValue{ + ColumnID: uint64(columnID), + Value: value, + }, nil + case ColumnKey: + return client.FieldValue{ + ColumnKey: parts[0], + Value: value, + }, nil + default: + return nil, fmt.Errorf("Invalid format: %d", format) + } + } +} diff --git a/client/csv/csv_it_test.go b/client/csv/csv_it_test.go new file mode 100644 index 000000000..159b801dd --- /dev/null +++ b/client/csv/csv_it_test.go @@ -0,0 +1,60 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//+build integration + +package csv_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2/client" + "github.com/pilosa/pilosa/v2/client/csv" +) + +func TestCSVIterate(t *testing.T) { + text := `10,7 + 10,5 + 2,3 + 7,1` + iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text)) + recs := consumeIterator(t, iterator) + target := []client.Record{ + client.Column{RowID: 10, ColumnID: 7}, + client.Column{RowID: 10, ColumnID: 5}, + client.Column{RowID: 2, ColumnID: 3}, + client.Column{RowID: 7, ColumnID: 1}, + } + if !reflect.DeepEqual(target, recs) { + t.Fatalf("%v != %v", target, recs) + } +} + +func consumeIterator(t *testing.T, it *csv.Iterator) []client.Record { + recs := []client.Record{} + for { + r, err := it.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + recs = append(recs, r) + } + return recs +} diff --git a/client/csv/csv_test.go b/client/csv/csv_test.go new file mode 100644 index 000000000..4ef162b8e --- /dev/null +++ b/client/csv/csv_test.go @@ -0,0 +1,267 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package csv_test + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/client" + "github.com/pilosa/pilosa/v2/client/csv" +) + +func TestCSVColumnIterator(t *testing.T) { + reader := strings.NewReader(`1,10,683793200 + 5,20,683793300 + 3,41,683793385`) + iterator := csv.NewColumnIterator(csv.RowIDColumnID, reader) + columns := []client.Record{} + for { + column, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + columns = append(columns, column) + } + if len(columns) != 3 { + t.Fatalf("There should be 3 columns") + } + target := []client.Column{ + {RowID: 1, ColumnID: 10, Timestamp: 683793200}, + {RowID: 5, ColumnID: 20, Timestamp: 683793300}, + {RowID: 3, ColumnID: 41, Timestamp: 683793385}, + } + for i := range target { + if !reflect.DeepEqual(target[i], columns[i]) { + t.Fatalf("%v != %v", target[i], columns[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatRowIDColumnID(t *testing.T) { + format := "2006-01-02T03:04" + reader := strings.NewReader(`1,10,1991-09-02T09:33 + 5,20,1991-09-02T09:35 + 3,41,1991-09-02T09:36`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format) + records := []client.Record{} + for { + record, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + records = append(records, record) + } + target := []client.Column{ + {RowID: 1, ColumnID: 10, Timestamp: 683803980000000000}, + {RowID: 5, ColumnID: 20, Timestamp: 683804100000000000}, + {RowID: 3, ColumnID: 41, Timestamp: 683804160000000000}, + } + if len(records) != len(target) { + t.Fatalf("There should be %d columns", len(target)) + } + for i := range target { + if !reflect.DeepEqual(target[i], records[i]) { + t.Fatalf("%v != %v", target[i], records[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatRowKeyColumnKey(t *testing.T) { + format := "2006-01-02T03:04" + reader := strings.NewReader(`one,ten,1991-09-02T09:33 + five,twenty,1991-09-02T09:35 + three,forty-one,1991-09-02T09:36`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowKeyColumnKey, reader, format) + records := []client.Record{} + for { + record, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + records = append(records, record) + } + target := []client.Column{ + {RowKey: "one", ColumnKey: "ten", Timestamp: 683803980000000000}, + {RowKey: "five", ColumnKey: "twenty", Timestamp: 683804100000000000}, + {RowKey: "three", ColumnKey: "forty-one", Timestamp: 683804160000000000}, + } + if len(records) != len(target) { + t.Fatalf("There should be %d columns", len(target)) + } + for i := range target { + if !reflect.DeepEqual(target[i], records[i]) { + t.Fatalf("%v != %v", target[i], records[i]) + } + } +} + +func TestCSVColumnIteratorWithTimestampFormatFail(t *testing.T) { + format := "2014-07-16" + reader := strings.NewReader(`1,10,X`) + iterator := csv.NewColumnIteratorWithTimestampFormat(csv.RowIDColumnID, reader, format) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("Should have failed") + } +} + +func TestCSVValueIteratorWithColumnID(t *testing.T) { + reader := strings.NewReader(`1,10 + 5,-20 + 3,41 + `) + iterator := csv.NewValueIterator(csv.ColumnID, reader) + values := []client.Record{} + for { + value, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + values = append(values, value) + } + target := []pilosa.FieldValue{ + {ColumnID: 1, Value: 10}, + {ColumnID: 5, Value: -20}, + {ColumnID: 3, Value: 41}, + } + if len(values) != len(target) { + t.Fatalf("There should be %d values, got %d", len(target), len(values)) + } + for i := range target { + v := values[i].(client.FieldValue) + if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) { + t.Fatalf("'%+v' != '%+v'", target[i], values[i]) + } + } +} + +func TestCSVValueIteratorWithColumnKey(t *testing.T) { + reader := strings.NewReader(`one,10 + five,-20 + three,41 + `) + iterator := csv.NewValueIterator(csv.ColumnKey, reader) + values := []client.Record{} + for { + value, err := iterator.NextRecord() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + values = append(values, value) + } + target := []pilosa.FieldValue{ + {ColumnKey: "one", Value: 10}, + {ColumnKey: "five", Value: -20}, + {ColumnKey: "three", Value: 41}, + } + if len(values) != len(target) { + t.Fatalf("There should be %d values, got %d", len(target), len(values)) + } + for i := range target { + v := values[i].(client.FieldValue) + if !reflect.DeepEqual(pilosa.FieldValue(v), target[i]) { + t.Fatalf("%v != %v", target[i], values[i]) + } + } +} + +func TestCSValueIteratorWithInvalidFormat(t *testing.T) { + reader := strings.NewReader("1,2") + iterator := csv.NewValueIterator(csv.RowIDColumnID, reader) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("should have failed") + } +} + +func TestCSVColumnIteratorInvalidInput(t *testing.T) { + invalidInputs := []string{ + // less than 2 columns + "155", + // invalid row ID + "a5,155", + // invalid column ID + "155,a5", + // invalid timestamp + "155,255,a5", + } + for _, text := range invalidInputs { + iterator := csv.NewColumnIterator(csv.RowIDColumnID, strings.NewReader(text)) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("CSVColumnIterator input: %s should fail", text) + } + } +} + +func TestCSVValueIteratorInvalidInput(t *testing.T) { + invalidInputs := []string{ + // less than 2 columns + "155", + // invalid column ID + "a5,155", + // invalid value + "155,a5", + } + for _, text := range invalidInputs { + iterator := csv.NewValueIterator(csv.ColumnID, strings.NewReader(text)) + _, err := iterator.NextRecord() + if err == nil { + t.Fatalf("CSVValueIterator input: %s should fail", text) + } + } +} + +func TestCSVColumnIteratorError(t *testing.T) { + iterator := csv.NewColumnIterator(csv.RowIDColumnID, &BrokenReader{}) + _, err := iterator.NextRecord() + if err == nil { + t.Fatal("CSVColumnIterator should fail with error") + } +} + +func TestCSVValueIteratorError(t *testing.T) { + iterator := csv.NewValueIterator(csv.ColumnID, &BrokenReader{}) + _, err := iterator.NextRecord() + if err == nil { + t.Fatal("CSVValueIterator should fail with error") + } +} + +type BrokenReader struct{} + +func (r BrokenReader) Read(p []byte) (n int, err error) { + return 0, errors.New("broken reader") +} diff --git a/client/doc.go b/client/doc.go new file mode 100644 index 000000000..ea7153e01 --- /dev/null +++ b/client/doc.go @@ -0,0 +1,68 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +/* +Package client enables querying a Pilosa server. + +This client uses Pilosa's http+protobuf API. + +Usage: + + import ( + "fmt" + "github.com/pilosa/pilosa/v2/client" + ) + + // Create a Client instance + cli := client.DefaultClient() + + // Create a Schema instance + schema, err := cli.Schema() + if err != nil { + panic(err) + } + + // Create an Index instance + index, err := schema.Index("repository") + if err != nil { + panic(err) + } + + // Create a Field instance + stargazer, err := index.Field("stargazer") + if err != nil { + panic(err) + } + + // Sync the schema with the server-side, so non-existing indexes/fields are created on the server-side. + err = cli.SyncSchema(schema) + if err != nil { + panic(err) + } + + // Execute a query + response, err := cli.Query(stargazer.Row(5)) + if err != nil { + panic(err) + } + + // Act on the result + fmt.Println(response.Result()) + +See also https://www.pilosa.com/docs/api-reference/ and https://www.pilosa.com/docs/query-language/. +*/ +package client diff --git a/client/docs/data-model-queries.md b/client/docs/data-model-queries.md new file mode 100644 index 000000000..9e88f7781 --- /dev/null +++ b/client/docs/data-model-queries.md @@ -0,0 +1,152 @@ +# Data Model and Queries + +## Indexes and Fields + +*Index* and *field*s are the main data models of Pilosa. You can check the [Pilosa documentation](https://www.pilosa.com/docs/latest/data-model/) for more detail about the data model. + +The `schema.Index` function is used to create an index instance. Note that this does not create an index on the server; the index object simply defines the schema. + +```go +schema := client.NewSchema() +repository := schema.Index("repository") +``` + +You can pass options while creating index instances: +```go +repository := schema.Index("repository", pilosa.OptIndexKeys(true)) +``` + +Field definitions are created with a call to the `Field` function of an index: + +```go +stargazer := repository.Field("stargazer") +``` + +You can pass options to fields: + +```go +stargazer := repository.Field("stargazer", pilosa.OptFieldTypeTime(TimeQuantumYearMonthDay)) +``` + +In case the schema already exists on the server, you can retrieve that instead of creating the schema: +```go +cli := client.DefaultClient() +schema, err := cli.Schema() +if err != nil { + // act on the error +} +repository := schema.Index("repository") +``` + +## Queries + +Once you have indexes and field definitions, you can create queries for them. Some of the queries work on the columns; corresponding methods are attached to the index. Other queries work on rows with related methods attached to fields. + +For instance, `Row` queries work on rows; use a `Field` object to create those queries: + +```go +rowQuery := stargazer.Row(1) // corresponds to PQL: Row(stargazer=1) +``` + +`Union` queries work on columns; use the index to create them: + +```go +query := repository.Union(rowQuery1, rowQuery2) +``` + +In order to increase throughput, you may want to batch queries sent to the Pilosa server. The `index.BatchQuery` function is used for that purpose: + +```go +query := repository.BatchQuery( + stargazer.Row(1), + repository.Union(stargazer.Row(100), stargazer.Row(5))) +``` + +The recommended way of creating query instances is using dedicated functions attached to index and field objects, but sometimes it would be desirable to send raw queries to Pilosa. You can use `index.RawQuery` method for that. Note that query string is not validated before sending to the server: + +```go +query := repository.RawQuery("Row(stargazer=5)") +``` + +Raw queries are only sent to the coordinator node of a Pilosa cluster, so currently there's a possible performance hit using them instead of ORM functions attached to index or field instances. + +This client supports [range queries using bit sliced indexes (BSI)](https://www.pilosa.com/docs/latest/query-language/#range-bsi). Read the [Range Encoded Bitmaps](https://www.pilosa.com/blog/range-encoded-bitmaps/) blog post for more information about the BSI implementation of range encoding in Pilosa. + +In order to use BSI range queries, an integer field should be created. The field should have its minimum and maximum set. Here's how you would do that: +```go +index := schema.Index("animals") +captivity := index.Field("captivity", pilosa.OptFieldTypeInt(0, 956)) +``` + +If the field with the necessary field already exists on the server, you don't need to create the field instance, `cli.SyncSchema(schema)` would load that to `schema`. You can then add some data: +```go +// Add the captivity values to the field. +data := []int{3, 392, 47, 956, 219, 14, 47, 504, 21, 0, 123, 318} +query := index.BatchQuery() +for i, x := range data { + column := uint64(i + 1) + query.Add(captivity.SetIntValue(column, x)) +} +cli.Query(query) +``` + +Let's write a range query: +```go +// Query for all animals with more than 100 specimens +response, _ := cli.Query(captivity.GT(100)) +fmt.Println(response.Result().Row().Columns) + +// Query for the total number of animals in captivity +response, _ = cli.Query(captivity.Sum(nil)) +fmt.Println(response.Result().Value()) +``` + +If you pass a row query to `Sum` as a filter, then only the columns matching the filter will be considered in the `Sum` calculation: +```go +// Let's run a few set queries first +cli.Query(index.BatchQuery( + field.Set(42, 1), + field.Set(42, 6))) +// Query for the total number of animals in captivity where row 42 is set +response, _ = cli.Query(captivity.Sum(field.Row(42))) +fmt.Println(response.Result().Value()) +``` + +See the functions further below for the list of functions that can be used with a `Field`. + +Please check [Pilosa documentation](https://www.pilosa.com/docs) for PQL details. Here is a list of methods corresponding to PQL calls: + +Index: + +* `Union(rows *PQLRowQuery...) *PQLRowQuery` +* `Intersect(rows *PQLRowQuery...) *PQLRowQuery` +* `Difference(rows *PQLRowQuery...) *PQLRowQuery` +* `Xor(rows ...*PQLRowQuery) *PQLRowQuery` +* `Not(row) *PQLRowQuery` +* `Count(row *PQLRowQuery) *PQLBaseQuery` +* `SetColumnAttrs(columnID uint64, attrs map[string]interface{}) *PQLBaseQuery` +* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery` + +Field: + +* `Row(rowID uint64) *PQLRowQuery` +* `Set(rowID uint64, columnID uint64) *PQLBaseQuery` +* `SetTimestamp(rowID uint64, columnID uint64, timestamp time.Time) *PQLBaseQuery` +* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery` +* `TopN(n uint64) *PQLRowQuery` +* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery` +* `FilterFieldTopN(n uint64, row *PQLRowQuery, field string, values ...interface{}) *PQLRowQuery` +* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery` +* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery` +* `SetRowAttrs(rowID uint64, attrs map[string]interface{}) *PQLBaseQuery` +* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery` +* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery` +* `LT(n int) *PQLRowQuery` +* `LTE(n int) *PQLRowQuery` +* `GT(n int) *PQLRowQuery` +* `GTE(n int) *PQLRowQuery` +* `Between(a int, b int) *PQLRowQuery` +* `Sum(row *PQLRowQuery) *PQLBaseQuery` +* `Min(row *PQLRowQuery) *PQLBaseQuery` +* `Max(row *PQLRowQuery) *PQLBaseQuery` +* `SetIntValue(columnID uint64, value int) *PQLBaseQuery` diff --git a/client/docs/server-interaction.md b/client/docs/server-interaction.md new file mode 100644 index 000000000..a713c30c3 --- /dev/null +++ b/client/docs/server-interaction.md @@ -0,0 +1,178 @@ +# Server Interaction + +## Pilosa URI + +A Pilosa URI has the `${SCHEME}://${HOST}:${PORT}` format: +* **Scheme**: Protocol of the URI. Default: `http`. +* **Host**: Hostname or ipv4/ipv6 IP address. Default: localhost. +* **Port**: Port number. Default: `10101`. + +All parts of the URI are optional, but at least one of them must be specified. The following are equivalent: + +* `http://localhost:10101` +* `http://localhost` +* `http://:10101` +* `localhost:10101` +* `localhost` +* `:10101` + +A Pilosa URI is represented by the `github.com/pilosa/pilosa/v2/net URI` struct. Below are a few ways to create `URI` objects: + +```go +import pnet "github.com/pilosa/pilosa/v2/net" + +// create the default URI: http://localhost:10101 +uri1 := pnet.DefaultURI() + +// create a URI from string address +uri2, err := pnet.NewURIFromAddress("index1.pilosa.com:20202"); + +// create a URI with the given host and port +uri3, err := pnet.NewURIFromHostPort("index1.pilosa.com", 20202); +``` + +## Pilosa Client + +In order to interact with a Pilosa server, an instance of `client.Client` should be created. The client is thread-safe and uses a pool of connections to the server, so we recommend creating a single instance of the client and sharing it when necessary. + +If the Pilosa server is running at the default address (`http://localhost:10101`) you can create the client with default options using: + +```go +import "github.com/pilosa/pilosa/v2/client" + +cli := client.DefaultClient() +``` + +To use a custom server address, use the `NewClient` function: + +```go +uri, err := pnet.NewURIFromAddress("http://index1.pilosa.com:15000") +if err != nil { + // Act on the error +} +cli, err := client.NewClient(uri) +``` + +Equivalently: +```go +cli, err := client.NewClient("http://index1.pilosa.com:15000") +``` + +If you are running a cluster of Pilosa servers, you can create a `Cluster` struct that keeps addresses of those servers: + +```go +uri1, err := pnet.NewURIFromAddress(":10101") +uri2, err := pnet.NewURIFromAddress(":10110") +uri3, err := pnet.NewURIFromAddress(":10111") +cluster := client.NewClusterWithHost(uri1, uri2, uri3) + +// Create a client with the cluster +cli, err := client.NewClient(cluster) +``` + +That is equivalent to: +```go +cli, err := client.NewClient([]string{":10101", ":10110", ":10111"}) + +``` + +It is possible to customize the behaviour of the underlying HTTP client by passing `ClientOption` structs to the `NewClient` function: + +```go +cli, err := client.NewClient(cluster, + client.OptClientConnectTimeout(1000), // if can't connect in a second, close the connection + client.OptClientSocketTimeout(10000), // if no response received in 10 seconds, close the connection + client.OptClientPoolSizePerRoute(3), // number of connections in the pool per host + client.OptClientTotalPoolSize(10)) // number of total connections in the pool +``` + +Once you create a client, you can create indexes, fields or start sending queries. + +Here is how you would create a index and field: + +```go +// materialize repository index definition and stargazer field definition initialized before +err := cli.SyncSchema(schema) +``` + +You can send queries to a Pilosa server using the `Query` function of the `Client` struct: + +```go +response, err := cli.Query(field.Row(5)); +``` + +`Query` accepts zero or more options: + +```go +response, err := cli.Query(field.Row(5), pilosa.ColumnAttrs(true), pilosa.ExcludeColumns(true)) +``` + +## Server Response + +When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned. + +A `QueryResponse` struct may contain zero or more results of `QueryResult` type. You can access all results using the `Results` function of `QueryResponse` (which returns a list of `QueryResult` objects), or you can use the `Result` method (which returns either the first result or `nil` if there are no results): + +```go +response, err := cli.Query(field.Row(5)) +if err != nil { + // Act on the error +} + +// check that there's a result and act on it +result := response.Result() +if result != nil { + // Act on the result +} + +// iterate over all results +for _, result := range response.Results() { + // Act on the result +} +``` + +Similarly, a `QueryResponse` struct may include a number of column attributes if `ColumnAttrs` query option was set to `true`: + +```go +var column *pilosa.ColumnItem + +// iterate over all columns +for _, column = range response.ColumnAttrs() { + // Act on the column item +} +``` + +`QueryResult` objects contain: + +* `Row()` function to retrieve a row result, +* `CountItems()` function to retrieve column count per row ID entries returned from `TopN` queries, +* `Count()` function to retrieve the number of rows per the given row ID returned from `Count` queries. +* `Value()` function to retrieve the result of `Min`, `Max` or `Sum` queries. +* `Changed()` function returns whether a `Set` or `Clear` query changed a column. + +```go +row := result.Row() +columns := row.Columns +attributes := row.Attributes + +countItems := result.CountItems() + +count := result.Count() + +value := result.Value() + +changed := result.Changed() +``` + +## SSL/TLS + +Make sure the Pilosa server runs on a TLS address. [How To Set Up a Secure Cluster](https://www.pilosa.com/docs/latest/tutorials/#how-to-set-up-a-secure-cluster) tutorial explains how to do that. + +In order to enable TLS support on the client side, the scheme of the address should be explicitly specified as `https`, e.g.: `https://01.pilosa.local:10501` + +This client library uses the `net/http` module of Go standard library. You can pass a [tls.Config](https://golang.org/pkg/crypto/tls/#Config) struct in a `pilosa.TLSConfig` option to the client. If the Pilosa server is using a certificate from a recognized authority, you can use the defaults. + +If you are using a self signed certificate, just pass `pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})` to `pilosa.NewClient` function: +```go +client, _ := pilosa.NewClient("https://01.pilosa.local:10501", pilosa.TLSConfig(&tls.Config{InsecureSkipVerify: true})) +``` diff --git a/client/docs/tracing.md b/client/docs/tracing.md new file mode 100644 index 000000000..868e374df --- /dev/null +++ b/client/docs/tracing.md @@ -0,0 +1,111 @@ +# Tracing + +Pilosa client supports distributed tracing via the [OpenTracing](https://opentracing.io/) API. + +In order to use a tracer with Go-Pilosa, you should: +1. Create the tracer, +2. Pass the `OptClientOption(tracer)` to `NewClient`. + +In this document, we will be using the [Jaeger](https://www.jaegertracing.io) tracer, but OpenTracing has support for [other tracing systems](https://opentracing.io/docs/supported-tracers/). + +## Running the Pilosa Server + +Let's run a temporary Pilosa container: + + $ docker run -it --rm -p 10101:10101 pilosa/pilosa:v1.2.0 + +Check that you can access Pilosa: + + $ curl localhost:10101 + Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information. + +## Running the Jaeger Server + +Let's run a Jaeger Server container: + + $ docker run -it --rm -p 5775:5775/udp -p 16686:16686 jaegertracing/all-in-one:latest + ...Jaeger UI... + +## Writing the Sample Code + +The sample code depdends on the Jaeger Go client, so let's install it first: + + $ go get -u github.com/uber/jaeger-client-go/ + +Save the following sample code as `gopilosa-tracing.go`: +```go +package main + +import ( + "log" + "time" + + "github.com/pilosa/pilosa/v2/client" + "github.com/uber/jaeger-client-go" + "github.com/uber/jaeger-client-go/config" +) + +func main() { + // Create the tracer. + cfg := config.Configuration{ + Sampler: &config.SamplerConfig{ + Type: "const", + Param: 1, + }, + Reporter: &config.ReporterConfig{ + LogSpans: true, + BufferFlushInterval: 1 * time.Second, + // Jaeger Server address + LocalAgentHostPort: "127.0.0.1:5775", + }, + } + tracer, closer, err := cfg.New( + "go_pilosa_test", + config.Logger(jaeger.StdLogger), + ) + + // Don't forget to close the tracer. + defer closer.Close() + + // Create the client, and pass the tracer. + cli, err := client.NewClient(":10101", pilosa.OptClientTracer(tracer)) + if err != nil { + log.Fatal(err) + } + + // Read the schema from the server. + // This should create a trace on the Jaeger server. + schema, err := cli.Schema() + if err != nil { + log.Fatal(err) + } + + // Create and sync the sample schema. + // This should create a trace on the Jaeger server. + myIndex := schema.Index("my-index") + myField := myIndex.Field("my-field") + err = cli.SyncSchema(schema) + if err != nil { + log.Fatal(err) + } + + // Run a query on Pilosa. + // This should create a trace on the Jaeger server. + _, err = cli.Query(myField.Set(1, 1000)) + if err != nil { + log.Fatal(err) + } +} +``` + +## Checking the Tracing Data + +Run the sample code: + + $ go run gopilosa-tracing.go + + +* Open http://localhost:16686 in your web browser to visit Jaeger UI. +* Click on the *Search* tab and select `go_pilosa_test` in the *Service* dropdown on the right. +* Click on *Find Traces* button at the bottom left. +* You should see a couple of traces, such as: `Client.Query`, `Client.CreateField`, `Client.Schema`, etc. diff --git a/client/egpool/egpool.go b/client/egpool/egpool.go new file mode 100644 index 000000000..92449cf7c --- /dev/null +++ b/client/egpool/egpool.go @@ -0,0 +1,123 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egpool + +import ( + "errors" + "fmt" + "sync" +) + +type Group struct { + PoolSize int + + jobs chan func() error + + sema chan struct{} + errMu sync.Mutex + firstErr error + errs []error +} + +func (eg *Group) Go(f func() error) { + if eg.PoolSize <= 0 { + eg.PoolSize = 1 + } + + if eg.jobs == nil { + eg.jobs = make(chan func() error) + eg.sema = make(chan struct{}, eg.PoolSize) + } + + // Start the job in an idle worker if possible. + select { + case eg.jobs <- f: + return + default: + } + + // Start a new worker if necessary. + select { + case eg.jobs <- f: + // A worker finished its previous job and took this one over. + return + case eg.sema <- struct{}{}: + // Start a new worker. + go eg.processJobs() + eg.jobs <- f + } +} + +func (eg *Group) err(err error) { + eg.errMu.Lock() + defer eg.errMu.Unlock() + + if eg.firstErr == nil { + eg.firstErr = err + } + eg.errs = append(eg.errs, err) +} + +type ErrPanic struct { + Value interface{} +} + +func (p ErrPanic) Error() string { + return fmt.Sprintf("panic: %v", p.Value) +} + +var ErrGoexit = errors.New("runtime.Goexit used in job function") + +func (eg *Group) processJobs() { + // Notify pool of shutdown. + defer func() { <-eg.sema }() + + // Handle panic and Goexit. + var finished bool + defer func() { + if !finished { + if p := recover(); p != nil { + eg.err(ErrPanic{p}) + } else { + eg.err(ErrGoexit) + } + } + }() + + // Run jobs from queue. + for jobFn := range eg.jobs { + err := jobFn() + if err != nil { + eg.err(err) + } + } + + finished = true +} + +func (eg *Group) Wait() error { + if eg.jobs == nil { + return nil + } + close(eg.jobs) + for i := 0; i < eg.PoolSize; i++ { + eg.sema <- struct{}{} + } + return eg.firstErr +} + +func (eg *Group) Errors() []error { + return eg.errs +} diff --git a/client/egpool/egpool_test.go b/client/egpool/egpool_test.go new file mode 100644 index 000000000..288c9fcb6 --- /dev/null +++ b/client/egpool/egpool_test.go @@ -0,0 +1,50 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egpool_test + +import ( + "errors" + "testing" + + "github.com/pilosa/pilosa/v2/client/egpool" +) + +func TestEGPool(t *testing.T) { + eg := egpool.Group{} + + a := make([]int, 10) + + for i := 0; i < 10; i++ { + i := i + eg.Go(func() error { + a[i] = i + if i == 7 { + return errors.New("blah") + } + return nil + }) + } + + err := eg.Wait() + if err == nil || err.Error() != "blah" { + t.Errorf("expected err blah, got: %v", err) + } + + for i := 0; i < 10; i++ { + if a[i] != i { + t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i]) + } + } +} diff --git a/client/error.go b/client/error.go new file mode 100644 index 000000000..ee011bbc7 --- /dev/null +++ b/client/error.go @@ -0,0 +1,38 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import "github.com/pkg/errors" + +// Predefined Pilosa errors. +var ( + ErrEmptyCluster = errors.New("No usable addresses in the cluster") + ErrIndexExists = errors.New("Index exists") + ErrFieldExists = errors.New("Field exists") + ErrInvalidIndexName = errors.New("Invalid index name") + ErrInvalidFieldName = errors.New("Invalid field name") + ErrInvalidLabel = errors.New("Invalid label") + ErrInvalidKey = errors.New("Invalid key") + ErrTriedMaxHosts = errors.New("Tried max hosts, still failing") + ErrAddrURIClusterExpected = errors.New("Addresses, URIs or a cluster is expected") + ErrInvalidQueryOption = errors.New("Invalid query option") + ErrInvalidIndexOption = errors.New("Invalid index option") + ErrInvalidFieldOption = errors.New("Invalid field option") + ErrNoFragmentNodes = errors.New("No fragment nodes") + ErrNoShard = errors.New("Index has no shards") + ErrUnknownType = errors.New("Unknown type") + ErrSingleServerAddressRequired = errors.New("OptClientManualServerAddress requires a single URI or address") + ErrPreconditionFailed = errors.New("Precondition failed") +) diff --git a/client/logimport.go b/client/logimport.go new file mode 100644 index 000000000..56202b3a0 --- /dev/null +++ b/client/logimport.go @@ -0,0 +1,45 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "encoding/gob" + "io" +) + +type importLog struct { + Index string + Path string + Shard uint64 + IsRoaring bool + Timestamp int64 // Unix Nanoseconds + Data []byte +} + +type encoder interface { + Encode(thing interface{}) error +} + +func newImportLogEncoder(w io.Writer) encoder { + return gob.NewEncoder(w) +} + +type decoder interface { + Decode(thing interface{}) error +} + +func newImportLogDecoder(r io.Reader) decoder { + return gob.NewDecoder(r) +} diff --git a/client/logimport_test.go b/client/logimport_test.go new file mode 100644 index 000000000..85ceab535 --- /dev/null +++ b/client/logimport_test.go @@ -0,0 +1,155 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" +) + +func TestEncodeDecode(t *testing.T) { + tests := []importLog{ + { + Index: "go-testindex", + Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false", + Shard: 0, + Data: make([]byte, 3918), + }, + { + Index: "go-testindex", + Path: "/index/go-testindex/field/importfield-batchsize/import?clear=false", + Shard: 0, + Data: make([]byte, 3918), + }, + { + Index: "eheh", + Path: "blah", + Shard: 9, + Data: []byte("something"), + }, + { + Index: "", + Path: "", + Shard: 0, + Data: nil, + }, + { + Index: "eheh", + Path: "blah", + Shard: 10, + Data: []byte("blahaslkdjfeoiwujf"), + }, + { + Index: "eheh", + Path: "blah", + Shard: 10, + Data: make([]byte, 10000), + }, + { + Index: "zoop", + Path: "blah", + Shard: 8923734, + Data: []byte("blahaslkdjfeoiwujf"), + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + nl := importLog{ + Index: test.Index, + Path: test.Path, + Shard: test.Shard, + Data: make([]byte, len(test.Data)), + } + copy(nl.Data, test.Data) + buf := &bytes.Buffer{} + enc := newImportLogEncoder(buf) + err := enc.Encode(nl) + if err != nil { + t.Fatalf("writing to buf: %v", err) + } + dec := newImportLogDecoder(buf) + l2 := &importLog{} + err = dec.Decode(l2) + if err != nil { + t.Fatalf("reading from buf: %v", err) + } + if l2.Index != test.Index { + t.Errorf("indexes not equal:\n%s\n%s", test.Index, l2.Index) + } + if l2.Path != test.Path { + t.Errorf("paths not equal:\n%s\n%s", test.Path, l2.Path) + } + if l2.Shard != test.Shard { + t.Errorf("shards not equal exp: %d got %d", test.Shard, l2.Shard) + } + if !reflect.DeepEqual(test.Data, l2.Data) { + t.Errorf("data not equal \n%v\n%v", test.Data, l2.Data) + } + + }) + } + buf, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("getting temp file: %v", err) + } + enc := newImportLogEncoder(buf) + for _, test := range tests { + a := &test + err := enc.Encode(a) + if err != nil { + t.Errorf("encoding to buf: %v", err) + } + } + + name := buf.Name() + err = buf.Close() + if err != nil { + t.Fatalf("closing temp file: %v", err) + } + + buf, err = os.Open(name) + if err != nil { + t.Fatalf("reopening: %v", err) + } + + dec := newImportLogDecoder(buf) + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + l := &importLog{} + err := dec.Decode(l) + // err := l.ReadFrom(buf) + if err != nil { + t.Errorf("reading from buf: %v", err) + } + if l.Index != test.Index { + t.Errorf("indexes not equal:\n%s\n%s", test.Index, l.Index) + } + if l.Path != test.Path { + t.Errorf("paths not equal:\n%s\n%s", test.Path, l.Path) + } + if l.Shard != test.Shard { + t.Errorf("shards not equal exp: %d got %d", test.Shard, l.Shard) + } + if !reflect.DeepEqual(test.Data, l.Data) { + t.Errorf("data not equal \n%v\n%v", test.Data, l.Data) + } + }) + } +} diff --git a/client/metrics.go b/client/metrics.go new file mode 100644 index 000000000..aaa14060b --- /dev/null +++ b/client/metrics.go @@ -0,0 +1,29 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +const ( + // MetricBatchImportDurationSeconds records the full time of the + // RecordBatch.Import call. This includes starting and finishing a + // transaction, doing key translation, building fragments locally, + // importing all data, and resetting internal structures. + MetricBatchImportDurationSeconds = "batch_import_duration_seconds" + + // MetricBatchFlushDurationSeconds records the full time for + // RecordBatch.Flush (if splitBatchMode is in use). This includes + // starting and finishing a transaction, importing all data, and + // resetting internal structures. + MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" +) diff --git a/client/orm.go b/client/orm.go new file mode 100644 index 000000000..f48c8e2dc --- /dev/null +++ b/client/orm.go @@ -0,0 +1,1613 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/pilosa/pilosa/v2/pql" + "github.com/pkg/errors" +) + +const timeFormat = "2006-01-02T15:04" + +// Schema contains the index properties +type Schema struct { + mu sync.RWMutex + indexes map[string]*Index +} + +func (s *Schema) String() string { + s.mu.RLock() + defer s.mu.RUnlock() + return fmt.Sprintf("%s", s.indexes) +} + +// NewSchema creates a new Schema +func NewSchema() *Schema { + return &Schema{ + indexes: make(map[string]*Index), + } +} + +// Index returns an index with a name. +func (s *Schema) Index(name string, options ...IndexOption) *Index { + s.mu.Lock() + defer s.mu.Unlock() + if index, ok := s.indexes[name]; ok { + return index + } + indexOptions := &IndexOptions{} + indexOptions.addOptions(options...) + return s.indexWithOptions(name, 0, 0, indexOptions) +} + +func (s *Schema) indexWithOptions(name string, createdAt int64, shardWidth uint64, options *IndexOptions) *Index { + index := NewIndex(name) + if createdAt != 0 { + index.createdAt = createdAt + } + + index.options = options.withDefaults() + index.shardWidth = shardWidth + if index.Opts().TrackExistence() { + index.Field("_exists") + } + s.indexes[name] = index + return index +} + +// Indexes return a copy of the indexes in this schema +func (s *Schema) Indexes() map[string]*Index { + s.mu.RLock() + defer s.mu.RUnlock() + result := make(map[string]*Index) + for k, v := range s.indexes { + result[k] = v.copy() + } + return result +} + +// HasIndex returns true if the given index is in the schema. +func (s *Schema) HasIndex(indexName string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.indexes[indexName] + return ok +} + +func (s *Schema) diff(other *Schema) *Schema { + result := NewSchema() + for indexName, index := range s.indexes { + if otherIndex, ok := other.indexes[indexName]; !ok { + // if the index doesn't exist in the other schema, simply copy it + result.indexes[indexName] = index.copy() + } else { + // the index exists in the other schema; check the fields + resultIndex := NewIndex(indexName) + for fieldName, field := range index.fields { + if _, ok := otherIndex.fields[fieldName]; !ok { + // the field doesn't exist in the other schema, copy it + resultIndex.fields[fieldName] = field.copy() + } + } + // check whether we modified result index + if len(resultIndex.fields) > 0 { + // if so, move it to the result + result.indexes[indexName] = resultIndex + } + } + } + return result +} + +type SerializedQuery interface { + String() string + HasWriteKeys() bool +} + +type serializedQuery struct { + query string + hasWriteKeys bool +} + +func newSerializedQuery(query string, hasWriteKeys bool) serializedQuery { + return serializedQuery{ + query: query, + hasWriteKeys: hasWriteKeys, + } +} + +func (s serializedQuery) String() string { + return s.query +} + +func (s serializedQuery) HasWriteKeys() bool { + return s.hasWriteKeys +} + +// PQLQuery is an interface for PQL queries. +type PQLQuery interface { + Index() *Index + Serialize() SerializedQuery + Error() error +} + +// PQLBaseQuery is the base implementation for PQLQuery. +type PQLBaseQuery struct { + index *Index + pql string + err error + hasKeys bool +} + +// NewPQLBaseQuery creates a new PQLQuery with the given PQL and index. +func NewPQLBaseQuery(pql string, index *Index, err error) *PQLBaseQuery { + var hasKeys bool + if index != nil { + hasKeys = index.options.keys + } + return &PQLBaseQuery{ + index: index, + pql: pql, + err: err, + hasKeys: hasKeys, + } +} + +// Index returns the index for this query +func (q *PQLBaseQuery) Index() *Index { + return q.index +} + +func (q *PQLBaseQuery) Serialize() SerializedQuery { + return newSerializedQuery(q.pql, q.hasKeys) +} + +// Error returns the error or nil for this query. +func (q PQLBaseQuery) Error() error { + return q.err +} + +// PQLRowQuery is the return type for row queries. +type PQLRowQuery struct { + index *Index + pql string + err error + hasKeys bool +} + +// Index returns the index for this query/ +func (q *PQLRowQuery) Index() *Index { + return q.index +} + +func (q *PQLRowQuery) Serialize() SerializedQuery { + return q.serialize() +} + +func (q *PQLRowQuery) serialize() SerializedQuery { + return newSerializedQuery(q.pql, q.hasKeys) +} + +// Error returns the error or nil for this query. +func (q PQLRowQuery) Error() error { + return q.err +} + +// PQLBatchQuery contains a batch of PQL queries. +// Use Index.BatchQuery function to create an instance. +// +// Usage: +// +// repo, err := NewIndex("repository") +// stargazer, err := repo.Field("stargazer") +// query := repo.BatchQuery( +// stargazer.Row(5), +// stargazer.Row(15), +// repo.Union(stargazer.Row(20), stargazer.Row(25))) +type PQLBatchQuery struct { + index *Index + queries []string + err error + hasKeys bool +} + +// Index returns the index for this query. +func (q *PQLBatchQuery) Index() *Index { + return q.index +} + +func (q *PQLBatchQuery) Serialize() SerializedQuery { + query := strings.Join(q.queries, "") + return newSerializedQuery(query, q.hasKeys) +} + +func (q *PQLBatchQuery) Error() error { + return q.err +} + +// Add adds a query to the batch. +func (q *PQLBatchQuery) Add(query PQLQuery) { + err := query.Error() + if err != nil { + q.err = err + } + serializedQuery := query.Serialize() + q.hasKeys = q.hasKeys || serializedQuery.HasWriteKeys() + q.queries = append(q.queries, serializedQuery.String()) +} + +// NewPQLRowQuery creates a new PqlRowQuery. +func NewPQLRowQuery(pql string, index *Index, err error) *PQLRowQuery { + return &PQLRowQuery{ + index: index, + pql: pql, + err: err, + hasKeys: index.options.keys, + } +} + +// IndexOptions contains options to customize Index objects. +type IndexOptions struct { + keys bool + keysSet bool + trackExistence bool + trackExistenceSet bool +} + +func (io *IndexOptions) withDefaults() (updated *IndexOptions) { + // copy options so the original is not updated + updated = &IndexOptions{} + *updated = *io + if !updated.keysSet { + updated.keys = false + } + if !updated.trackExistenceSet { + updated.trackExistence = true + } + return +} + +// Keys return true if this index has keys. +func (io IndexOptions) Keys() bool { + return io.keys +} + +// TrackExistence returns true if existence is tracked for this index. +func (io IndexOptions) TrackExistence() bool { + return io.trackExistence +} + +// String serializes this index to a JSON string. +func (io IndexOptions) String() string { + mopt := map[string]interface{}{} + if io.keysSet { + mopt["keys"] = io.keys + } + if io.trackExistenceSet { + mopt["trackExistence"] = io.trackExistence + } + return fmt.Sprintf(`{"options":%s}`, encodeMap(mopt)) +} + +func (io *IndexOptions) addOptions(options ...IndexOption) { + for _, option := range options { + if option == nil { + continue + } + option(io) + } +} + +// IndexOption is used to pass an option to Index function. +type IndexOption func(options *IndexOptions) + +// OptIndexKeys sets whether index uses string keys. +func OptIndexKeys(keys bool) IndexOption { + return func(options *IndexOptions) { + options.keys = keys + options.keysSet = true + } +} + +// OptIndexTrackExistence enables keeping track of existence of columns. +func OptIndexTrackExistence(trackExistence bool) IndexOption { + return func(options *IndexOptions) { + options.trackExistence = trackExistence + options.trackExistenceSet = true + } +} + +// OptionsOptions is used to pass an option to Option call. +type OptionsOptions struct { + columnAttrs bool + excludeColumns bool + excludeRowAttrs bool + shards []uint64 +} + +func (oo OptionsOptions) marshal() string { + part1 := fmt.Sprintf("columnAttrs=%s,excludeColumns=%s,excludeRowAttrs=%s", + strconv.FormatBool(oo.columnAttrs), + strconv.FormatBool(oo.excludeColumns), + strconv.FormatBool(oo.excludeRowAttrs)) + if oo.shards != nil { + shardsStr := make([]string, len(oo.shards)) + for i, shard := range oo.shards { + shardsStr[i] = strconv.FormatUint(shard, 10) + } + return fmt.Sprintf("%s,shards=[%s]", part1, strings.Join(shardsStr, ",")) + } + return part1 +} + +// OptionsOption is an option for Index.Options call. +type OptionsOption func(options *OptionsOptions) + +// OptOptionsColumnAttrs enables returning column attributes. +func OptOptionsColumnAttrs(enable bool) OptionsOption { + return func(options *OptionsOptions) { + options.columnAttrs = enable + } +} + +// OptOptionsExcludeColumns enables preventing returning columns. +func OptOptionsExcludeColumns(enable bool) OptionsOption { + return func(options *OptionsOptions) { + options.excludeColumns = enable + } +} + +// OptOptionsExcludeRowAttrs enables preventing returning row attributes. +func OptOptionsExcludeRowAttrs(enable bool) OptionsOption { + return func(options *OptionsOptions) { + options.excludeRowAttrs = enable + } +} + +// OptOptionsShards run the query using only the data from the given shards. +// By default, the entire data set (i.e. data from all shards) is used. +func OptOptionsShards(shards ...uint64) OptionsOption { + return func(options *OptionsOptions) { + options.shards = shards + } +} + +// Index is a Pilosa index. The purpose of the Index is to represent a data namespace. +// You cannot perform cross-index queries. Column-level attributes are global to the Index. +type Index struct { + mu sync.RWMutex + name string + createdAt int64 + options *IndexOptions + fields map[string]*Field + shardWidth uint64 +} + +func (idx *Index) String() string { + return fmt.Sprintf(`{name: "%s", options: "%s", fields: %s, shardWidth: %d}`, idx.name, idx.options, idx.fields, idx.shardWidth) +} + +// NewIndex creates an index with a name. +func NewIndex(name string) *Index { + options := &IndexOptions{} + return &Index{ + name: name, + options: options.withDefaults(), + fields: map[string]*Field{}, + } +} + +func (idx *Index) ShardWidth() uint64 { + return idx.shardWidth +} + +// Fields return a copy of the fields in this index +func (idx *Index) Fields() map[string]*Field { + idx.mu.Lock() + defer idx.mu.Unlock() + result := make(map[string]*Field) + for k, v := range idx.fields { + result[k] = v.copy() + } + return result +} + +// HasFields returns true if the given field exists in the index. +func (idx *Index) HasField(fieldName string) bool { + idx.mu.Lock() + defer idx.mu.Unlock() + _, ok := idx.fields[fieldName] + return ok +} + +func (idx *Index) copy() *Index { + idx.mu.Lock() + defer idx.mu.Unlock() + fields := make(map[string]*Field) + for name, f := range idx.fields { + fields[name] = f.copy() + } + index := &Index{ + name: idx.name, + createdAt: idx.createdAt, + options: &IndexOptions{}, + fields: fields, + shardWidth: idx.shardWidth, + } + *index.options = *idx.options + return index +} + +// Name returns the name of this index. +func (idx *Index) Name() string { + return idx.name +} + +func (idx *Index) CreatedAt() int64 { + idx.mu.RLock() + defer idx.mu.RUnlock() + return idx.createdAt +} + +// Opts returns the options of this index. +func (idx *Index) Opts() IndexOptions { + return *idx.options +} + +// Field creates a Field struct with the specified name and defaults. +func (idx *Index) Field(name string, options ...FieldOption) *Field { + idx.mu.Lock() + defer idx.mu.Unlock() + if field, ok := idx.fields[name]; ok { + return field + } + fieldOptions := &FieldOptions{} + fieldOptions = fieldOptions.withDefaults() + fieldOptions.addOptions(options...) + return idx.fieldWithOptions(name, 0, fieldOptions) +} + +func (idx *Index) fieldWithOptions(name string, createdAt int64, fieldOptions *FieldOptions) *Field { + field := newField(name, idx) + if createdAt != 0 { + field.createdAt = createdAt + } + fieldOptions = fieldOptions.withDefaults() + field.options = fieldOptions + idx.fields[name] = field + return field +} + +// BatchQuery creates a batch query with the given queries. +func (idx *Index) BatchQuery(queries ...PQLQuery) *PQLBatchQuery { + stringQueries := make([]string, 0, len(queries)) + hasKeys := false + for _, query := range queries { + serializedQuery := query.Serialize() + hasKeys = hasKeys || serializedQuery.HasWriteKeys() + stringQueries = append(stringQueries, serializedQuery.String()) + } + return &PQLBatchQuery{ + index: idx, + queries: stringQueries, + hasKeys: hasKeys, + } +} + +// RawQuery creates a query with the given string. +// Note that the query is not validated before sending to the server. +func (idx *Index) RawQuery(query string) *PQLBaseQuery { + q := NewPQLBaseQuery(query, idx, nil) + // NOTE: raw queries always assumed to have keys set + q.hasKeys = true + return q +} + +// Union creates a Union query. +// Union performs a logical OR on the results of each ROW_CALL query passed to it. +func (idx *Index) Union(rows ...*PQLRowQuery) *PQLRowQuery { + return idx.rowOperation("Union", rows...) +} + +// Intersect creates an Intersect query. +// Intersect performs a logical AND on the results of each ROW_CALL query passed to it. +func (idx *Index) Intersect(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 1 { + return NewPQLRowQuery("", idx, errors.New("Intersect operation requires at least 1 row")) + } + return idx.rowOperation("Intersect", rows...) +} + +// Difference creates an Intersect query. +// Difference returns all of the columns from the first ROW_CALL argument passed to it, without the columns from each subsequent ROW_CALL. +func (idx *Index) Difference(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 1 { + return NewPQLRowQuery("", idx, errors.New("Difference operation requires at least 1 row")) + } + return idx.rowOperation("Difference", rows...) +} + +// Xor creates an Xor query. +func (idx *Index) Xor(rows ...*PQLRowQuery) *PQLRowQuery { + if len(rows) < 2 { + return NewPQLRowQuery("", idx, errors.New("Xor operation requires at least 2 rows")) + } + return idx.rowOperation("Xor", rows...) +} + +// Not creates a Not query. +func (idx *Index) Not(row *PQLRowQuery) *PQLRowQuery { + return NewPQLRowQuery(fmt.Sprintf("Not(%s)", row.serialize()), idx, row.Error()) +} + +// Count creates a Count query. +// Returns the number of set columns in the ROW_CALL passed in. +func (idx *Index) Count(row *PQLRowQuery) *PQLBaseQuery { + serializedQuery := row.serialize() + q := NewPQLBaseQuery(fmt.Sprintf("Count(%s)", serializedQuery.String()), idx, nil) + q.hasKeys = q.hasKeys || serializedQuery.HasWriteKeys() + return q +} + +// All creates an All query. +// Returns the set columns with existence true. +func (idx *Index) All() *PQLRowQuery { + q := NewPQLRowQuery("All()", idx, nil) + return q +} + +// TODO: impelement AllLimit(limit, offset uint64) *PQLRowQuery + +// SetColumnAttrs creates a SetColumnAttrs query. +// SetColumnAttrs associates arbitrary key/value pairs with a column in an index. +// Following types are accepted: integer, float, string and boolean types. +func (idx *Index) SetColumnAttrs(colIDOrKey interface{}, attrs map[string]interface{}) *PQLBaseQuery { + colStr, err := formatIDKey(colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", idx, err) + } + attrsString, err := createAttributesString(attrs) + if err != nil { + return NewPQLBaseQuery("", idx, err) + } + q := fmt.Sprintf("SetColumnAttrs(%s,%s)", colStr, attrsString) + return NewPQLBaseQuery(q, idx, nil) +} + +// Options creates an Options query. +func (idx *Index) Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery { + oo := &OptionsOptions{} + for _, opt := range opts { + opt(oo) + } + text := fmt.Sprintf("Options(%s,%s)", row.serialize(), oo.marshal()) + return NewPQLBaseQuery(text, idx, nil) +} + +type groupByBuilder struct { + rows []*PQLRowsQuery + limit int64 + filter *PQLRowQuery + aggregate *PQLBaseQuery + having *PQLBaseQuery +} + +// GroupByBuilderOption is a functional option type for index.GroupBy +type GroupByBuilderOption func(g *groupByBuilder) error + +// OptGroupByBuilderRows is a functional option on groupByBuilder +// used to set the rows. +func OptGroupByBuilderRows(rows ...*PQLRowsQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.rows = rows + return nil + } +} + +// OptGroupByBuilderLimit is a functional option on groupByBuilder +// used to set the limit. +func OptGroupByBuilderLimit(l int64) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.limit = l + return nil + } +} + +// OptGroupByBuilderFilter is a functional option on groupByBuilder +// used to set the filter. +func OptGroupByBuilderFilter(q *PQLRowQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.filter = q + return nil + } +} + +// OptGroupByBuilderAggregate is a functional option on groupByBuilder +// used to set the aggregate. +func OptGroupByBuilderAggregate(agg *PQLBaseQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.aggregate = agg + return nil + } +} + +// OptGroupByBuilderHaving is a functional option on groupByBuilder +// used to set the having clause. +func OptGroupByBuilderHaving(having *PQLBaseQuery) GroupByBuilderOption { + return func(g *groupByBuilder) error { + g.having = having + return nil + } +} + +// GroupByBase creates a GroupBy query with the given functional options. +func (idx *Index) GroupByBase(opts ...GroupByBuilderOption) *PQLBaseQuery { + bldr := &groupByBuilder{} + for _, opt := range opts { + err := opt(bldr) + if err != nil { + return NewPQLBaseQuery("", idx, errors.Wrap(err, "applying option")) + } + } + + if len(bldr.rows) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if bldr.limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + + // rows + text := fmt.Sprintf("GroupBy(%s", strings.Join(serializeGroupBy(bldr.rows...), ",")) + + // limit + if bldr.limit > 0 { + text += fmt.Sprintf(",limit=%d", bldr.limit) + } + + // filter + if bldr.filter != nil { + filterText := bldr.filter.serialize().String() + text += fmt.Sprintf(",filter=%s", filterText) + } + + // aggregate + if bldr.aggregate != nil { + aggregateText := bldr.aggregate.Serialize().String() + text += fmt.Sprintf(",aggregate=%s", aggregateText) + } + + // having + if bldr.having != nil { + havingText := bldr.having.Serialize().String() + text += fmt.Sprintf(",having=%s", havingText) + } + + text += ")" + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupBy creates a GroupBy query with the given Rows queries +func (idx *Index) GroupBy(rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + text := fmt.Sprintf("GroupBy(%s)", strings.Join(serializeGroupBy(rowsQueries...), ",")) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByLimit creates a GroupBy query with the given limit and Rows queries +func (idx *Index) GroupByLimit(limit int64, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + text := fmt.Sprintf("GroupBy(%s,limit=%d)", strings.Join(serializeGroupBy(rowsQueries...), ","), limit) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByFilter creates a GroupBy query with the given filter and Rows queries +func (idx *Index) GroupByFilter(filterQuery *PQLRowQuery, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + filterText := filterQuery.serialize().String() + text := fmt.Sprintf("GroupBy(%s,filter=%s)", strings.Join(serializeGroupBy(rowsQueries...), ","), filterText) + return NewPQLBaseQuery(text, idx, nil) +} + +// GroupByLimitFilter creates a GroupBy query with the given filter and Rows queries +func (idx *Index) GroupByLimitFilter(limit int64, filterQuery *PQLRowQuery, rowsQueries ...*PQLRowsQuery) *PQLBaseQuery { + if len(rowsQueries) < 1 { + return NewPQLBaseQuery("", idx, errors.New("there should be at least one rows query")) + } + if limit < 0 { + return NewPQLBaseQuery("", idx, errors.New("limit must be non-negative")) + } + filterText := filterQuery.serialize().String() + text := fmt.Sprintf("GroupBy(%s,limit=%d,filter=%s)", strings.Join(serializeGroupBy(rowsQueries...), ","), limit, filterText) + return NewPQLBaseQuery(text, idx, nil) +} + +func (idx *Index) rowOperation(name string, rows ...*PQLRowQuery) *PQLRowQuery { + var err error + args := make([]string, 0, len(rows)) + for _, row := range rows { + if err = row.Error(); err != nil { + return NewPQLRowQuery("", idx, err) + } + args = append(args, row.serialize().String()) + } + query := NewPQLRowQuery(fmt.Sprintf("%s(%s)", name, strings.Join(args, ",")), idx, nil) + return query +} + +func serializeGroupBy(rowsQueries ...*PQLRowsQuery) []string { + qs := make([]string, 0, len(rowsQueries)) + for _, qry := range rowsQueries { + qs = append(qs, qry.serialize().String()) + } + return qs +} + +// FieldInfo represents schema information for a field. +type FieldInfo struct { + Name string `json:"name"` +} + +// FieldOptions contains options to customize Field objects and field queries. +type FieldOptions struct { + fieldType FieldType + timeQuantum TimeQuantum + cacheType CacheType + cacheSize int + min pql.Decimal + max pql.Decimal + scale int64 + keys bool + noStandardView bool + foreignIndex string +} + +// Type returns the type of the field. Currently "set", "int", or "time". +func (fo FieldOptions) Type() FieldType { + return fo.fieldType +} + +// TimeQuantum returns the configured time quantum for a time field. Empty +// string otherwise. +func (fo FieldOptions) TimeQuantum() TimeQuantum { + return fo.timeQuantum +} + +// CacheType returns the configured cache type for a "set" field. Empty string +// otherwise. +func (fo FieldOptions) CacheType() CacheType { + return fo.cacheType +} + +// CacheSize returns the cache size for a set field. Zero otherwise. +func (fo FieldOptions) CacheSize() int { + return fo.cacheSize +} + +// Min returns the minimum accepted value for an integer field. Zero otherwise. +func (fo FieldOptions) Min() pql.Decimal { + return fo.min +} + +// Max returns the maximum accepted value for an integer field. Zero otherwise. +func (fo FieldOptions) Max() pql.Decimal { + return fo.max +} + +// Scale returns the scale for a decimal field. +func (fo FieldOptions) Scale() int64 { + return fo.scale +} + +// Keys returns whether this field uses keys instead of IDs +func (fo FieldOptions) Keys() bool { + return fo.keys +} + +func (fo FieldOptions) ForeignIndex() string { + return fo.foreignIndex +} + +// NoStandardView suppresses creating the standard view for supported field types (currently, time) +func (fo FieldOptions) NoStandardView() bool { + return fo.noStandardView +} + +func (fo *FieldOptions) withDefaults() (updated *FieldOptions) { + // copy options so the original is not updated + updated = &FieldOptions{} + *updated = *fo + if updated.fieldType == "" { + updated.fieldType = FieldTypeSet + } + return +} + +func (fo FieldOptions) String() string { + mopt := map[string]interface{}{} + + switch fo.fieldType { + case FieldTypeSet, FieldTypeMutex: + if fo.cacheType != CacheTypeDefault { + mopt["cacheType"] = string(fo.cacheType) + } + if fo.cacheSize > 0 { + mopt["cacheSize"] = fo.cacheSize + } + case FieldTypeInt: + mopt["min"] = fo.min + mopt["max"] = fo.max + case FieldTypeDecimal: + mopt["min"] = fo.min + mopt["max"] = fo.max + mopt["scale"] = fo.scale + case FieldTypeTime: + mopt["timeQuantum"] = string(fo.timeQuantum) + mopt["noStandardView"] = fo.noStandardView + } + + if fo.fieldType != FieldTypeDefault { + mopt["type"] = string(fo.fieldType) + } + if fo.keys { + mopt["keys"] = fo.keys + } + if fo.foreignIndex != "" { + mopt["foreignIndex"] = fo.foreignIndex + } + return fmt.Sprintf(`{"options":%s}`, encodeMap(mopt)) +} + +func (fo *FieldOptions) addOptions(options ...FieldOption) { + for _, option := range options { + if option == nil { + continue + } + option(fo) + } +} + +// FieldOption is used to pass an option to index.Field function. +type FieldOption func(options *FieldOptions) + +// OptFieldTypeSet adds a set field. +// Specify CacheTypeDefault for the default cache type. +// Specify CacheSizeDefault for the default cache size. +func OptFieldTypeSet(cacheType CacheType, cacheSize int) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeSet + options.cacheType = cacheType + options.cacheSize = cacheSize + } +} + +// OptFieldTypeInt adds an integer field. +// No arguments: min = min_int, max = max_int +// 1 argument: min = limit[0], max = max_int +// 2 or more arguments: min = limit[0], max = limit[1] +func OptFieldTypeInt(limits ...int64) FieldOption { + min := pql.NewDecimal(math.MinInt64, 0) + max := pql.NewDecimal(math.MaxInt64, 0) + + if len(limits) > 2 { + panic("error: OptFieldTypeInt accepts at most 2 arguments") + } + if len(limits) > 0 { + min = pql.NewDecimal(limits[0], 0) + } + if len(limits) > 1 { + max = pql.NewDecimal(limits[1], 0) + } + + return func(options *FieldOptions) { + options.fieldType = FieldTypeInt + options.min = min + options.max = max + } +} + +// OptFieldTypeTime adds a time field. +func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeTime + options.timeQuantum = quantum + if len(opts) > 0 && opts[0] { + options.noStandardView = true + } + } +} + +// OptFieldTypeMutex adds a mutex field. +func OptFieldTypeMutex(cacheType CacheType, cacheSize int) FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeMutex + options.cacheType = cacheType + options.cacheSize = cacheSize + } +} + +// OptFieldTypeBool adds a bool field. +func OptFieldTypeBool() FieldOption { + return func(options *FieldOptions) { + options.fieldType = FieldTypeBool + } +} + +func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption { + min, max := pql.MinMax(scale) + if len(minmax) > 2 { + panic("error: OptFieldTypeDecimal accepts at most 2 arguments") + } + if len(minmax) > 0 { + min = minmax[0] + } + if len(minmax) > 1 { + max = minmax[1] + } + return func(options *FieldOptions) { + options.fieldType = FieldTypeDecimal + options.scale = scale + options.min = min + options.max = max + } +} + +// OptFieldKeys sets whether field uses string keys. +func OptFieldKeys(keys bool) FieldOption { + return func(options *FieldOptions) { + options.keys = keys + } +} + +func OptFieldForeignIndex(index string) FieldOption { + return func(options *FieldOptions) { + options.foreignIndex = index + } +} + +// Field structs are used to segment and define different functional characteristics within your entire index. +// You can think of a Field as a table-like data partition within your Index. +// Row-level attributes are namespaced at the Field level. +type Field struct { + name string + createdAt int64 + index *Index + options *FieldOptions +} + +func (f *Field) String() string { + return fmt.Sprintf(`{name: "%s", index: "%s", options: "%s"}`, f.name, f.index.name, f.options) +} + +func newField(name string, index *Index) *Field { + return &Field{ + name: name, + index: index, + options: &FieldOptions{}, + } +} + +// Name returns the name of the field +func (f *Field) Name() string { + return f.name +} + +func (f *Field) CreatedAt() int64 { + return f.createdAt +} + +// Opts returns the options of the field +func (f *Field) Opts() FieldOptions { + return *f.options +} + +func (f *Field) copy() *Field { + field := newField(f.name, f.index) + field.createdAt = f.createdAt + *field.options = *f.options + return field +} + +// Row creates a Row query. +// Row retrieves the indices of all the set columns in a row. +// It also retrieves any attributes set on that row or column. +func (f *Field) Row(rowIDOrKey interface{}) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Row(%s=%s)", f.name, rowStr) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// Set creates a Set query. +// Set, assigns a value of 1 to a bit in the binary matrix, thus associating the given row in the given field with the given column. +func (f *Field) Set(rowIDOrKey, colIDOrKey interface{}) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Set(%s,%s=%s)", colStr, f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// SetTimestamp creates a Set query with timestamp. +// Set, assigns a value of 1 to a column in the binary matrix, +// thus associating the given row in the given field with the given column. +func (f *Field) SetTimestamp(rowIDOrKey, colIDOrKey interface{}, timestamp time.Time) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Set(%s,%s=%s,%s)", colStr, f.name, rowStr, timestamp.Format(timeFormat)) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Clear creates a Clear query. +// Clear, assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given field from the given column. +func (f *Field) Clear(rowIDOrKey, colIDOrKey interface{}) *PQLBaseQuery { + rowStr, colStr, err := formatRowColIDKey(rowIDOrKey, colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("Clear(%s,%s=%s)", colStr, f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// ClearRow creates a ClearRow query. +// ClearRow sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns. +func (f *Field) ClearRow(rowIDOrKey interface{}) *PQLBaseQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("ClearRow(%s=%s)", f.name, rowStr) + q := NewPQLBaseQuery(text, f.index, nil) + return q +} + +// TopN creates a TopN query with the given item count. +// Returns the id and count of the top n rows (by count of columns) in the field. +func (f *Field) TopN(n uint64) *PQLRowQuery { + q := NewPQLRowQuery(fmt.Sprintf("TopN(%s,n=%d)", f.name, n), f.index, nil) + return q +} + +// RowTopN creates a TopN query with the given item count and row. +// This variant supports customizing the row query. +func (f *Field) RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery { + q := NewPQLRowQuery(fmt.Sprintf("TopN(%s,%s,n=%d)", + f.name, row.serialize(), n), f.index, nil) + return q +} + +// FilterAttrTopN creates a TopN query with the given item count, row, attribute name and filter values for that field +// The attrName and attrValues arguments work together to only return Rows which have the attribute specified by attrName with one of the values specified in attrValues. +func (f *Field) FilterAttrTopN(n uint64, row *PQLRowQuery, attrName string, attrValues ...interface{}) *PQLRowQuery { + return f.filterAttrTopN(n, row, attrName, attrValues...) +} + +func (f *Field) filterAttrTopN(n uint64, row *PQLRowQuery, field string, values ...interface{}) *PQLRowQuery { + if err := validateLabel(field); err != nil { + return NewPQLRowQuery("", f.index, err) + } + b, err := json.Marshal(values) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + var q *PQLRowQuery + if row == nil { + q = NewPQLRowQuery(fmt.Sprintf("TopN(%s,n=%d,attrName='%s',attrValues=%s)", + f.name, n, field, string(b)), f.index, nil) + } else { + serializedRow := row.serialize() + q = NewPQLRowQuery(fmt.Sprintf("TopN(%s,%s,n=%d,attrName='%s',attrValues=%s)", + f.name, serializedRow.String(), n, field, string(b)), f.index, nil) + } + return q +} + +// Range creates a Range query. +// Similar to Row, but only returns columns which were set with timestamps between the given start and end timestamps. +// *Deprecated at Pilosa 1.3* +func (f *Field) Range(rowIDOrKey interface{}, start time.Time, end time.Time) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Range(%s=%s,%s,%s)", f.name, rowStr, start.Format(timeFormat), end.Format(timeFormat)) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// RowRange creates a Row query with timestamps. +// Similar to Row, but only returns columns which were set with timestamps between the given start and end timestamps. +// *Introduced at Pilosa 1.3* +func (f *Field) RowRange(rowIDOrKey interface{}, start time.Time, end time.Time) *PQLRowQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLRowQuery("", f.index, err) + } + text := fmt.Sprintf("Row(%s=%s,from='%s',to='%s')", f.name, rowStr, start.Format(timeFormat), end.Format(timeFormat)) + q := NewPQLRowQuery(text, f.index, nil) + return q +} + +// SetRowAttrs creates a SetRowAttrs query. +// SetRowAttrs associates arbitrary key/value pairs with a row in a field. +// Following types are accepted: integer, float, string and boolean types. +func (f *Field) SetRowAttrs(rowIDOrKey interface{}, attrs map[string]interface{}) *PQLBaseQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + attrsString, err := createAttributesString(attrs) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + text := fmt.Sprintf("SetRowAttrs(%s,%s,%s)", f.name, rowStr, attrsString) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Store creates a Store call. +// Store writes the result of the row query to the specified row. If the row already exists, it will be replaced. The destination field must be of field type set. +func (f *Field) Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + return NewPQLBaseQuery(fmt.Sprintf("Store(%s,%s=%s)", row.serialize().String(), f.name, rowStr), f.index, nil) +} + +func createAttributesString(attrs map[string]interface{}) (string, error) { + attrsList := make([]string, 0, len(attrs)) + for k, v := range attrs { + // TODO: validate the type of v is one of string, int64, float64, bool + if err := validateLabel(k); err != nil { + return "", err + } + if vs, ok := v.(string); ok { + attrsList = append(attrsList, fmt.Sprintf("%s=%s", k, strconv.Quote(vs))) + } else { + attrsList = append(attrsList, fmt.Sprintf("%s=%v", k, v)) + } + } + sort.Strings(attrsList) + return strings.Join(attrsList, ","), nil +} + +func formatIDKey(idKey interface{}) (string, error) { + switch v := idKey.(type) { + case uint: + return strconv.FormatUint(uint64(v), 10), nil + case uint32: + return strconv.FormatUint(uint64(v), 10), nil + case uint64: + return strconv.FormatUint(v, 10), nil + case int: + return strconv.FormatInt(int64(v), 10), nil + case int32: + return strconv.FormatInt(int64(v), 10), nil + case int64: + return strconv.FormatInt(v, 10), nil + case string: + v = strings.ReplaceAll(v, `\`, `\\`) + return fmt.Sprintf(`'%s'`, strings.ReplaceAll(v, `'`, `\'`)), nil + default: + return "", errors.Errorf("id/key is not a string or integer type: %#v", idKey) + } +} + +func formatIDKeyBool(idKeyBool interface{}) (string, error) { + if b, ok := idKeyBool.(bool); ok { + return strconv.FormatBool(b), nil + } + if flt, ok := idKeyBool.(float64); ok { + return fmt.Sprintf("%f", flt), nil + } + return formatIDKey(idKeyBool) +} + +func formatRowColIDKey(rowIDOrKey, colIDOrKey interface{}) (string, string, error) { + rowStr, err := formatIDKeyBool(rowIDOrKey) + if err != nil { + return "", "", errors.Wrap(err, "formatting row") + } + colStr, err := formatIDKey(colIDOrKey) + if err != nil { + return "", "", errors.Wrap(err, "formatting column") + } + return rowStr, colStr, err +} + +// FieldType is the type of a field. +// See: https://www.pilosa.com/docs/latest/data-model/#field-type +type FieldType string + +const ( + // FieldTypeDefault is the default field type. + FieldTypeDefault FieldType = "" + // FieldTypeSet is the set field type. + // See: https://www.pilosa.com/docs/latest/data-model/#set + FieldTypeSet FieldType = "set" + // FieldTypeInt is the int field type. + // See: https://www.pilosa.com/docs/latest/data-model/#int + FieldTypeInt FieldType = "int" + // FieldTypeTime is the time field type. + // See: https://www.pilosa.com/docs/latest/data-model/#time + FieldTypeTime FieldType = "time" + // FieldTypeMutex is the mutex field type. + // See: https://www.pilosa.com/docs/latest/data-model/#mutex + FieldTypeMutex FieldType = "mutex" + // FieldTypeBool is the boolean field type. + // See: https://www.pilosa.com/docs/latest/data-model/#boolean + FieldTypeBool FieldType = "bool" + // FieldTypeDecimal can store floating point numbers as integers + // with a scale factor. This field type is only available in + // Molecula's Pilosa with enterprise extensions. + FieldTypeDecimal FieldType = "decimal" +) + +// TimeQuantum type represents valid time quantum values time fields. +type TimeQuantum string + +// TimeQuantum constants +const ( + TimeQuantumNone TimeQuantum = "" + TimeQuantumYear TimeQuantum = "Y" + TimeQuantumMonth TimeQuantum = "M" + TimeQuantumDay TimeQuantum = "D" + TimeQuantumHour TimeQuantum = "H" + TimeQuantumYearMonth TimeQuantum = "YM" + TimeQuantumMonthDay TimeQuantum = "MD" + TimeQuantumDayHour TimeQuantum = "DH" + TimeQuantumYearMonthDay TimeQuantum = "YMD" + TimeQuantumMonthDayHour TimeQuantum = "MDH" + TimeQuantumYearMonthDayHour TimeQuantum = "YMDH" +) + +// CacheType represents cache type for a field +type CacheType string + +// CacheType constants +const ( + CacheTypeDefault CacheType = "" + CacheTypeLRU CacheType = "lru" + CacheTypeRanked CacheType = "ranked" + CacheTypeNone CacheType = "none" +) + +// CacheSizeDefault is the default cache size +const CacheSizeDefault = 0 + +// Options returns the options set for the field. Which fields of the +// FieldOptions struct are actually being used depends on the field's type. +// *DEPRECATED* +func (f *Field) Options() *FieldOptions { + return f.options +} + +type IntOrFloat interface{} + +type intOrFloatVal struct { + IntOrFloat +} + +func (i intOrFloatVal) String() string { + switch i.IntOrFloat.(type) { + case float64: + // In order to test expected values, we set the precision + // to 8. TODO: It's likely we'll need to address this + // at some point. + return fmt.Sprintf("%.8f", i.IntOrFloat) + default: + return fmt.Sprintf("%d", i.IntOrFloat) + } +} + +// LT creates a less than query. +func (f *Field) LT(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("<", n) +} + +// LTE creates a less than or equal query. +func (f *Field) LTE(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("<=", n) +} + +// GT creates a greater than query. +func (f *Field) GT(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation(">", n) +} + +// GTE creates a greater than or equal query. +func (f *Field) GTE(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation(">=", n) +} + +// Equals creates an equals query. +func (f *Field) Equals(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("==", n) +} + +// NotEquals creates a not equals query. +func (f *Field) NotEquals(n IntOrFloat) *PQLRowQuery { + return f.binaryOperation("!=", n) +} + +// NotNull creates a not equal to null query. +func (f *Field) NotNull() *PQLRowQuery { + text := fmt.Sprintf("Row(%s != null)", f.name) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Between creates a between query. +func (f *Field) Between(a IntOrFloat, b IntOrFloat) *PQLRowQuery { + text := fmt.Sprintf("Row(%s >< [%s,%s])", f.name, intOrFloatVal{a}, intOrFloatVal{b}) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +// Sum creates a sum query. +func (f *Field) Sum(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Sum", row) +} + +// Min creates a min query. +func (f *Field) Min(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Min", row) +} + +// Max creates a max query. +func (f *Field) Max(row *PQLRowQuery) *PQLBaseQuery { + return f.valQuery("Max", row) +} + +// MinRow creates a min row query. +func (f *Field) MinRow() *PQLBaseQuery { + q := fmt.Sprintf("MinRow(field='%s')", f.name) + return NewPQLBaseQuery(q, f.index, nil) +} + +// MaxRow creates a max row query. +func (f *Field) MaxRow() *PQLBaseQuery { + q := fmt.Sprintf("MaxRow(field='%s')", f.name) + return NewPQLBaseQuery(q, f.index, nil) +} + +// SetIntValue creates a Set query. +func (f *Field) SetIntValue(colIDOrKey interface{}, value int) *PQLBaseQuery { + colStr, err := formatIDKey(colIDOrKey) + if err != nil { + return NewPQLBaseQuery("", f.index, err) + } + q := fmt.Sprintf("Set(%s, %s=%d)", colStr, f.name, value) + return NewPQLBaseQuery(q, f.index, nil) +} + +// PQLRowsQuery is the return type for Rows calls. +type PQLRowsQuery struct { + index *Index + pql string + err error +} + +// NewPQLRowsQuery creates a new PQLRowsQuery. +func NewPQLRowsQuery(pql string, index *Index, err error) *PQLRowsQuery { + return &PQLRowsQuery{ + index: index, + pql: pql, + err: err, + } +} + +// Index returns the index for this query/ +func (q *PQLRowsQuery) Index() *Index { + return q.index +} + +func (q *PQLRowsQuery) Serialize() SerializedQuery { + return q.serialize() +} + +func (q *PQLRowsQuery) serialize() SerializedQuery { + return newSerializedQuery(q.pql, false) +} + +// Error returns the error or nil for this query. +func (q PQLRowsQuery) Error() error { + return q.err +} + +// Union returns the union of all matched rows. +func (q *PQLRowsQuery) Union() *PQLRowQuery { + return NewPQLRowQuery(fmt.Sprintf("UnionRows(%s)", q.serialize().String()), q.index, nil) +} + +// Rows creates a Rows query with defaults +func (f *Field) Rows() *PQLRowsQuery { + text := fmt.Sprintf("Rows(field='%s')", f.name) + return NewPQLRowsQuery(text, f.index, nil) +} + +// Like creates a Rows query filtered by a pattern. +// An underscore ('_') can be used as a placeholder for a single UTF-8 codepoint or a percent sign ('%') can be used as a placeholder for 0 or more codepoints. +// All other codepoints in the pattern are matched exactly. +func (f *Field) Like(pattern string) *PQLRowsQuery { + pattern = strings.ReplaceAll(pattern, `\`, `\\`) + pattern = strings.ReplaceAll(pattern, `'`, `\'`) + text := fmt.Sprintf("Rows(field='%s',like='%s')", f.name, pattern) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPrevious creates a Rows query with the given previous row ID/key +func (f *Field) RowsPrevious(rowIDOrKey interface{}) *PQLRowsQuery { + idKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s)", f.name, idKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsLimit creates a Rows query with the given limit +func (f *Field) RowsLimit(limit int64) *PQLRowsQuery { + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + text := fmt.Sprintf("Rows(field='%s',limit=%d)", f.name, limit) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsColumn creates a Rows query with the given column ID/key +func (f *Field) RowsColumn(columnIDOrKey interface{}) *PQLRowsQuery { + idKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',column=%s)", f.name, idKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousLimit creates a Rows query with the given previous row ID/key and limit +func (f *Field) RowsPreviousLimit(rowIDOrKey interface{}, limit int64) *PQLRowsQuery { + idKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,limit=%d)", f.name, idKey, limit) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousColumn creates a Rows query with the given previous row ID/key and column ID/key +func (f *Field) RowsPreviousColumn(rowIDOrKey interface{}, columnIDOrKey interface{}) *PQLRowsQuery { + rowIDKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,column=%s)", f.name, rowIDKey, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsLimitColumn creates a Row query with the given limit and column ID/key +func (f *Field) RowsLimitColumn(limit int64, columnIDOrKey interface{}) *PQLRowsQuery { + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',limit=%d,column=%s)", f.name, limit, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// RowsPreviousLimitColumn creates a Row query with the given previous row ID/key, limit and column ID/key +func (f *Field) RowsPreviousLimitColumn(rowIDOrKey interface{}, limit int64, columnIDOrKey interface{}) *PQLRowsQuery { + rowIDKey, err := formatIDKey(rowIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + if limit < 0 { + return NewPQLRowsQuery("", f.index, errors.New("rows limit must be non-negative")) + } + columnIDKey, err := formatIDKey(columnIDOrKey) + if err != nil { + return NewPQLRowsQuery("", f.index, err) + } + text := fmt.Sprintf("Rows(field='%s',previous=%s,limit=%d,column=%s)", f.name, rowIDKey, limit, columnIDKey) + return NewPQLRowsQuery(text, f.index, nil) +} + +// Distinct creates a Distinct query. +func (f *Field) Distinct() *PQLRowQuery { + text := fmt.Sprintf("Distinct(Row(%s!=null),index='%s',field='%s')", f.name, f.index.Name(), f.name) + return NewPQLRowQuery(text, f.index, nil) +} + +// RowDistinct creates a Distinct query with the given row filter. +func (f *Field) RowDistinct(row *PQLRowQuery) *PQLRowQuery { + text := fmt.Sprintf("Distinct(%s,index='%s',field='%s')", row.serialize(), f.index.Name(), f.name) + return NewPQLRowQuery(text, f.index, nil) +} + +func (f *Field) binaryOperation(op string, n IntOrFloat) *PQLRowQuery { + text := fmt.Sprintf("Row(%s %s %s)", f.name, op, intOrFloatVal{n}) + q := NewPQLRowQuery(text, f.index, nil) + q.hasKeys = f.options.keys || f.index.options.keys + return q +} + +func (f *Field) valQuery(op string, row *PQLRowQuery) *PQLBaseQuery { + rowStr := "" + hasKeys := f.options.keys || f.index.options.keys + if row != nil { + serializedRow := row.serialize() + hasKeys = hasKeys || serializedRow.HasWriteKeys() + rowStr = fmt.Sprintf("%s,", serializedRow.String()) + } + text := fmt.Sprintf("%s(%sfield='%s')", op, rowStr, f.name) + q := NewPQLBaseQuery(text, f.index, nil) + q.hasKeys = hasKeys + return q +} + +func encodeMap(m map[string]interface{}) string { + result, err := json.Marshal(m) + if err != nil { + panic(err) + } + return string(result) +} diff --git a/client/orm_test.go b/client/orm_test.go new file mode 100644 index 000000000..57a7d128f --- /dev/null +++ b/client/orm_test.go @@ -0,0 +1,1242 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "fmt" + "math" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/pilosa/pilosa/v2/pql" + "github.com/pkg/errors" +) + +func TestORM(t *testing.T) { + var schema = NewSchema() + var sampleIndex = schema.Index("sample-index") + var sampleField = sampleIndex.Field("sample-field") + var projectIndex = schema.Index("project-index") + var collabField = projectIndex.Field("collaboration") + var b1 = sampleField.Row(10) + var b2 = sampleField.Row(20) + var b3 = sampleField.Row(42) + var b4 = collabField.Row(2) + + t.Run("SchemaDiff", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1") + index11.Field("field1-1") + index11.Field("field1-2") + index12 := schema1.Index("diff-index2", OptIndexKeys(true), OptIndexTrackExistence(false)) + index12.Field("field2-1") + + schema2 := NewSchema() + index21 := schema2.Index("diff-index1") + index21.Field("another-field") + + targetDiff12 := NewSchema() + targetIndex1 := targetDiff12.Index("diff-index1", OptIndexTrackExistence(false)) + targetIndex1.Field("field1-1") + targetIndex1.Field("field1-2") + targetIndex2 := targetDiff12.Index("diff-index2", OptIndexKeys(true), OptIndexTrackExistence(false)) + targetIndex2.Field("field2-1") + targetIndex1.options = &IndexOptions{} + + diff12 := schema1.diff(schema2) + + strTargetDiff12 := fmt.Sprintf("%+v", targetDiff12.indexes) + strDiff12 := fmt.Sprintf("%+v", diff12.indexes) + if strDiff12 != strTargetDiff12 { + t.Fatalf("The diff must be correctly calculated, but exp/got\n%s\n%s", strTargetDiff12, strDiff12) + } + }) + + t.Run("SchemaIndexes", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1") + index12 := schema1.Index("diff-index2") + indexes := schema1.Indexes() + target := map[string]*Index{ + "diff-index1": index11, + "diff-index2": index12, + } + if !reflect.DeepEqual(target, indexes) { + t.Fatalf("calling schema.Indexes should return indexes") + } + }) + + t.Run("SchemaToString", func(t *testing.T) { + schema1 := NewSchema() + _ = schema1.Index("test-index") + target := `map[test-index:{name: "test-index", options: "{"options":{}}", fields: map[_exists:{name: "_exists", index: "test-index", options: "{"options":{"type":"set"}}"}], shardWidth: 0}]` + if target != schema1.String() { + t.Fatalf("%s != %s", target, schema1.String()) + } + }) + + t.Run("NewIndex", func(t *testing.T) { + index1 := schema.Index("index-name") + if index1.Name() != "index-name" { + t.Fatalf("index name was not set") + } + // calling schema.Index again should return the same index + index2 := schema.Index("index-name") + if index1 != index2 { + t.Fatalf("calling schema.Index again should return the same index") + } + if !schema.HasIndex("index-name") { + t.Fatalf("HasIndex should return true") + } + if schema.HasIndex("index-x") { + t.Fatalf("HasIndex should return false") + } + }) + + t.Run("NewIndexCopy", func(t *testing.T) { + index := schema.Index("my-index-4copy", OptIndexKeys(true)) + index.Field("my-field-4copy", OptFieldTypeTime(TimeQuantumDayHour)) + copiedIndex := index.copy() + if !reflect.DeepEqual(index, copiedIndex) { + t.Fatalf("copied index should be equivalent") + } + }) + + t.Run("NewIndexOptions", func(t *testing.T) { + schema := NewSchema() + // test the defaults + index := schema.Index("index-default-options") + target := `{"options":{}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + + index = schema.Index("index-keys", OptIndexKeys(true)) + if true != index.Opts().Keys() { + t.Fatalf("index keys %v != %v", true, index.Opts().Keys()) + } + target = `{"options":{"keys":true}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + + index = schema.Index("index-trackexistence", OptIndexTrackExistence(false)) + if false != index.Opts().TrackExistence() { + t.Fatalf("index trackExistene %v != %v", true, index.Opts().TrackExistence()) + } + target = `{"options":{"trackExistence":false}}` + if target != index.options.String() { + t.Fatalf("%s != %s", target, index.options.String()) + } + }) + + t.Run("NilIndexOption", func(t *testing.T) { + schema.Index("index-with-nil-option", nil) + }) + + t.Run("IndexFields", func(t *testing.T) { + schema1 := NewSchema() + index11 := schema1.Index("diff-index1", OptIndexTrackExistence(false)) + field11 := index11.Field("field1-1") + field12 := index11.Field("field1-2") + fields := index11.Fields() + target := map[string]*Field{ + "field1-1": field11, + "field1-2": field12, + } + if !reflect.DeepEqual(target, fields) { + t.Fatalf("calling index.Fields should return fields") + } + if !index11.HasField("field1-1") { + t.Fatalf("HasField should return true") + } + if index11.HasField("field-x") { + t.Fatalf("HasField should return false") + } + }) + + t.Run("IndexToString", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + target := `{name: "test-index", options: "{"options":{}}", fields: map[_exists:{name: "_exists", index: "test-index", options: "{"options":{"type":"set"}}"}], shardWidth: 0}` + if target != index.String() { + t.Fatalf("indexes not equal exp/got:\n%s\n%s", target, index.String()) + } + }) + + t.Run("Field", func(t *testing.T) { + field1 := sampleIndex.Field("nonexistent-field") + field2 := sampleIndex.Field("nonexistent-field") + if field1 != field2 { + t.Fatalf("calling index.Field again should return the same field") + } + if field1.Name() != "nonexistent-field" { + t.Fatalf("calling field.Name should return field's name") + } + }) + + t.Run("FieldCopy", func(t *testing.T) { + field := sampleIndex.Field("my-field-4copy", OptFieldTypeSet(CacheTypeRanked, 123456)) + copiedField := field.copy() + if !reflect.DeepEqual(field, copiedField) { + t.Fatalf("copied field should be equivalent") + } + }) + + t.Run("FieldToString", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + field := index.Field("test-field") + target := `{name: "test-field", index: "test-index", options: "{"options":{"type":"set"}}"}` + if target != field.String() { + t.Fatalf("%s != %s", target, field.String()) + } + }) + + t.Run("NilFieldOption", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + index.Field("test-field-with-nil-option", nil) + }) + + t.Run("FieldSetType", func(t *testing.T) { + schema1 := NewSchema() + index := schema1.Index("test-index") + field := index.Field("test-set-field", OptFieldTypeSet(CacheTypeLRU, 1000), OptFieldKeys(true)) + target := `{"options":{"type":"set","cacheType":"lru","cacheSize":1000,"keys":true}}` + if sortedString(target) != sortedString(field.options.String()) { + t.Fatalf("%s != %s", target, field.options.String()) + } + + field = index.Field("test-set-field2", OptFieldTypeSet(CacheTypeLRU, -10), OptFieldKeys(true)) + target = `{"options":{"type":"set","cacheType":"lru","keys":true}}` + if sortedString(target) != sortedString(field.options.String()) { + t.Fatalf("%s != %s", target, field.options.String()) + } + }) + + t.Run("Row", func(t *testing.T) { + comparePQL(t, + "Row(collaboration=5)", + collabField.Row(5)) + + comparePQL(t, + "Row(collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", + collabField.Row("b7feb014-8ea7-49a8-9cd8-19709161ab63")) + + q := collabField.Row(nil) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Set", func(t *testing.T) { + comparePQL(t, + "Set(10,collaboration=5)", + collabField.Set(5, 10)) + + comparePQL(t, + `Set('some_id',collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')`, + collabField.Set("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id")) + + q := collabField.Set(nil, 10) + if q.err == nil { + t.Fatalf("should have failed") + } + q = collabField.Set(5, false) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Timestamp", func(t *testing.T) { + timestamp := time.Date(2017, time.April, 24, 12, 14, 0, 0, time.UTC) + comparePQL(t, + "Set(20,collaboration=10,2017-04-24T12:14)", + collabField.SetTimestamp(10, 20, timestamp)) + + comparePQL(t, + "Set('mycol',collaboration='myrow',2017-04-24T12:14)", + collabField.SetTimestamp("myrow", "mycol", timestamp)) + + q := collabField.SetTimestamp(nil, 20, timestamp) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Clear", func(t *testing.T) { + comparePQL(t, + "Clear(10,collaboration=5)", + collabField.Clear(5, 10)) + + comparePQL(t, + "Clear('some_id',collaboration='b7feb014-8ea7-49a8-9cd8-19709161ab63')", + collabField.Clear("b7feb014-8ea7-49a8-9cd8-19709161ab63", "some_id")) + comparePQL(t, + `Clear('bill\'s',collaboration='will\'s')`, + collabField.Clear("will's", "bill's")) + + q := collabField.Clear(nil, 10) + if q.err == nil { + t.Fatalf("should have failed") + } + q = collabField.Clear(5, false) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("ClearRow", func(t *testing.T) { + comparePQL(t, + "ClearRow(collaboration=5)", + collabField.ClearRow(5)) + + comparePQL(t, + "ClearRow(collaboration='five')", + collabField.ClearRow("five")) + + comparePQL(t, + "ClearRow(collaboration=true)", + collabField.ClearRow(true)) + + q := collabField.ClearRow(nil) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Union", func(t *testing.T) { + comparePQL(t, + "Union(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Union(b1, b2)) + comparePQL(t, + "Union(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Union(b1, b2, b3)) + comparePQL(t, + "Union(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Union(b1, b4)) + comparePQL(t, + "Union(Row(sample-field=10))", + sampleIndex.Union(b1)) + comparePQL(t, + "Union()", + sampleIndex.Union()) + }) + + t.Run("Intersect", func(t *testing.T) { + comparePQL(t, + "Intersect(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Intersect(b1, b2)) + comparePQL(t, + "Intersect(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Intersect(b1, b2, b3)) + comparePQL(t, + "Intersect(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Intersect(b1, b4)) + comparePQL(t, + "Intersect(Row(sample-field=10))", + sampleIndex.Intersect(b1)) + }) + + t.Run("Difference", func(t *testing.T) { + comparePQL(t, + "Difference(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Difference(b1, b2)) + comparePQL(t, + "Difference(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Difference(b1, b2, b3)) + comparePQL(t, + "Difference(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Difference(b1, b4)) + comparePQL(t, + "Difference(Row(sample-field=10))", + sampleIndex.Difference(b1)) + }) + + t.Run("Xor", func(t *testing.T) { + comparePQL(t, + "Xor(Row(sample-field=10),Row(sample-field=20))", + sampleIndex.Xor(b1, b2)) + comparePQL(t, + "Xor(Row(sample-field=10),Row(sample-field=20),Row(sample-field=42))", + sampleIndex.Xor(b1, b2, b3)) + comparePQL(t, + "Xor(Row(sample-field=10),Row(collaboration=2))", + sampleIndex.Xor(b1, b4)) + }) + + t.Run("Not", func(t *testing.T) { + comparePQL(t, + "Not(Row(sample-field=10))", + sampleIndex.Not(sampleField.Row(10))) + }) + + t.Run("TopN", func(t *testing.T) { + comparePQL(t, + "TopN(collaboration,n=27)", + collabField.TopN(27)) + comparePQL(t, + "TopN(collaboration,Row(collaboration=3),n=10)", + collabField.RowTopN(10, collabField.Row(3))) + comparePQL(t, + "TopN(sample-field,Row(collaboration=7),n=12,attrName='category',attrValues=[80,81])", + sampleField.FilterAttrTopN(12, collabField.Row(7), "category", 80, 81)) + comparePQL(t, + "TopN(sample-field,n=12,attrName='category',attrValues=[80,81])", + sampleField.FilterAttrTopN(12, nil, "category", 80, 81)) + }) + + t.Run("FieldLT", func(t *testing.T) { + comparePQL(t, + "Row(collaboration < 10)", + collabField.LT(10)) + comparePQL(t, + "Row(collaboration < 10.12300000)", + collabField.LT(10.123)) + }) + + t.Run("FieldLTE", func(t *testing.T) { + comparePQL(t, + "Row(collaboration <= 10)", + collabField.LTE(10)) + comparePQL(t, + "Row(collaboration <= 10.12300000)", + collabField.LTE(10.123)) + }) + + t.Run("FieldGT", func(t *testing.T) { + comparePQL(t, + "Row(collaboration > 10)", + collabField.GT(10)) + comparePQL(t, + "Row(collaboration > 10.12300000)", + collabField.GT(10.123)) + }) + + t.Run("FieldGTE", func(t *testing.T) { + comparePQL(t, + "Row(collaboration >= 10)", + collabField.GTE(10)) + comparePQL(t, + "Row(collaboration >= 10.12300000)", + collabField.GTE(10.123)) + }) + + t.Run("FieldEQ", func(t *testing.T) { + comparePQL(t, + "Row(collaboration == 10)", + collabField.Equals(10)) + comparePQL(t, + "Row(collaboration == 10.12300000)", + collabField.Equals(10.123)) + }) + + t.Run("FieldNEQ", func(t *testing.T) { + comparePQL(t, + "Row(collaboration != 10)", + collabField.NotEquals(10)) + comparePQL(t, + "Row(collaboration != 10.12300000)", + collabField.NotEquals(10.123)) + }) + + t.Run("FieldNotNull", func(t *testing.T) { + comparePQL(t, + "Row(collaboration != null)", + collabField.NotNull()) + }) + + t.Run("FieldBetween", func(t *testing.T) { + comparePQL(t, + "Row(collaboration >< [10,20])", + collabField.Between(10, 20)) + comparePQL(t, + "Row(collaboration >< [10.12300000,20.45600000])", + collabField.Between(10.123, 20.456)) + }) + + t.Run("FieldSum", func(t *testing.T) { + comparePQL(t, + "Sum(Row(collaboration=10),field='collaboration')", + collabField.Sum(collabField.Row(10))) + comparePQL(t, + "Sum(field='collaboration')", + collabField.Sum(nil)) + }) + + t.Run("FieldMinRow", func(t *testing.T) { + comparePQL(t, + "MinRow(field='sample-field')", + sampleField.MinRow()) + }) + + t.Run("FieldMaxRow", func(t *testing.T) { + comparePQL(t, + "MaxRow(field='sample-field')", + sampleField.MaxRow()) + }) + + t.Run("FieldSetValue", func(t *testing.T) { + comparePQL(t, + "Set(50, collaboration=15)", + collabField.SetIntValue(50, 15)) + + comparePQL(t, + "Set('mycol', sample-field=22)", + sampleField.SetIntValue("mycol", 22)) + + q := sampleField.SetIntValue(false, 22) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("FilterFieldTopNInvalidField", func(t *testing.T) { + q := sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("FilterFieldTopNInvalidValue", func(t *testing.T) { + q := sampleField.FilterAttrTopN(12, collabField.Row(7), "category", 80, func() {}) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowOperationInvalidArg", func(t *testing.T) { + invalid := sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81) + // invalid argument in pos 1 + q := sampleIndex.Union(invalid, b1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // invalid argument in pos 2 + q = sampleIndex.Intersect(b1, invalid) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // invalid argument in pos 3 + q = sampleIndex.Intersect(b1, b2, invalid) + if q.Error() == nil { + t.Fatalf("should have failed") + } + // not enough rows supplied + q = sampleIndex.Difference() + if q.Error() == nil { + t.Fatalf("should have failed") + } + // not enough rows supplied + q = sampleIndex.Intersect() + if q.Error() == nil { + t.Fatalf("should have failed") + } + + // not enough rows supplied + q = sampleIndex.Xor(b1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("SetColumnAttrs", func(t *testing.T) { + attrs := map[string]interface{}{ + "quote": "\"Don't worry, be happy\"", + "happy": true, + } + comparePQL(t, + "SetColumnAttrs(5,happy=true,quote=\"\\\"Don't worry, be happy\\\"\")", + projectIndex.SetColumnAttrs(5, attrs)) + + q := projectIndex.SetColumnAttrs(false, attrs) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("SetColumnAttrsInvalidAttr", func(t *testing.T) { + attrs := map[string]interface{}{ + "color": "blue", + "$invalid$": true, + } + if projectIndex.SetColumnAttrs(5, attrs).Error() == nil { + t.Fatalf("Should have failed") + } + }) + + t.Run("SetRowAttrs", func(t *testing.T) { + attrs := map[string]interface{}{ + "quote": "\"Don't worry, be happy\"", + "active": true, + } + comparePQL(t, + `SetRowAttrs(collaboration,5,active=true,quote="\"Don't worry, be happy\"")`, + collabField.SetRowAttrs(5, attrs)) + + comparePQL(t, + "SetRowAttrs(collaboration,'foo',active=true,quote=\"\\\"Don't worry, be happy\\\"\")", + collabField.SetRowAttrs("foo", attrs)) + + q := collabField.SetRowAttrs(nil, attrs) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("SetRowAttrsInvalidAttr", func(t *testing.T) { + attrs := map[string]interface{}{ + "color": "blue", + "$invalid$": true, + } + if collabField.SetRowAttrs(5, attrs).Error() == nil { + t.Fatalf("Should have failed") + } + + if collabField.SetRowAttrs("foo", attrs).Error() == nil { + t.Fatalf("Should have failed") + } + }) + + t.Run("Store", func(t *testing.T) { + comparePQL(t, + "Store(Row(collaboration=5),sample-field=10)", + sampleField.Store(collabField.Row(5), 10)) + q := sampleField.Store(collabField.Row(5), nil) + if q.Error() == nil { + t.Fatalf("query error should be not nil") + } + }) + + t.Run("Options", func(t *testing.T) { + comparePQL(t, + "Options(Row(collaboration=5),columnAttrs=true,excludeColumns=true,excludeRowAttrs=true,shards=[1,3])", + sampleIndex.Options(collabField.Row(5), + OptOptionsColumnAttrs(true), + OptOptionsExcludeColumns(true), + OptOptionsExcludeRowAttrs(true), + OptOptionsShards(1, 3), + )) + comparePQL(t, + "Options(Row(collaboration=5),columnAttrs=true,excludeColumns=false,excludeRowAttrs=false)", + sampleIndex.Options(collabField.Row(5), + OptOptionsColumnAttrs(true), + )) + }) + + t.Run("BatchQuery", func(t *testing.T) { + q := sampleIndex.BatchQuery() + if q.Index() != sampleIndex { + t.Fatalf("The correct index should be assigned") + } + q.Add(sampleField.Row(44)) + q.Add(sampleField.Row(10101)) + if q.Error() != nil { + t.Fatalf("Error should be nil") + } + comparePQL(t, "Row(sample-field=44)Row(sample-field=10101)", q) + + q2 := sampleField.Row(nil) + if q2.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("BatchQueryWithError", func(t *testing.T) { + q := sampleIndex.BatchQuery() + q.Add(sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81)) + if q.Error() == nil { + t.Fatalf("The error must be set") + } + }) + + t.Run("Count", func(t *testing.T) { + q := projectIndex.Count(collabField.Row(42)) + comparePQL(t, "Count(Row(collaboration=42))", q) + }) + + t.Run("Range", func(t *testing.T) { + start := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2000, time.February, 2, 3, 4, 0, 0, time.UTC) + comparePQL(t, + "Range(collaboration=10,1970-01-01T00:00,2000-02-02T03:04)", + collabField.Range(10, start, end)) + + comparePQL(t, + "Range(collaboration='foo',1970-01-01T00:00,2000-02-02T03:04)", + collabField.Range("foo", start, end)) + + q := collabField.Range(nil, start, end) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowRange", func(t *testing.T) { + start := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2000, time.February, 2, 3, 4, 0, 0, time.UTC) + comparePQL(t, + "Row(collaboration=10,from='1970-01-01T00:00',to='2000-02-02T03:04')", + collabField.RowRange(10, start, end)) + + comparePQL(t, + "Row(collaboration='foo',from='1970-01-01T00:00',to='2000-02-02T03:04')", + collabField.RowRange("foo", start, end)) + comparePQL(t, + `Row(collaboration='bill\'s',from='1970-01-01T00:00',to='2000-02-02T03:04')`, + collabField.RowRange("bill's", start, end)) + + q := collabField.RowRange(nil, start, end) + if q.err == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("Rows", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration')", + collabField.Rows()) + }) + + t.Run("UnionRows", func(t *testing.T) { + comparePQL(t, + "UnionRows(Rows(field='collaboration'))", + collabField.Rows().Union()) + }) + + t.Run("Like", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',like='_')", + collabField.Like("_")) + comparePQL(t, + `Rows(field='collaboration',like='_\\')`, + collabField.Like(`_\`)) + comparePQL(t, + `Rows(field='collaboration',like='_\'')`, + collabField.Like(`_'`)) + }) + + t.Run("RowPrevious", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42)", + collabField.RowsPrevious(42)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two')", + collabField.RowsPrevious("forty-two")) + comparePQL(t, + `Rows(field='collaboration',previous='bill\'s')`, + collabField.RowsPrevious("bill's")) + q := collabField.RowsPrevious(1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowLimit", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',limit=10)", + collabField.RowsLimit(10)) + q := collabField.RowsLimit(-1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',column=1000)", + collabField.RowsColumn(1000)) + comparePQL(t, + "Rows(field='collaboration',column='one-thousand')", + collabField.RowsColumn("one-thousand")) + q := collabField.RowsColumn(1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousLimit", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,limit=10)", + collabField.RowsPreviousLimit(42, 10)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',limit=10)", + collabField.RowsPreviousLimit("forty-two", 10)) + q := collabField.RowsPreviousLimit(1.2, 10) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimit("forty-two", -1) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,column=1000)", + collabField.RowsPreviousColumn(42, 1000)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',column='one-thousand')", + collabField.RowsPreviousColumn("forty-two", "one-thousand")) + q := collabField.RowsPreviousColumn(1.2, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousColumn("forty-two", 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("All", func(t *testing.T) { + comparePQL(t, + "All()", + projectIndex.All()) + }) + + t.Run("Distinct", func(t *testing.T) { + comparePQL(t, + "Distinct(Row(collaboration!=null),index='project-index',field='collaboration')", + collabField.Distinct()) + }) + + t.Run("RowDistinct", func(t *testing.T) { + comparePQL(t, + "Distinct(Row(sample-field=44),index='project-index',field='collaboration')", + collabField.RowDistinct(sampleField.Row(44))) + }) + + t.Run("RowLimitColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',limit=10,column=1000)", + collabField.RowsLimitColumn(10, 1000)) + comparePQL(t, + "Rows(field='collaboration',limit=10,column='one-thousand')", + collabField.RowsLimitColumn(10, "one-thousand")) + q := collabField.RowsLimitColumn(10, 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsLimitColumn(-1, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("RowsPreviousLimitColumn", func(t *testing.T) { + comparePQL(t, + "Rows(field='collaboration',previous=42,limit=10,column=1000)", + collabField.RowsPreviousLimitColumn(42, 10, 1000)) + comparePQL(t, + "Rows(field='collaboration',previous='forty-two',limit=10,column='one-thousand')", + collabField.RowsPreviousLimitColumn("forty-two", 10, "one-thousand")) + q := collabField.RowsPreviousLimitColumn(1.2, 10, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimitColumn(42, -1, 1000) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = collabField.RowsPreviousLimitColumn(42, 10, 1.2) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupBy", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'))", + sampleIndex.GroupBy(collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'))", + sampleIndex.GroupBy(collabField.Rows(), field.Rows())) + q := sampleIndex.GroupBy() + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByLimit", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10)", + sampleIndex.GroupByLimit(10, collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10)", + sampleIndex.GroupByLimit(10, collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByLimit(10) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = sampleIndex.GroupByLimit(-1, collabField.Rows()) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByFilter", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),filter=Row(test=5))", + sampleIndex.GroupByFilter(field.Row(5), collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),filter=Row(test=5))", + sampleIndex.GroupByFilter(field.Row(5), collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByFilter(field.Row(5)) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByLimitFilter", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByLimitFilter(10, field.Row(5), collabField.Rows())) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByLimitFilter(10, field.Row(5), collabField.Rows(), field.Rows())) + q := sampleIndex.GroupByLimitFilter(10, field.Row(5)) + if q.Error() == nil { + t.Fatalf("should have failed") + } + q = sampleIndex.GroupByLimitFilter(-1, field.Row(5), collabField.Rows()) + if q.Error() == nil { + t.Fatalf("should have failed") + } + }) + + t.Run("GroupByBase", func(t *testing.T) { + field := sampleIndex.Field("test") + comparePQL(t, + "GroupBy(Rows(field='collaboration'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10)", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10)", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + comparePQL(t, + "GroupBy(Rows(field='collaboration'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows()), + ), + ) + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),limit=10,filter=Row(test=5))", + sampleIndex.GroupByBase( + OptGroupByBuilderLimit(10), + OptGroupByBuilderFilter(field.Row(5)), + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + ), + ) + + field2 := sampleIndex.Field("age") + comparePQL(t, + "GroupBy(Rows(field='collaboration'),Rows(field='test'),aggregate=Sum(Row(age=20),field='age'))", + sampleIndex.GroupByBase( + OptGroupByBuilderRows(collabField.Rows(), field.Rows()), + OptGroupByBuilderAggregate(field2.Sum(field2.Row(20))), + ), + ) + }) + + t.Run("FieldOptions", func(t *testing.T) { + field := sampleIndex.Field("foo", OptFieldKeys(true)) + if true != field.Opts().Keys() { + t.Fatalf("field keys: %v != %v", true, field.Opts().Keys()) + } + }) + + t.Run("SetFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("set-field", OptFieldTypeSet(CacheTypeRanked, 9999)) + jsonString := field.options.String() + targetString := `{"options":{"type":"set","cacheType":"ranked","cacheSize":9999}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeSet, + TimeQuantumNone, + CacheTypeRanked, + 9999, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "") + }) + + t.Run("IntFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("int-field", OptFieldTypeInt(-10, 100)) + jsonString := field.options.String() + targetString := `{"options":{"type":"int","min":-10,"max":100}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(-10, 0), + pql.NewDecimal(100, 0), + "") + + field = sampleIndex.Field("int-field2", OptFieldTypeInt(-10)) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":-10,"max":%d}}`, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(-10, 0), + pql.NewDecimal(math.MaxInt64, 0), + "") + field = sampleIndex.Field("int-field3", OptFieldTypeInt()) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d}}`, math.MinInt64, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(math.MinInt64, 0), + pql.NewDecimal(math.MaxInt64, 0), + "") + + field = sampleIndex.Field("int-field4", OptFieldTypeInt(), OptFieldForeignIndex("blerg")) + jsonString = field.options.String() + targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d,"foreignIndex":"blerg"}}`, math.MinInt64, math.MaxInt64) + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeInt, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(math.MinInt64, 0), + pql.NewDecimal(math.MaxInt64, 0), + "blerg") + }) + + t.Run("TimeFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("time-field", OptFieldTypeTime(TimeQuantumDayHour, true)) + if true != field.Opts().NoStandardView() { + t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) + } + jsonString := field.options.String() + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeTime, + TimeQuantumDayHour, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "") + }) + + t.Run("MutexFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("mutex-field", OptFieldTypeMutex(CacheTypeRanked, 9999)) + jsonString := field.options.String() + targetString := `{"options":{"type":"mutex","cacheType":"ranked","cacheSize":9999}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeMutex, + TimeQuantumNone, + CacheTypeRanked, + 9999, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "") + }) + + t.Run("BoolFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("bool-field", OptFieldTypeBool()) + jsonString := field.options.String() + targetString := `{"options":{"type":"bool"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeBool, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "") + }) + + t.Run("DecimalFieldOptions", func(t *testing.T) { + field := sampleIndex.Field("decimal-field", OptFieldTypeDecimal(3, pql.NewDecimal(7, 3), pql.NewDecimal(999, 3))) + jsonString := field.options.String() + targetString := `{"options":{"type":"decimal","scale":3,"max":0.999,"min":0.007}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeDecimal, + TimeQuantumNone, + CacheTypeDefault, + 0, + pql.NewDecimal(7, 3), + pql.NewDecimal(999, 3), + "") + }) + + t.Run("EncodeMapPanicsOnMarshalFailure", func(t *testing.T) { + defer func() { + _ = recover() + }() + m := map[string]interface{}{ + "foo": func() {}, + } + encodeMap(m) + t.Fatal("Should have panicked") + }) + + t.Run("FormatIDKey", func(t *testing.T) { + testCase := [][]interface{}{ + {uint(42), "42", nil}, + {uint32(42), "42", nil}, + {uint64(42), "42", nil}, + {42, "42", nil}, + {int32(42), "42", nil}, + {int64(42), "42", nil}, + {"foo", `'foo'`, nil}, + {false, "", errors.New("error")}, + } + for i, item := range testCase { + s, err := formatIDKey(item[0]) + if item[2] != nil { + if err == nil { + t.Fatalf("Should have failed: %d", i) + } + continue + } + if item[1] != s { + t.Fatalf("%s != %s", item[1], s) + } + } + }) +} + +func comparePQL(t *testing.T, target string, q PQLQuery) { + t.Helper() + pql := q.Serialize().String() + if target != pql { + t.Fatalf("%s != %s", target, pql) + } +} + +func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string) { + if fieldType != opts.Type() { + t.Fatalf("%s != %s", fieldType, opts.Type()) + } + if timeQuantum != opts.TimeQuantum() { + t.Fatalf("%s != %s", timeQuantum, opts.TimeQuantum()) + } + if cacheType != opts.CacheType() { + t.Fatalf("%s != %s", cacheType, opts.CacheType()) + } + if cacheSize != opts.CacheSize() { + t.Fatalf("%d != %d", cacheSize, opts.CacheSize()) + } + if min != opts.Min() { + t.Fatalf("%d != %d", min, opts.Min()) + } + if max != opts.Max() { + t.Fatalf("%d != %d", max, opts.Max()) + } + if foreignIndex != opts.ForeignIndex() { + t.Fatalf("%s != %s", foreignIndex, opts.ForeignIndex()) + } +} + +func sortedString(s string) string { + arr := strings.Split(s, "") + sort.Strings(arr) + return strings.Join(arr, "") +} diff --git a/client/record.go b/client/record.go new file mode 100644 index 000000000..9f48cf750 --- /dev/null +++ b/client/record.go @@ -0,0 +1,75 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +// Record is a Column or a FieldValue. +type Record interface { + Shard(shardWidth uint64) uint64 + Less(other Record) bool +} + +// RecordIterator is an iterator for a record. +type RecordIterator interface { + NextRecord() (Record, error) +} + +// Column defines a single Pilosa column. +type Column struct { + RowID uint64 + ColumnID uint64 + RowKey string + ColumnKey string + Timestamp int64 +} + +// Shard returns the shard for this column. +func (b Column) Shard(shardWidth uint64) uint64 { + return b.ColumnID / shardWidth +} + +// Less returns true if this column sorts before the given Record. +func (b Column) Less(other Record) bool { + if ob, ok := other.(Column); ok { + if b.RowID == ob.RowID { + return b.ColumnID < ob.ColumnID + } + return b.RowID < ob.RowID + } + return false +} + +// FieldValue represents the value for a column within a +// range-encoded field. +type FieldValue struct { + ColumnID uint64 + ColumnKey string + Value int64 +} + +// Shard returns the shard for this field value. +func (v FieldValue) Shard(shardWidth uint64) uint64 { + return v.ColumnID / shardWidth +} + +// Less returns true if this field value sorts before the given Record. +func (v FieldValue) Less(other Record) bool { + if ov, ok := other.(FieldValue); ok { + return v.ColumnID < ov.ColumnID + } + return false +} diff --git a/client/record_test.go b/client/record_test.go new file mode 100644 index 000000000..0bf6ac903 --- /dev/null +++ b/client/record_test.go @@ -0,0 +1,83 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client_test + +import ( + "testing" + + "github.com/pilosa/pilosa/v2/client" +) + +func TestColumnShard(t *testing.T) { + a := client.Column{RowID: 15, ColumnID: 55, Timestamp: 100101} + target := uint64(0) + if a.Shard(100) != target { + t.Fatalf("shard %d != %d", target, a.Shard(100)) + } + target = 5 + if a.Shard(10) != target { + t.Fatalf("shard %d != %d", target, a.Shard(10)) + } +} + +func TestColumnLess(t *testing.T) { + a := client.Column{RowID: 10, ColumnID: 200} + a2 := client.Column{RowID: 10, ColumnID: 1000} + b := client.Column{RowID: 200, ColumnID: 10} + c := client.FieldValue{ColumnID: 1} + if !a.Less(a2) { + t.Fatalf("%v should be less than %v", a, a2) + } + if !a.Less(b) { + t.Fatalf("%v should be less than %v", a, b) + } + if b.Less(a) { + t.Fatalf("%v should not be less than %v", b, a) + } + if c.Less(a) { + t.Fatalf("%v should not be less than %v", c, a) + } +} + +func TestFieldValueShard(t *testing.T) { + a := client.FieldValue{ColumnID: 55, Value: 125} + target := uint64(0) + if a.Shard(100) != target { + t.Fatalf("shard %d != %d", target, a.Shard(100)) + } + target = 5 + if a.Shard(10) != target { + t.Fatalf("shard %d != %d", target, a.Shard(10)) + } + +} + +func TestFieldValueLess(t *testing.T) { + a := client.FieldValue{ColumnID: 55, Value: 125} + b := client.FieldValue{ColumnID: 100, Value: 125} + c := client.Column{ColumnID: 1, RowID: 2} + if !a.Less(b) { + t.Fatalf("%v should be less than %v", a, b) + } + if b.Less(a) { + t.Fatalf("%v should not be less than %v", b, a) + } + if c.Less(a) { + t.Fatalf("%v should not be less than %v", c, a) + } +} diff --git a/client/response.go b/client/response.go new file mode 100644 index 000000000..0ec9382ee --- /dev/null +++ b/client/response.go @@ -0,0 +1,595 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/pilosa/pilosa/v2/pb" +) + +// QueryResponse types. +const ( + QueryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + QueryResultTypePairsField + QueryResultTypeValCount + QueryResultTypeUint64 + QueryResultTypeBool + QueryResultTypeRowIDs // this is not used by the client + QueryResultTypeGroupCounts + QueryResultTypeRowIdentifiers + QueryResultTypePair + QueryResultTypePairField + QueryResultTypeSignedRow +) + +// QueryResponse represents the response from a Pilosa query. +type QueryResponse struct { + ResultList []QueryResult `json:"results,omitempty"` + ColumnList []ColumnItem `json:"columns,omitempty"` + ErrorMessage string `json:"error-message,omitempty"` + Success bool `json:"success,omitempty"` +} + +func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, error) { + if response.Err != "" { + return &QueryResponse{ + ErrorMessage: response.Err, + Success: false, + }, nil + } + results := make([]QueryResult, 0, len(response.Results)) + for _, r := range response.Results { + result, err := newQueryResultFromInternal(r) + if err != nil { + return nil, err + } + results = append(results, result) + } + columns := make([]ColumnItem, 0, len(response.ColumnAttrSets)) + for _, p := range response.ColumnAttrSets { + columnItem, err := newColumnItemFromInternal(p) + if err != nil { + return nil, err + } + columns = append(columns, columnItem) + } + + return &QueryResponse{ + ResultList: results, + ColumnList: columns, + Success: true, + }, nil +} + +// Results returns all results in the response. +func (qr *QueryResponse) Results() []QueryResult { + return qr.ResultList +} + +// Result returns the first result or nil. +func (qr *QueryResponse) Result() QueryResult { + if len(qr.ResultList) == 0 { + return nil + } + return qr.ResultList[0] +} + +// Columns returns all column attributes in the response. +// *DEPRECATED* +func (qr *QueryResponse) Columns() []ColumnItem { + return qr.ColumnList +} + +// Column returns the attributes for first column. +// *DEPRECATED* +func (qr *QueryResponse) Column() ColumnItem { + if len(qr.ColumnList) == 0 { + return ColumnItem{} + } + return qr.ColumnList[0] +} + +// ColumnAttrs returns all column attributes in the response. +func (qr *QueryResponse) ColumnAttrs() []ColumnItem { + return qr.ColumnList +} + +// QueryResult represents one of the results in the response. +type QueryResult interface { + Type() uint32 + Row() RowResult + CountItems() []CountResultItem + CountItem() CountResultItem + Count() int64 + Value() int64 + Changed() bool + GroupCounts() []GroupCount + RowIdentifiers() RowIdentifiersResult +} + +func newQueryResultFromInternal(result *pb.QueryResult) (QueryResult, error) { + switch result.Type { + case QueryResultTypeNil: + return NilResult{}, nil + case QueryResultTypeRow: + return newRowResultFromInternal(result.Row) + case QueryResultTypePairs: + return countItemsFromInternal(result.Pairs), nil + case QueryResultTypePairsField: + return countItemsFromInternal(result.PairsField.Pairs), nil + case QueryResultTypeValCount: + return &ValCountResult{ + Val: result.ValCount.Val, + Cnt: result.ValCount.Count, + }, nil + case QueryResultTypeUint64: + return IntResult(result.N), nil + case QueryResultTypeBool: + return BoolResult(result.Changed), nil + case QueryResultTypeRowIdentifiers: + return &RowIdentifiersResult{ + IDs: result.RowIdentifiers.Rows, + Keys: result.RowIdentifiers.Keys, + }, nil + case QueryResultTypeGroupCounts: + return groupCountsFromInternal(result.GroupCounts), nil + case QueryResultTypePair: + return CountItem{CountResultItem: countItemFromInternal(result.Pairs[0])}, nil + case QueryResultTypePairField: + return CountItem{CountResultItem: countItemFromInternal(result.PairField.Pair)}, nil + } + + return nil, ErrUnknownType +} + +// CountResultItem represents a result from TopN call. +type CountResultItem struct { + ID uint64 `json:"id"` + Key string `json:"key,omitempty"` + Count uint64 `json:"count"` +} + +func (c *CountResultItem) String() string { + if c.Key != "" { + return fmt.Sprintf("%s:%d", c.Key, c.Count) + } + return fmt.Sprintf("%d:%d", c.ID, c.Count) +} + +type CountItem struct { + CountResultItem +} + +// Type is the type of this result. +func (CountItem) Type() uint32 { return QueryResultTypePairField } + +// Row returns a RowResult. +func (CountItem) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (t CountItem) CountItems() []CountResultItem { return []CountResultItem{t.CountResultItem} } + +// CountItem returns a CountResultItem +func (t CountItem) CountItem() CountResultItem { return t.CountResultItem } + +// Count returns the result of a Count call. +func (CountItem) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (CountItem) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (CountItem) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (CountItem) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (CountItem) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +func countItemFromInternal(item *pb.Pair) CountResultItem { + return CountResultItem{ID: item.ID, Key: item.Key, Count: item.Count} +} + +func countItemsFromInternal(items []*pb.Pair) TopNResult { + result := make([]CountResultItem, 0, len(items)) + for _, v := range items { + result = append(result, countItemFromInternal(v)) + } + return TopNResult(result) +} + +// TopNResult is returned from TopN call. +type TopNResult []CountResultItem + +// Type is the type of this result. +func (TopNResult) Type() uint32 { return QueryResultTypePairsField } + +// Row returns a RowResult. +func (TopNResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (t TopNResult) CountItems() []CountResultItem { return t } + +// CountItem returns a CountResultItem +func (t TopNResult) CountItem() CountResultItem { + if len(t) >= 1 { + return t[0] + } + return CountResultItem{} +} + +// Count returns the result of a Count call. +func (TopNResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (TopNResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (TopNResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (TopNResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls. +type RowResult struct { + Attributes map[string]interface{} `json:"attrs"` + Columns []uint64 `json:"columns"` + Keys []string `json:"keys"` +} + +func newRowResultFromInternal(row *pb.Row) (*RowResult, error) { + attrs, err := convertInternalAttrsToMap(row.Attrs) + if err != nil { + return nil, err + } + result := &RowResult{ + Attributes: attrs, + Columns: row.Columns, + Keys: row.Keys, + } + return result, nil +} + +// Type is the type of this result. +func (RowResult) Type() uint32 { return QueryResultTypeRow } + +// Row returns a RowResult. +func (b RowResult) Row() RowResult { return b } + +// CountItems returns a CountResultItem slice. +func (RowResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (RowResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (RowResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (RowResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (RowResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (RowResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (RowResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// MarshalJSON serializes this row result. +func (b RowResult) MarshalJSON() ([]byte, error) { + columns := b.Columns + if columns == nil { + columns = []uint64{} + } + keys := b.Keys + if keys == nil { + keys = []string{} + } + return json.Marshal(struct { + Attributes map[string]interface{} `json:"attrs"` + Columns []uint64 `json:"columns"` + Keys []string `json:"keys"` + }{ + Attributes: b.Attributes, + Columns: columns, + Keys: keys, + }) +} + +// ValCountResult is returned from Min, Max and Sum calls. +type ValCountResult struct { + Val int64 `json:"val"` + Cnt int64 `json:"count"` +} + +// Type is the type of this result. +func (ValCountResult) Type() uint32 { return QueryResultTypeValCount } + +// Row returns a RowResult. +func (ValCountResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (ValCountResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (ValCountResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (c ValCountResult) Count() int64 { return c.Cnt } + +// Value returns the result of a Min, Max or Sum call. +func (c ValCountResult) Value() int64 { return c.Val } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (ValCountResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (ValCountResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (ValCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// IntResult is returned from Count call. +type IntResult int64 + +// Type is the type of this result. +func (IntResult) Type() uint32 { return QueryResultTypeUint64 } + +// Row returns a RowResult. +func (IntResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (IntResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (IntResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (i IntResult) Count() int64 { return int64(i) } + +// Value returns the result of a Min, Max or Sum call. +func (IntResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (IntResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (IntResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (IntResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// BoolResult is returned from Set and Clear calls. +type BoolResult bool + +// Type is the type of this result. +func (BoolResult) Type() uint32 { return QueryResultTypeBool } + +// Row returns a RowResult. +func (BoolResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (BoolResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (BoolResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (BoolResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (BoolResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (b BoolResult) Changed() bool { return bool(b) } + +// GroupCounts returns the result of a GroupBy call. +func (BoolResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// NilResult is returned from calls which don't return a value, such as SetRowAttrs. +type NilResult struct{} + +// Type is the type of this result. +func (NilResult) Type() uint32 { return QueryResultTypeNil } + +// Row returns a RowResult. +func (NilResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (NilResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (NilResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (NilResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (NilResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (NilResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (NilResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (NilResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// FieldRow represents a Group in a GroupBy call result. +type FieldRow struct { + FieldName string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey"` + Value *int64 `json:"value,omitempty"` +} + +// GroupCount contains groups and their count in a GroupBy call result. +type GroupCount struct { + Groups []FieldRow `json:"groups"` + Count int64 `json:"count"` + Agg int64 `json:"agg"` +} + +// GroupCountResult is returned from GroupBy call. +type GroupCountResult []GroupCount + +// Type is the type of this result. +func (GroupCountResult) Type() uint32 { return QueryResultTypeGroupCounts } + +// Row returns a RowResult. +func (GroupCountResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (GroupCountResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (GroupCountResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (GroupCountResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (GroupCountResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (GroupCountResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (r GroupCountResult) GroupCounts() []GroupCount { return r } + +// RowIdentifiers returns the result of a Rows call. +func (GroupCountResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} } + +// RowIdentifiersResult is returned from a Rows call. +type RowIdentifiersResult struct { + IDs []uint64 `json:"ids"` + Keys []string `json:"keys,omitempty"` +} + +// Type is the type of this result. +func (RowIdentifiersResult) Type() uint32 { return QueryResultTypeRowIdentifiers } + +// Row returns a RowResult. +func (RowIdentifiersResult) Row() RowResult { return RowResult{} } + +// CountItems returns a CountResultItem slice. +func (RowIdentifiersResult) CountItems() []CountResultItem { return nil } + +// CountItem returns a CountResultItem +func (RowIdentifiersResult) CountItem() CountResultItem { return CountResultItem{} } + +// Count returns the result of a Count call. +func (RowIdentifiersResult) Count() int64 { return 0 } + +// Value returns the result of a Min, Max or Sum call. +func (RowIdentifiersResult) Value() int64 { return 0 } + +// Changed returns whether the corresponding Set or Clear call changed the value of a bit. +func (RowIdentifiersResult) Changed() bool { return false } + +// GroupCounts returns the result of a GroupBy call. +func (RowIdentifiersResult) GroupCounts() []GroupCount { return nil } + +// RowIdentifiers returns the result of a Rows call. +func (r RowIdentifiersResult) RowIdentifiers() RowIdentifiersResult { return r } + +func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult { + result := make([]GroupCount, 0, len(items.Groups)) + for _, g := range items.Groups { + groups := make([]FieldRow, 0, len(g.Group)) + for _, f := range g.Group { + fr := FieldRow{ + FieldName: f.Field, + RowID: f.RowID, + RowKey: f.RowKey, + } + if f.Value != nil { + fr.Value = &f.Value.Value + } + groups = append(groups, fr) + } + result = append(result, GroupCount{ + Groups: groups, + Count: int64(g.Count), + Agg: int64(g.Agg), + }) + } + return GroupCountResult(result) +} + +const ( + stringType = 1 + intType = 2 + boolType = 3 + floatType = 4 +) + +func convertInternalAttrsToMap(attrs []*pb.Attr) (attrsMap map[string]interface{}, err error) { + attrsMap = make(map[string]interface{}, len(attrs)) + for _, attr := range attrs { + switch attr.Type { + case stringType: + attrsMap[attr.Key] = attr.StringValue + case intType: + attrsMap[attr.Key] = attr.IntValue + case boolType: + attrsMap[attr.Key] = attr.BoolValue + case floatType: + attrsMap[attr.Key] = attr.FloatValue + default: + return nil, errors.New("Unknown attribute type") + } + } + + return attrsMap, nil +} + +// ColumnItem represents data about a column. +// Column data is only returned if QueryOptions.Columns was set to true. +type ColumnItem struct { + ID uint64 `json:"id,omitempty"` + Key string `json:"key,omitempty"` + Attributes map[string]interface{} `json:"attributes,omitempty"` +} + +func newColumnItemFromInternal(column *pb.ColumnAttrSet) (ColumnItem, error) { + attrs, err := convertInternalAttrsToMap(column.Attrs) + if err != nil { + return ColumnItem{}, err + } + return ColumnItem{ + ID: column.ID, + Key: column.Key, + Attributes: attrs, + }, nil +} diff --git a/client/response_test.go b/client/response_test.go new file mode 100644 index 000000000..12efdf533 --- /dev/null +++ b/client/response_test.go @@ -0,0 +1,348 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "encoding/json" + "fmt" + "log" + "reflect" + "testing" + + "github.com/pilosa/pilosa/v2/pb" +) + +func TestNewRowResultFromInternal(t *testing.T) { + targetAttrs := map[string]interface{}{ + "name": "some string", + "age": int64(95), + "registered": true, + "height": 1.83, + } + targetColumns := []uint64{5, 10} + attrs := []*pb.Attr{ + {Key: "name", StringValue: "some string", Type: 1}, + {Key: "age", IntValue: 95, Type: 2}, + {Key: "registered", BoolValue: true, Type: 3}, + {Key: "height", FloatValue: 1.83, Type: 4}, + } + row := &pb.Row{ + Attrs: attrs, + Columns: []uint64{5, 10}, + } + result, err := newRowResultFromInternal(row) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + // assertMapEquals(t, targetAttrs, result.Attributes) + if !reflect.DeepEqual(targetAttrs, result.Attributes) { + t.Fatal() + } + if !reflect.DeepEqual(targetColumns, result.Columns) { + t.Fatal() + } +} + +func TestNewQueryResponseFromInternal(t *testing.T) { + targetAttrs := map[string]interface{}{ + "name": "some string", + "age": int64(95), + "registered": true, + "height": 1.83, + } + targetColumns := []uint64{5, 10} + targetCountItems := []CountResultItem{ + {ID: 10, Count: 100}, + } + attrs := []*pb.Attr{ + {Key: "name", StringValue: "some string", Type: 1}, + {Key: "age", IntValue: 95, Type: 2}, + {Key: "registered", BoolValue: true, Type: 3}, + {Key: "height", FloatValue: 1.83, Type: 4}, + } + row := &pb.Row{ + Attrs: attrs, + Columns: []uint64{5, 10}, + } + pairs := []*pb.Pair{ + {ID: 10, Count: 100}, + } + response := &pb.QueryResponse{ + Results: []*pb.QueryResult{ + {Type: QueryResultTypeRow, Row: row}, + {Type: QueryResultTypePairs, Pairs: pairs}, + }, + Err: "", + } + qr, err := newQueryResponseFromInternal(response) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + if qr.ErrorMessage != "" { + t.Fatalf("ErrorMessage should be empty") + } + if !qr.Success { + t.Fatalf("IsSuccess should be true") + } + + results := qr.Results() + if len(results) != 2 { + t.Fatalf("Number of results should be 2") + } + if results[0] != qr.Result() { + t.Fatalf("Result() should return the first result") + } + if !reflect.DeepEqual(targetAttrs, results[0].Row().Attributes) { + t.Fatalf("The row result should contain the attributes") + } + if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) { + t.Fatalf("The row result should contain the columns") + } + if !reflect.DeepEqual(targetCountItems, results[1].CountItems()) { + t.Fatalf("The response should include count items") + } +} + +func TestNewQueryResponseWithErrorFromInternal(t *testing.T) { + response := &pb.QueryResponse{ + Err: "some error", + } + qr, err := newQueryResponseFromInternal(response) + if err != nil { + t.Fatalf("Failed with error: %s", err) + } + if qr.ErrorMessage != "some error" { + t.Fatalf("The response should include the error message") + } + if qr.Success { + t.Fatalf("IsSuccess should be false") + } + if qr.Result() != nil { + t.Fatalf("If there are no results, Result should return nil") + } +} + +func TestNewQueryResponseFromInternalFailure(t *testing.T) { + attrs := []*pb.Attr{ + {Key: "name", StringValue: "some string", Type: 99}, + } + row := &pb.Row{ + Attrs: attrs, + } + response := &pb.QueryResponse{ + Results: []*pb.QueryResult{{Type: QueryResultTypeRow, Row: row}}, + } + qr, err := newQueryResponseFromInternal(response) + if qr != nil && err == nil { + t.Fatalf("Should have failed") + } + response = &pb.QueryResponse{ + ColumnAttrSets: []*pb.ColumnAttrSet{{ID: 1, Attrs: attrs}}, + } + qr, err = newQueryResponseFromInternal(response) + if qr != nil && err == nil { + t.Fatalf("Should have failed") + } +} + +func TestCountResultItemToString(t *testing.T) { + tests := []struct { + item *CountResultItem + expected string + }{ + {item: &CountResultItem{ID: 100, Count: 50}, expected: "100:50"}, + {item: &CountResultItem{Key: "blah", Count: 50}, expected: "blah:50"}, + {item: &CountResultItem{Key: "blah", ID: 22, Count: 50}, expected: "blah:50"}, + {item: &CountResultItem{Key: "blah", ID: 22}, expected: "blah:0"}, + {item: &CountResultItem{}, expected: "0:0"}, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d: ", i), func(t *testing.T) { + if tst.expected != tst.item.String() { + t.Fatalf("%s != %s", tst.expected, tst.item.String()) + } + }) + } +} + +func TestMarshalResults(t *testing.T) { + attrs := []*pb.Attr{ + {Key: "name", StringValue: "some string", Type: 1}, + {Key: "age", IntValue: 95, Type: 2}, + {Key: "registered", BoolValue: true, Type: 3}, + {Key: "height", FloatValue: 1.83, Type: 4}, + } + row := &pb.Row{ + Attrs: attrs, + Columns: []uint64{5, 10}, + } + pairs := []*pb.Pair{ + {ID: 10, Count: 100}, + } + pbufResults := []*pb.QueryResult{ + {Type: QueryResultTypeRow, Row: row}, + {Type: QueryResultTypePairs, Pairs: pairs}, + } + resultJSONStrings := make([]string, len(pbufResults)) + for i, pr := range pbufResults { + r, err := newQueryResultFromInternal(pr) + if err != nil { + t.Fatal(err) + } + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + resultJSONStrings[i] = string(b) + } + targetJSON := []string{ + `{"attrs":{"age":95,"height":1.83,"name":"some string","registered":true},"columns":[5,10],"keys":[]}`, + `[{"id":10,"count":100}]`, + } + for i := range targetJSON { + if sortedString(targetJSON[i]) != sortedString(resultJSONStrings[i]) { + t.Fatalf("%v != %v ", targetJSON[i], resultJSONStrings[i]) + } + } + +} + +func TestUnknownQueryResultType(t *testing.T) { + result := &pb.QueryResult{ + Type: 999, + } + _, err := newQueryResultFromInternal(result) + if err != ErrUnknownType { + t.Fatalf("Should have failed with ErrUnknownType") + } +} + +func TestTopNResult(t *testing.T) { + result := TopNResult{ + CountResultItem{ID: 100, Count: 10}, + } + expectResult(t, result, QueryResultTypePairsField, RowResult{}, []CountResultItem{{100, "", 10}}, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestRowResult(t *testing.T) { + result := RowResult{ + Columns: []uint64{1, 2, 3}, + } + targetBmp := RowResult{ + Columns: []uint64{1, 2, 3}, + } + expectResult(t, result, QueryResultTypeRow, targetBmp, nil, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestRowResultNilColumns(t *testing.T) { + result := RowResult{ + Columns: nil, + } + _, err := result.MarshalJSON() + if err != nil { + t.Fatal(err) + } +} + +func TestSumCountResult(t *testing.T) { + result := ValCountResult{ + Val: 100, + Cnt: 50, + } + expectResult(t, result, QueryResultTypeValCount, RowResult{}, nil, 100, 50, false, nil, RowIdentifiersResult{}) +} + +func TestIntResult(t *testing.T) { + result := IntResult(11) + expectResult(t, result, QueryResultTypeUint64, RowResult{}, nil, 0, 11, false, nil, RowIdentifiersResult{}) +} + +func TestBoolResult(t *testing.T) { + result := BoolResult(true) + expectResult(t, result, QueryResultTypeBool, RowResult{}, nil, 0, 0, true, nil, RowIdentifiersResult{}) +} + +func TestNilResult(t *testing.T) { + result := NilResult{} + expectResult(t, result, QueryResultTypeNil, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{}) +} + +func TestGroupCountResult(t *testing.T) { + result := GroupCountResult{ + {Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1}, + } + expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{ + {Groups: []FieldRow{{FieldName: "f1", RowID: 1}}, Count: 2}, + {Groups: []FieldRow{{FieldName: "f1", RowID: 2}}, Count: 1}, + }, RowIdentifiersResult{}) +} + +func TestGroupCountWithValueResult(t *testing.T) { + var a, b int64 = -1, 1 + + result := GroupCountResult{ + {Groups: []FieldRow{{FieldName: "f1", Value: &a}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "f1", Value: &b}}, Count: 1}, + } + + var aa, bb int64 = -1, 1 + expectResult(t, result, QueryResultTypeGroupCounts, RowResult{}, nil, 0, 0, false, []GroupCount{ + {Groups: []FieldRow{{FieldName: "f1", Value: &aa}}, Count: 1}, + {Groups: []FieldRow{{FieldName: "f1", Value: &bb}}, Count: 1}, + }, RowIdentifiersResult{}) +} + +func TestRowIdentifiersResult(t *testing.T) { + result := RowIdentifiersResult{ + IDs: []uint64{1, 2, 3, 4}, + } + expectResult(t, result, QueryResultTypeRowIdentifiers, RowResult{}, nil, 0, 0, false, nil, RowIdentifiersResult{ + IDs: []uint64{1, 2, 3, 4}, + }) +} + +func expectResult(t *testing.T, r QueryResult, resultType uint32, bmp RowResult, + countItems []CountResultItem, sum int64, count int64, changed bool, + groupCounts []GroupCount, rowIdentifiers RowIdentifiersResult) { + if resultType != r.Type() { + log.Fatalf("Result type: %d != %d", resultType, r.Type()) + } + if !reflect.DeepEqual(bmp, r.Row()) { + log.Fatalf("Row: %v != %v", bmp, r.Row()) + } + if !reflect.DeepEqual(countItems, r.CountItems()) { + log.Fatalf("Count items: %v != %v", countItems, r.CountItems()) + } + if count != r.Count() { + log.Fatalf("Count: %d != %d", count, r.Count()) + } + if sum != r.Value() { + log.Fatalf("Sum: %d != %d", sum, r.Value()) + } + if changed != r.Changed() { + log.Fatalf("Changed: %v != %v", changed, r.Changed()) + } + if !reflect.DeepEqual(groupCounts, r.GroupCounts()) { + log.Fatalf("Group counts: %v != %v", groupCounts, r.GroupCounts()) + } + if !reflect.DeepEqual(rowIdentifiers, r.RowIdentifiers()) { + log.Fatalf("Row identifiers: %v != %v", rowIdentifiers, r.RowIdentifiers()) + } +} diff --git a/client/shardnodes.go b/client/shardnodes.go new file mode 100644 index 000000000..a0f0a296f --- /dev/null +++ b/client/shardnodes.go @@ -0,0 +1,66 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "sync" + + pnet "github.com/pilosa/pilosa/v2/net" +) + +type shardNodes struct { + data map[string]map[uint64][]*pnet.URI + mu *sync.RWMutex +} + +func newShardNodes() shardNodes { + return shardNodes{ + data: make(map[string]map[uint64][]*pnet.URI), + mu: &sync.RWMutex{}, + } +} + +func (s shardNodes) Get(index string, shard uint64) ([]*pnet.URI, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if idx, ok := s.data[index]; ok { + if uris, ok := idx[shard]; ok { + return uris, true + } + } + return nil, false +} + +func (s shardNodes) Put(index string, shard uint64, uris []*pnet.URI) { + s.mu.Lock() + defer s.mu.Unlock() + idx, ok := s.data[index] + if !ok { + idx = make(map[uint64][]*pnet.URI) + } + idx[shard] = uris + s.data[index] = idx +} + +func (s shardNodes) Invalidate() { + s.mu.Lock() + defer s.mu.Unlock() + for k := range s.data { + delete(s.data, k) + } +} diff --git a/client/tracer.go b/client/tracer.go new file mode 100644 index 000000000..f838bd3b6 --- /dev/null +++ b/client/tracer.go @@ -0,0 +1,89 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/log" +) + +type NoopTracer struct{} + +type NoopSpan struct{} + +func (s NoopSpan) Finish() { + // pass +} +func (s NoopSpan) FinishWithOptions(opts opentracing.FinishOptions) { + // pass +} + +func (s NoopSpan) Context() opentracing.SpanContext { + return nil +} +func (s NoopSpan) SetOperationName(operationName string) opentracing.Span { + return s +} + +func (s NoopSpan) SetTag(key string, value interface{}) opentracing.Span { + return s +} + +func (s NoopSpan) LogFields(fields ...log.Field) { + // pass +} + +func (s NoopSpan) LogKV(alternatingKeyValues ...interface{}) { + // pass +} + +func (s NoopSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span { + return s +} + +func (s NoopSpan) BaggageItem(restrictedKey string) string { + return "" +} + +func (s NoopSpan) Tracer() opentracing.Tracer { + return nil +} + +func (s NoopSpan) LogEvent(event string) { + // pass +} + +func (s NoopSpan) LogEventWithPayload(event string, payload interface{}) { + // pass +} + +func (s NoopSpan) Log(data opentracing.LogData) { + // pass +} + +func (t NoopTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span { + return NoopSpan{} +} + +func (t NoopTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error { + return nil +} + +func (t NoopTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + return nil, nil +} diff --git a/client/validate.go b/client/validate.go new file mode 100644 index 000000000..77f89c42c --- /dev/null +++ b/client/validate.go @@ -0,0 +1,54 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import ( + "regexp" +) + +const ( + maxLabel = 64 + maxKey = 64 +) + +var labelRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9_-]*$") +var keyRegex = regexp.MustCompile("^[A-Za-z0-9_{}+/=.~%:-]*$") + +// ValidLabel returns true if the given label is valid, otherwise false. +func ValidLabel(label string) bool { + return len(label) <= maxLabel && labelRegex.Match([]byte(label)) +} + +// ValidKey returns true if the given key is valid, otherwise false. +func ValidKey(key string) bool { + return len(key) <= maxKey && keyRegex.Match([]byte(key)) +} + +func validateLabel(label string) error { + if ValidLabel(label) { + return nil + } + return ErrInvalidLabel +} + +func validateKey(key string) error { + if ValidKey(key) { + return nil + } + return ErrInvalidKey +} diff --git a/client/validate_test.go b/client/validate_test.go new file mode 100644 index 000000000..1284ccf10 --- /dev/null +++ b/client/validate_test.go @@ -0,0 +1,73 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package ctl contains all pilosa subcommands other than 'server'. These are +// generally administration, testing, and debugging tools. + +package client + +import "testing" + +func TestValidateLabel(t *testing.T) { + labels := []string{ + "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, label := range labels { + if validateLabel(label) != nil { + t.Fatalf("Should be valid label: %s", label) + } + } +} + +func TestValidateLabelInvalid(t *testing.T) { + labels := []string{ + "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, label := range labels { + if validateLabel(label) == nil { + t.Fatalf("Should be invalid label: %s", label) + } + } +} + +func TestValidateKey(t *testing.T) { + keys := []string{ + "", "1", "ab", "ab1", "b-c", "d_e", "pilosa.com", + "bbf8d41c-7dba-40c4-94dc-94677b43bcf3", // UUID + "{bbf8d41c-7dba-40c4-94dc-94677b43bcf3}", // Windows GUID + "https%3A//www.pilosa.com/about/%23contact", // escaped URL + "aHR0cHM6Ly93d3cucGlsb3NhLmNvbS9hYm91dC8jY29udGFjdA==", // base64 + "urn:isbn:1234567", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, key := range keys { + if validateKey(key) != nil { + t.Fatalf("Should be valid key: %s", key) + } + } +} + +func TestValidateKeyInvalid(t *testing.T) { + keys := []string{ + "\"", "'", "slice\\dice", "valid?no", "yüce", "*xyz", "with space", "