scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove attributes

Browse source 
Attributes are unmaintained and unused.
They have become more of a liability than a benefit.
This change eliminates them from the codebase.
The only user-visible change (assuming that attrs are not used) is that the attrs field will no longer appear in row JSON.
This commit is contained in:
Nia Weiss 2021-05-13 16:03:17 -04:00
parent e4be3583d7
commit f4ba34247f
No known key found for this signature in database
GPG key ID: 895E83409BFDA1BB
57 changed files with 2175 additions and 8252 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
api.go
View file

@ -189,13 +189,10 @@ func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, er
// TODO can we get rid of exec options and pass the QueryRequest directly to executor?
execOpts := &execOptions{
Remote: req.Remote,
Profile: req.Profile,
ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat.
ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat.
ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat.
PreTranslated: req.PreTranslated,
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
Remote: req.Remote,
Profile: req.Profile,
PreTranslated: req.PreTranslated,
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
}
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
if err != nil {
@ -280,15 +277,6 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
return nil
}
func (api *API) WriteColumnAttrDataTo(ctx context.Context, w io.Writer, indexName string) error {
index := api.holder.Index(indexName)
if index == nil {
return newNotFoundError(ErrIndexNotFound, indexName)
}
_, err := index.ColumnAttrStore().WriteTo(w)
return err
}
// CreateField makes the named field in the named index with the given options.
// This method currently only takes a single functional option, but that may be
// changed in the future to support multiple options.
@ -346,15 +334,6 @@ func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field,
return field, nil
}
func (api *API) WriteRowAttrDataTo(ctx context.Context, w io.Writer, indexName, fieldName string) error {
field := api.holder.Field(indexName, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound, fieldName)
}
_, err := field.RowAttrStore().WriteTo(w)
return err
}
func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) {
options := &ImportOptions{}
for _, opt := range opts {
@ -1187,82 +1166,6 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
return errors.Wrap(err, "sending DeleteView message")
}
// IndexAttrDiff determines the local column attribute data blocks which differ from those provided.
func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.IndexAttrDiff")
defer span.Finish()
if err := api.validate(apiIndexAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve index from holder.
index := api.holder.Index(indexName)
if index == nil {
return nil, newNotFoundError(ErrIndexNotFound, indexName)
}
// Retrieve local blocks.
localBlocks, err := index.ColumnAttrStore().Blocks()
if err != nil {
return nil, errors.Wrap(err, "getting blocks")
}
// Read all attributes from all mismatched blocks.
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range attrBlocks(localBlocks).Diff(blocks) {
// Retrieve block data.
m, err := index.ColumnAttrStore().BlockData(blockID)
if err != nil {
return nil, errors.Wrap(err, "getting block")
}
// Copy to index-wide struct.
for k, v := range m {
attrs[k] = v
}
}
return attrs, nil
}
// FieldAttrDiff determines the local row attribute data blocks which differ from those provided.
func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FieldAttrDiff")
defer span.Finish()
if err := api.validate(apiFieldAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve index from holder.
f := api.holder.Field(indexName, fieldName)
if f == nil {
return nil, newNotFoundError(ErrFieldNotFound, fieldName)
}
// Retrieve local blocks.
localBlocks, err := f.RowAttrStore().Blocks()
if err != nil {
return nil, errors.Wrap(err, "getting blocks")
}
// Read all attributes from all mismatched blocks.
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range attrBlocks(localBlocks).Diff(blocks) {
// Retrieve block data.
m, err := f.RowAttrStore().BlockData(blockID)
if err != nil {
return nil, errors.Wrap(err, "getting block")
}
// Copy to index-wide struct.
for k, v := range m {
attrs[k] = v
}
}
return attrs, nil
}
// IndexShardSnapshot returns a reader that contains the contents of an RBF snapshot for an index/shard.
func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64) (io.ReadCloser, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.IndexShardSnapshot")
@ -1768,36 +1671,6 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
return nil
}
func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.ImportColumnAttrs")
defer span.Finish()
index, err := api.Index(ctx, req.Index)
if err != nil {
return errors.Wrap(err, "getting index")
}
if err := api.validateShardOwnership(req.Index, uint64(req.Shard)); err != nil {
return errors.Wrap(err, "validating shard ownership")
}
if req.IndexCreatedAt != 0 {
if index.CreatedAt() != req.IndexCreatedAt {
return ErrPreconditionFailed
}
}
bulkAttrs := make(map[uint64]map[string]interface{})
for n := 0; n < len(req.ColumnIDs); n++ {
bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]}
}
if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil {
api.server.logger.Errorf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err)
return errors.Wrap(err, "importing column attrs")
}
return nil
}
func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64) error {
ef := index.existenceField()
if ef == nil {
@ -2369,12 +2242,10 @@ const (
apiTranslateData
apiFieldTranslateData
apiField
apiFieldAttrDiff
//apiHosts // not implemented
apiImport
apiImportValue
apiIndex
apiIndexAttrDiff
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
//apiMaxShards // not implemented
@ -2418,9 +2289,7 @@ var methodsDegraded = map[apiMethod]struct{}{
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiFieldAttrDiff: {},
apiIndex: {},
apiIndexAttrDiff: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},
@ -2446,11 +2315,9 @@ var methodsNormal = map[apiMethod]struct{}{
apiFragmentBlocks: {},
apiField: {},
apiFieldTranslateData: {},
apiFieldAttrDiff: {},
apiImport: {},
apiImportValue: {},
apiIndex: {},
apiIndexAttrDiff: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},

140
api_test.go
View file

@ -21,7 +21,6 @@ import (
"fmt"
"math"
"reflect"
"strconv"
"strings"
"testing"
"time"
@ -34,145 +33,6 @@ import (
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// attrFun defines a mapping from columnID -> attr value
func attrFun(id uint64) string {
//return fmt.Sprintf("%x", md5.Sum([]byte(strconv.FormatInt(int64(id), 10))))
return strconv.FormatInt(int64(id), 10)
}
func TestAPI_ImportColumnAttrs(t *testing.T) {
/*
columns seconds
100 1.150
1000 1.568
10000 5.156
100000 38.179
*/
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
)},
)
defer c.Close()
m0 := c.GetNode(0)
m1 := c.GetNode(1)
t.Run("ImportColumnAttrs", func(t *testing.T) {
ctx := context.Background()
indexName := "i"
fieldName := "f"
attrKey := "k"
index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = m0.API.CreateField(ctx, indexName, fieldName)
if err != nil {
t.Fatalf("creating field: %v", err)
}
// Generate some attrs for two shards
numAttrs := 100
columnIDs0 := make([]uint64, 0, numAttrs)
attrVals0 := make([]string, 0, numAttrs)
columnIDs1 := make([]uint64, 0, numAttrs)
attrVals1 := make([]string, 0, numAttrs)
for n := 0; n < 1000000; n += 1000000 / numAttrs {
columnIDs0 = append(columnIDs0, uint64(n))
val0 := attrFun(uint64(n))
attrVals0 = append(attrVals0, val0)
setPql0 := fmt.Sprintf("Set(%d, %s=0) ", n, fieldName)
if _, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql0}); err != nil {
t.Fatal(err)
}
columnIDs1 = append(columnIDs1, uint64(n+ShardWidth))
val1 := attrFun(uint64(n + ShardWidth))
attrVals1 = append(attrVals1, val1)
setPql1 := fmt.Sprintf("Set(%d, %s=0) ", n+ShardWidth, fieldName)
if _, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: setPql1}); err != nil {
t.Fatal(err)
}
}
// send shard0 to node1
req := &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs0,
AttrVals: attrVals0,
Shard: 0,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m0.API.ImportColumnAttrs(ctx, req); err != nil {
t.Fatal(err)
}
// send shard1 to node0
req = &pilosa.ImportColumnAttrsRequest{
AttrKey: attrKey,
ColumnIDs: columnIDs1,
AttrVals: attrVals1,
Shard: 1,
Index: indexName,
IndexCreatedAt: index.CreatedAt(),
}
if err := m1.API.ImportColumnAttrs(ctx, req); err != nil {
t.Fatal(err)
}
// Query node0.
pql := fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
m := len(res.ColumnAttrSets)
if m != 100 {
t.Fatalf("incorrect number of column attrs set; m = %v", m)
}
for _, v := range res.ColumnAttrSets {
attrVal := attrFun(v.ID)
if attrVal != v.Attrs[attrKey] {
t.Fatal(err)
}
}
// Query node1.
pql = fmt.Sprintf("Options(Row(%s=0), columnAttrs=true)", fieldName)
res, err = m1.API.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql})
if err != nil {
t.Fatal(err)
}
if len(res.ColumnAttrSets) != 100 {
t.Fatal("incorrect number of column attrs set")
}
for _, v := range res.ColumnAttrSets {
attrVal := attrFun(v.ID)
if attrVal != v.Attrs[attrKey] {
t.Fatal(err)
}
}
})
}
func TestAPI_Import(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{

48
apimethod_string.go
View file

@ -22,34 +22,32 @@ func _() {
_ = x[apiTranslateData-11]
_ = x[apiFieldTranslateData-12]
_ = x[apiField-13]
_ = x[apiFieldAttrDiff-14]
_ = x[apiImport-15]
_ = x[apiImportValue-16]
_ = x[apiIndex-17]
_ = x[apiIndexAttrDiff-18]
_ = x[apiQuery-19]
_ = x[apiRecalculateCaches-20]
_ = x[apiRemoveNode-21]
_ = x[apiResizeAbort-22]
_ = x[apiSchema-23]
_ = x[apiShardNodes-24]
_ = x[apiState-25]
_ = x[apiViews-26]
_ = x[apiApplySchema-27]
_ = x[apiStartTransaction-28]
_ = x[apiFinishTransaction-29]
_ = x[apiTransactions-30]
_ = x[apiGetTransaction-31]
_ = x[apiActiveQueries-32]
_ = x[apiPastQueries-33]
_ = x[apiIDReserve-34]
_ = x[apiIDCommit-35]
_ = x[apiIDReset-36]
_ = x[apiImport-14]
_ = x[apiImportValue-15]
_ = x[apiIndex-16]
_ = x[apiQuery-17]
_ = x[apiRecalculateCaches-18]
_ = x[apiRemoveNode-19]
_ = x[apiResizeAbort-20]
_ = x[apiSchema-21]
_ = x[apiShardNodes-22]
_ = x[apiState-23]
_ = x[apiViews-24]
_ = x[apiApplySchema-25]
_ = x[apiStartTransaction-26]
_ = x[apiFinishTransaction-27]
_ = x[apiTransactions-28]
_ = x[apiGetTransaction-29]
_ = x[apiActiveQueries-30]
_ = x[apiPastQueries-31]
_ = x[apiIDReserve-32]
_ = x[apiIDCommit-33]
_ = x[apiIDReset-34]
}
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 234, 243, 257, 265, 281, 289, 309, 322, 336, 345, 358, 366, 374, 388, 407, 427, 442, 459, 475, 489, 501, 512, 522}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

211
attr.go
View file

@ -1,211 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"io"
"sort"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/v2/pb"
)
// Attribute data type enum.
const (
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
// AttrStore represents an interface for handling row/column attributes.
type AttrStore interface {
io.WriterTo
Path() string
Open() error
Close() error
Attrs(id uint64) (m map[string]interface{}, err error)
SetAttrs(id uint64, m map[string]interface{}) error
SetBulkAttrs(m map[uint64]map[string]interface{}) error
Blocks() ([]AttrBlock, error)
BlockData(i uint64) (map[uint64]map[string]interface{}, error)
}
// nopStore represents an AttrStore that doesn't do anything.
var nopStore AttrStore = nopAttrStore{}
// newNopAttrStore returns an attr store which does nothing. It returns a global
// object to avoid unnecessary allocations.
func newNopAttrStore(string) AttrStore { return nopStore }
// nopAttrStore represents a no-op implementation of the AttrStore interface.
type nopAttrStore struct{}
// Path is a no-op implementation of AttrStore Path method.
func (s nopAttrStore) Path() string { return "" }
// Open is a no-op implementation of AttrStore Open method.
func (s nopAttrStore) Open() error { return nil }
// Close is a no-op implementation of AttrStore Close method.
func (s nopAttrStore) Close() error { return nil }
// Attrs is a no-op implementation of AttrStore Attrs method.
func (s nopAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { return nil, nil }
// SetAttrs is a no-op implementation of AttrStore SetAttrs method.
func (s nopAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { return nil }
// SetBulkAttrs is a no-op implementation of AttrStore SetBulkAttrs method.
func (s nopAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { return nil }
// Blocks is a no-op implementation of AttrStore Blocks method.
func (s nopAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
// BlockData is a no-op implementation of AttrStore BlockData method.
func (s nopAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { return nil, nil }
// WriteTo is a no-op implementation of AttrStore WriteTo method.
func (s nopAttrStore) WriteTo(w io.Writer) (int64, error) { return 0, nil }
// AttrBlock represents a checksummed block of the attribute store.
type AttrBlock struct {
ID uint64 `json:"id"`
Checksum []byte `json:"checksum"`
}
// attrBlocks represents a list of blocks.
type attrBlocks []AttrBlock
// Diff returns a list of block ids that are different or are new in other.
// Block lists must be in sorted order.
func (a attrBlocks) Diff(other []AttrBlock) []uint64 {
var ids []uint64
for {
// Read next block from each list.
var blk0, blk1 *AttrBlock
if len(a) > 0 {
blk0 = &a[0]
}
if len(other) > 0 {
blk1 = &other[0]
}
// Exit if "a" contains no more blocks.
if blk0 == nil {
return ids
}
// Add block ID if it's different or if it's only in "a".
if blk1 == nil || blk0.ID < blk1.ID {
ids = append(ids, blk0.ID)
a = a[1:]
} else if blk1.ID < blk0.ID {
other = other[1:]
} else {
if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
ids = append(ids, blk0.ID)
}
a, other = a[1:], other[1:]
}
}
}
func encodeAttrs(m map[string]interface{}) []*pb.Attr {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
a := make([]*pb.Attr, len(keys))
for i := range keys {
a[i] = encodeAttr(keys[i], m[keys[i]])
}
return a
}
func decodeAttrs(pb []*pb.Attr) map[string]interface{} {
m := make(map[string]interface{}, len(pb))
for i := range pb {
key, value := decodeAttr(pb[i])
m[key] = value
}
return m
}
// encodeAttr converts a key/value pair into an Attr pb.representation.
func encodeAttr(key string, value interface{}) *pb.Attr {
pb := &pb.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
}
// decodeAttr converts from an Attr pb.representation to a key/value pair.
func decodeAttr(attr *pb.Attr) (key string, value interface{}) {
switch attr.Type {
case attrTypeString:
return attr.Key, attr.StringValue
case attrTypeInt:
return attr.Key, attr.IntValue
case attrTypeBool:
return attr.Key, attr.BoolValue
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil
}
}
// cloneAttrs returns a shallow clone of m.
func cloneAttrs(m map[string]interface{}) map[string]interface{} {
other := make(map[string]interface{}, len(m))
for k, v := range m {
other[k] = v
}
return other
}
// EncodeAttrs encodes an attribute map into a byte slice.
func EncodeAttrs(attr map[string]interface{}) ([]byte, error) {
return proto.Marshal(&pb.AttrMap{Attrs: encodeAttrs(attr)})
}
// DecodeAttrs decodes a byte slice into an attribute map.
func DecodeAttrs(v []byte) (map[string]interface{}, error) {
var pb pb.AttrMap
if err := proto.Unmarshal(v, &pb); err != nil {
return nil, err
}
return decodeAttrs(pb.GetAttrs()), nil
}

204
attr_test.go
View file

@ -1,204 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"os"
"reflect"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/testhook"
)
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := MustOpenAttrStore(t)
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
t.Fatal(err)
}
// Retrieve attributes for column #1.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
t.Fatalf("unexpected attrs(1): %#v", m)
}
// Retrieve attributes for column #2.
if m, err := s.Attrs(2); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
t.Fatalf("unexpected attrs(2): %#v", m)
}
}
// Ensure database returns a non-nil empty map if unset.
func TestAttrStore_Attrs_Empty(t *testing.T) {
s := MustOpenAttrStore(t)
defer s.Close()
if m, err := s.Attrs(100); err != nil {
t.Fatal(err)
} else if m == nil || len(m) > 0 {
t.Fatalf("unexpected attrs: %#v", m)
}
}
// Ensure database can unset attributes if explicitly set to nil.
func TestAttrStore_Attrs_Unset(t *testing.T) {
s := MustOpenAttrStore(t)
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil {
t.Fatal(err)
}
// Verify attributes.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) {
t.Fatalf("unexpected attrs: %#v", m)
}
}
// Ensure attribute block checksums can be returned.
func TestAttrStore_Blocks(t *testing.T) {
s := MustOpenAttrStore(t)
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil {
t.Fatal(err)
}
// Retrieve blocks.
blks0, err := s.Blocks()
if err != nil {
t.Fatal(err)
} else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 {
t.Fatalf("unexpected blocks: %#v", blks0)
}
// Change second block.
if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil {
t.Fatal(err)
}
// Ensure second block changed.
blks1, err := s.Blocks()
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(blks0[0], blks1[0]) {
t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0])
} else if reflect.DeepEqual(blks0[1], blks1[1]) {
t.Fatalf("block 1 match: %#v ", blks0[0])
} else if !reflect.DeepEqual(blks0[2], blks1[2]) {
t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
}
}
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(tb testing.TB) pilosa.AttrStore {
f, err := testhook.TempFile(tb, "pilosa-attr-")
if err != nil {
panic(err)
}
// Note, even though the file is closed, TempFile will still avoid
// creating the same name again if we leave it existing. The boltdb
// code may already be deleting this, so the TestHook deletion
// may not matter but it's more reliable this way.
f.Close()
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore(b)
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
errchan := make(chan error)
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
errchan <- err
}
}
}()
}
go func() {
wg.Wait()
close(errchan)
}()
if err := <-errchan; err != nil {
b.Fatal(err)
}
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore(tb testing.TB) pilosa.AttrStore {
s := NewAttrStore(tb)
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

433
boltdb/attrstore.go
View file

@ -1,433 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"sort"
"sync"
"time"
"github.com/cespare/xxhash"
"github.com/pilosa/pilosa/v2"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
// attrBlockSize is the size of attribute blocks for anti-entropy.
const attrBlockSize = 100
// attrCache represents a cache for attributes.
type attrCache struct {
mu sync.RWMutex
attrs map[uint64]map[string]interface{}
}
// Get returns the cached attributes for a given id.
func (c *attrCache) Get(id uint64) map[string]interface{} {
c.mu.RLock()
defer c.mu.RUnlock()
attrs := c.attrs[id]
if attrs == nil {
return nil
}
// Make a copy for safety
ret := make(map[string]interface{})
for k, v := range attrs {
ret[k] = v
}
return ret
}
// Set updates the cached attributes for a given id.
func (c *attrCache) Set(id uint64, attrs map[string]interface{}) {
c.mu.Lock()
defer c.mu.Unlock()
c.attrs[id] = attrs
}
// attrStore represents a storage layer for attributes.
type attrStore struct {
mu sync.RWMutex
path string
db *bolt.DB
attrCache *attrCache
}
// newAttrCache returns a new instance of AttrCache.
func newAttrCache() *attrCache {
return &attrCache{
attrs: make(map[uint64]map[string]interface{}),
}
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(path string) pilosa.AttrStore {
return &attrStore{
path: path,
attrCache: newAttrCache(),
}
}
// Path returns path to the store's data file.
func (s *attrStore) Path() string { return s.path }
// Open opens and initializes the store.
func (s *attrStore) Open() error {
// Open storage.
db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return errors.Wrap(err, "opening storage")
}
s.db = db
// Initialize database.
if err := s.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte("attrs"))
return err
}); err != nil {
return errors.Wrap(err, "initializing")
}
return nil
}
// Close closes the store.
func (s *attrStore) Close() error {
if s.db != nil {
s.db.Close()
}
return nil
}
// Attrs returns a set of attributes by ID.
func (s *attrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
// Check cache for map.
if m = s.attrCache.Get(id); m != nil {
return m, nil
}
// Find attributes from storage.
if err = s.db.View(func(tx *bolt.Tx) error {
m, err = txAttrs(tx, id)
return err
}); err != nil {
return nil, errors.Wrap(err, "finding attributes")
}
// Add to cache.
s.attrCache.Set(id, m)
return m, nil
}
// SetAttrs sets attribute values for a given ID.
func (s *attrStore) SetAttrs(id uint64, m map[string]interface{}) error {
// Ignore empty maps.
if len(m) == 0 {
return nil
}
// Check if the attributes already exist under a read-only lock.
if attr, err := s.Attrs(id); err != nil {
return errors.Wrap(err, "checking attrs")
} else if attr != nil && mapContains(attr, m) {
return nil
}
// Obtain write lock.
s.mu.Lock()
defer s.mu.Unlock()
var attr map[string]interface{}
if err := s.db.Update(func(tx *bolt.Tx) error {
tmp, err := txUpdateAttrs(tx, id, m)
if err != nil {
return err
}
attr = tmp
return nil
}); err != nil {
return errors.Wrap(err, "updating store")
}
// Swap attributes map in cache.
s.attrCache.Set(id, attr)
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *attrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
s.mu.Lock()
defer s.mu.Unlock()
attrs := make(map[uint64]map[string]interface{})
if err := s.db.Update(func(tx *bolt.Tx) error {
// Collect and sort keys.
ids := make([]uint64, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
// Update attributes for each id.
for _, id := range ids {
attr, err := txUpdateAttrs(tx, id, m[id])
if err != nil {
return err
}
attrs[id] = attr
}
return nil
}); err != nil {
return err
}
// Swap attributes map in cache.
for id, attr := range attrs {
s.attrCache.Set(id, attr)
}
return nil
}
// Blocks returns a list of all blocks in the store.
func (s *attrStore) Blocks() (blocks []pilosa.AttrBlock, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
// Wrap cursor to segment by block.
cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), attrBlockSize)
// Iterate over each block.
for cur.nextBlock() {
block := pilosa.AttrBlock{ID: cur.blockID()}
// Compute checksum of every key/value in block.
h := xxhash.New()
for k, v := cur.next(); k != nil; k, v = cur.next() {
// hash function writes don't usually need to be checked
_, _ = h.Write(k)
_, _ = h.Write(v)
}
block.Checksum = h.Sum(nil)
// Append block.
blocks = append(blocks, block)
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "getting blocks")
}
return blocks, nil
}
// BlockData returns all data for a single block.
func (s *attrStore) BlockData(i uint64) (m map[uint64]map[string]interface{}, err error) {
m = make(map[uint64]map[string]interface{})
// Start read-only transaction.
err = s.db.View(func(tx *bolt.Tx) error {
// Move to the start of the block.
min := u64tob(i * attrBlockSize)
max := u64tob((i + 1) * attrBlockSize)
cur := tx.Bucket([]byte("attrs")).Cursor()
for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
// Exit if we're past the end of the block.
if bytes.Compare(k, max) != -1 {
break
}
// Decode attribute map and associate with id.
attrs, err := pilosa.DecodeAttrs(v)
if err != nil {
return errors.Wrap(err, "decoding attrs")
}
m[btou64(k)] = attrs
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "getting block data")
}
return m, nil
}
// WriteTo writes the underlying database to w.
func (s *attrStore) WriteTo(w io.Writer) (int64, error) {
tx, err := s.db.Begin(false)
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback() }()
return tx.WriteTo(w)
}
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {
return emptyMap, nil
}
return pilosa.DecodeAttrs(v)
}
// txUpdateAttrs updates the attributes for an id.
// Returns the new combined set of attributes for the id.
func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
attr, err := txAttrs(tx, id)
if err != nil {
return nil, err
}
// Create a new map if it is empty so we don't update emptyMap.
if len(attr) == 0 {
attr = make(map[string]interface{}, len(m))
}
// Merge attributes with original values.
// Nil values should delete keys.
for k, v := range m {
if v == nil {
delete(attr, k)
continue
}
switch v := v.(type) {
case int:
attr[k] = int64(v)
case uint:
attr[k] = int64(v)
case uint64:
attr[k] = int64(v)
case string, int64, bool, float64:
attr[k] = v
default:
return nil, fmt.Errorf("invalid attr type: %T", v)
}
}
// Marshal and save new values.
buf, err := pilosa.EncodeAttrs(attr)
if err != nil {
return nil, errors.Wrap(err, "encoding attrs")
}
if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
return nil, errors.Wrap(err, "saving attrs")
}
return attr, nil
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
// emptyMap is a reusable map that contains no keys.
var emptyMap = make(map[string]interface{})
// mapContains returns true if all keys & values of subset are in m.
func mapContains(m, subset map[string]interface{}) bool {
for k, v := range subset {
value, ok := m[k]
if !ok || value != v {
return false
}
}
return true
}
// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
type blockCursor struct {
cur *bolt.Cursor
base uint64
n uint64
buf struct {
key []byte
value []byte
filled bool
}
}
// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
func newBlockCursor(c *bolt.Cursor, n int) blockCursor { // nolint: unparam
cur := blockCursor{
cur: c,
n: uint64(n),
}
cur.buf.key, cur.buf.value = c.First()
cur.buf.filled = true
return cur
}
// blockID returns the current block ID. Only valid after call to nextBlock().
func (cur *blockCursor) blockID() uint64 { return cur.base }
// nextBlock moves the cursor to the next block.
// Returns true if another block exists, otherwise returns false.
func (cur *blockCursor) nextBlock() bool {
if cur.buf.key == nil {
return false
}
cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
return true
}
// next returns the next key/value within the block.
// Returns nils at the end of the block.
func (cur *blockCursor) next() (key, value []byte) {
// Use buffered value, if set.
if cur.buf.filled {
key, value = cur.buf.key, cur.buf.value
cur.buf.filled = false
return key, value
}
// Read next key.
key, value = cur.cur.Next()
// Fill buffer for EOF.
if key == nil {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
return nil, nil
}
// Parse key and buffer if outside of block.
id := binary.BigEndian.Uint64(key)
if id/cur.n > cur.base {
cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
return nil, nil
}
return key, value
}

10
boltdb/translate.go
View file

@ -615,3 +615,13 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string {
}
return string(boltKey)
}
// u64tob encodes v to big endian encoding.
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
// btou64 decodes b from big endian encoding.
func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }

29
cache.go
View file

@ -580,35 +580,6 @@ func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
ret := make([]uint64, 0, len(p))
i, j := 0, 0
for i < len(p) && j < len(other) {
a, b := p[i], other[j]
if a == b {
ret = append(ret, a)
i, j = i+1, j+1
} else if a < b {
ret = append(ret, a)
i++
} else {
ret = append(ret, b)
j++
}
}
if i < len(p) {
ret = append(ret, p[i:]...)
} else if j < len(other) {
ret = append(ret, other[j:]...)
}
return ret
}
// simpleCache implements a bitmap Rowcache.
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same row within a short time frame (i.e. good for write-heavy loads)

23
client.go
View file

@ -74,20 +74,15 @@ type InternalClient interface {
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error
ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error)
IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
IndexAttrDataReader(ctx context.Context, index string) (io.ReadCloser, error)
FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error)
FieldAttrDataReader(ctx context.Context, index, field string) (io.ReadCloser, error)
StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
FinishTransaction(ctx context.Context, id string) (*Transaction, error)
@ -205,10 +200,6 @@ func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, ind
return nil
}
func (n nopInternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *ImportColumnAttrsRequest) error {
return nil
}
func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) {
return nil, nil
}
@ -221,18 +212,10 @@ func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index s
return nil, nil
}
func (n nopInternalClient) IndexAttrDataReader(ctx context.Context, index string) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) FieldAttrDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
return nil
}
@ -261,12 +244,6 @@ func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, in
func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n nopInternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
return nil
}

39
client/client.go
View file

@ -36,8 +36,8 @@ import (
"github.com/golang/protobuf/proto" //nolint:staticcheck
"github.com/opentracing/opentracing-go"
"github.com/pilosa/pilosa/v2"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/logger"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/pb"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
@ -1318,11 +1318,8 @@ func newHTTPClient(options *ClientOptions) *http.Client {
func makeRequestData(query string, options *QueryOptions) ([]byte, error) {
request := &pb.QueryRequest{
Query: query,
Shards: options.Shards,
ColumnAttrs: options.ColumnAttrs,
ExcludeRowAttrs: options.ExcludeRowAttrs,
ExcludeColumns: options.ExcludeColumns,
Query: query,
Shards: options.Shards,
}
r, err := proto.Marshal(request)
if err != nil {
@ -1492,12 +1489,6 @@ func (co *ClientOptions) withDefaults() (updated *ClientOptions) {
type QueryOptions struct {
// Shards restricts query to a subset of shards. Queries all shards if nil.
Shards []uint64
// ColumnAttrs enables returning columns in the query response.
ColumnAttrs bool
// ExcludeRowAttrs inhibits returning attributes
ExcludeRowAttrs bool
// ExcludeColumns inhibits returning columns
ExcludeColumns bool
}
func (qo *QueryOptions) addOptions(options ...interface{}) error {
@ -1528,14 +1519,6 @@ func (qo *QueryOptions) addOptions(options ...interface{}) error {
// QueryOption is used when using options with a client.Query,
type QueryOption func(options *QueryOptions) error
// OptQueryColumnAttrs enables returning column attributes in the result.
func OptQueryColumnAttrs(enable bool) QueryOption {
return func(options *QueryOptions) error {
options.ColumnAttrs = enable
return nil
}
}
// OptQueryShards restricts the set of shards on which a query operates.
func OptQueryShards(shards ...uint64) QueryOption {
return func(options *QueryOptions) error {
@ -1544,22 +1527,6 @@ func OptQueryShards(shards ...uint64) QueryOption {
}
}
// OptQueryExcludeAttrs enables discarding attributes from a result,
func OptQueryExcludeAttrs(enable bool) QueryOption {
return func(options *QueryOptions) error {
options.ExcludeRowAttrs = enable
return nil
}
}
// OptQueryExcludeColumns enables discarding columns from a result,
func OptQueryExcludeColumns(enable bool) QueryOption {
return func(options *QueryOptions) error {
options.ExcludeColumns = enable
return nil
}
}
// ImportOptions are the options for controlling the importer
type ImportOptions struct {
threadCount int

69
client/client_it_test.go
View file

@ -132,59 +132,6 @@ func TestClientAgainstCluster(t *testing.T) {
require.Equalf([]uint64{1, shardWidth * 3}, cols, "Unexpected results: %#v", cols)
})
t.Run("QueryWithColumns", func(t *testing.T) {
setup(t, require, cli)
defer tearDown(t, require, cli)
targetAttrs := map[string]interface{}{
"name": "some string",
"age": int64(95),
"registered": true,
"height": 1.83,
}
_, err := cli.Query(testField.Set(1, 100))
require.NoErrorf(err, "Set(1, 100)")
resp, err := cli.Query(testIndex.SetColumnAttrs(100, targetAttrs))
require.NoErrorf(err, "SetColumnAttrs(100, %v)", targetAttrs)
require.Equalf(ColumnItem{}, resp.Column(), "No columns should be returned if it wasn't explicitly requested")
resp, err = cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true})
require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}")
require.Equalf(1, len(resp.ColumnAttrs()), "ColumnAttrs count should be == 1")
cols := resp.Columns()
require.Equalf(1, len(cols), "Column count")
require.Equalf(uint64(100), cols[0].ID, "Column ID")
require.Equalf(targetAttrs, cols[0].Attributes, "Column attrs.")
require.Equalf(cols[0], resp.Column(), "Column() should be equivalent to first column in the response")
})
t.Run("SetRowAttrs", func(t *testing.T) {
setup(t, require, cli)
defer tearDown(t, require, cli)
targetAttrs := map[string]interface{}{
"name": "some string",
"age": int64(95),
"registered": true,
"height": 1.83,
}
_, err := cli.Query(testField.Set(1, 100))
require.NoErrorf(err, "Set(1, 100)")
_, err = cli.Query(testField.SetRowAttrs(1, targetAttrs))
require.NoErrorf(err, "SetRowAttrs(1, %v)", targetAttrs)
resp, err := cli.Query(testField.Row(1), &QueryOptions{ColumnAttrs: true})
require.NoErrorf(err, "Row(1) QueryOptions{ColumnAttrs: true}")
require.Equalf(targetAttrs, resp.Result().Row().Attributes, "Row attributes should be set")
})
t.Run("OrmCount", func(t *testing.T) {
setup(t, require, cli)
defer tearDown(t, require, cli)
@ -278,19 +225,6 @@ func TestClientAgainstCluster(t *testing.T) {
item := items[0]
require.Equalf(uint64(10), item.ID, "TopN result item[0].ID")
require.Equalf(uint64(3), item.Count, "TopN result item[0].Count")
_, err = cli.Query(testFieldTopN.SetRowAttrs(10, map[string]interface{}{"foo": "bar"}))
require.NoErrorf(err, "SetRowAttrs(10)")
resp, err = cli.Query(testFieldTopN.FilterAttrTopN(5, nil, "foo", "bar"))
require.NoErrorf(err, `FilterAttrTopN(5, nil, "foo", "bar")`)
items = resp.Result().CountItems()
require.Equalf(1, len(items), "FilterAttrTopN result CountItems")
item = items[0]
require.Equalf(uint64(10), item.ID, "FilterAttrTopN result item[0].ID")
require.Equalf(uint64(3), item.Count, "FilterAttrTopN result item[0].Count")
})
t.Run("MinMaxRow", func(t *testing.T) {
@ -575,8 +509,7 @@ func TestClientAgainstCluster(t *testing.T) {
uri, _ := pnet.NewURIFromAddress("does-not-resolve.foo.bar")
tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0))
attrs := map[string]interface{}{"a": 1}
_, err := tmpcli.Query(testIndex.SetColumnAttrs(0, attrs))
_, err := tmpcli.Query(testIndex.All())
require.Error(err, ErrTriedMaxHosts)
})

73
client/client_test.go
View file

@ -30,8 +30,7 @@ func TestQueryWithError(t *testing.T) {
var err error
client := DefaultClient()
index := NewIndex("foo")
field := index.Field("foo")
invalid := field.FilterAttrTopN(12, field.Row(7), "$invalid$", 80, 81)
invalid := NewPQLRowQuery("", index, errors.New("invalid"))
_, err = client.Query(invalid, nil)
if err == nil {
t.Fatalf("Should have failed")
@ -207,76 +206,6 @@ func ClientOptionErr(int) ClientOption {
}
}
func TestQueryOptions(t *testing.T) {
targets := []*QueryOptions{
{ColumnAttrs: true},
{ColumnAttrs: false},
{ExcludeRowAttrs: true},
{ExcludeRowAttrs: false},
{ExcludeColumns: true},
{ExcludeColumns: false},
}
optionsList := [][]interface{}{
{OptQueryColumnAttrs(true)},
{OptQueryColumnAttrs(false)},
{OptQueryExcludeAttrs(true)},
{OptQueryExcludeAttrs(false)},
{OptQueryExcludeColumns(true)},
{OptQueryExcludeColumns(false)},
}
for i := 0; i < len(targets); i++ {
options := &QueryOptions{}
err := options.addOptions(optionsList[i]...)
if err != nil {
t.Fatal(err)
}
target := targets[i]
if !reflect.DeepEqual(target, options) {
t.Fatalf("%v != %v", target, options)
}
}
target := &QueryOptions{
ColumnAttrs: true,
ExcludeRowAttrs: true,
ExcludeColumns: true,
}
options := &QueryOptions{}
err := options.addOptions(&QueryOptions{
ColumnAttrs: true,
ExcludeRowAttrs: true,
ExcludeColumns: true,
})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(target, options) {
t.Fatalf("%v != %v", target, options)
}
}
func TestQueryOptionsWithError(t *testing.T) {
options := &QueryOptions{}
err := options.addOptions(1)
if err == nil {
t.Fatalf("should have failed")
}
err = options.addOptions(OptQueryColumnAttrs(true), nil)
if err == nil {
t.Fatalf("should have failed")
}
err = options.addOptions(OptQueryColumnAttrs(true), &QueryOptions{})
if err == nil {
t.Fatalf("should have failed")
}
err = options.addOptions(QueryOptionErr(0))
if err == nil {
t.Fatalf("should have failed")
}
}
func TestQueryOptionsError(t *testing.T) {
client := DefaultClient()
index := NewIndex("foo")

3
client/docs/data-model-queries.md
View file

@ -124,7 +124,6 @@ Index:
* `Xor(rows ...*PQLRowQuery) *PQLRowQuery`
* `Not(row) *PQLRowQuery`
* `Count(row *PQLRowQuery) *PQLBaseQuery`
* `SetColumnAttrs(columnID uint64, attrs map[string]interface{}) *PQLBaseQuery`
* `Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery`
Field:
@ -135,10 +134,8 @@ Field:
* `Clear(rowID uint64, columnID uint64) *PQLBaseQuery`
* `TopN(n uint64) *PQLRowQuery`
* `RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery`
* `FilterFieldTopN(n uint64, row *PQLRowQuery, field string, values ...interface{}) *PQLRowQuery`
* `Range(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
* `RowRange(rowID uint64, start time.Time, end time.Time) *PQLRowQuery`
* `SetRowAttrs(rowID uint64, attrs map[string]interface{}) *PQLBaseQuery`
* `ClearRow(rowIDOrKey interface{}) *PQLBaseQuery`
* `Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery`
* `LT(n int) *PQLRowQuery`

18
client/docs/server-interaction.md
View file

@ -101,12 +101,6 @@ You can send queries to a Pilosa server using the `Query` function of the `Clien
response, err := cli.Query(field.Row(5));
```
`Query` accepts zero or more options:
```go
response, err := cli.Query(field.Row(5), pilosa.ColumnAttrs(true), pilosa.ExcludeColumns(true))
```
## Server Response
When a query is sent to a Pilosa server, the server either fulfills the query or sends an error message. In the case of an error, a `pilosa.Error` struct is returned, otherwise a `QueryResponse` struct is returned.
@ -131,17 +125,6 @@ for _, result := range response.Results() {
}
```
Similarly, a `QueryResponse` struct may include a number of column attributes if `ColumnAttrs` query option was set to `true`:
```go
var column *pilosa.ColumnItem
// iterate over all columns
for _, column = range response.ColumnAttrs() {
// Act on the column item
}
```
`QueryResult` objects contain:
* `Row()` function to retrieve a row result,
@ -153,7 +136,6 @@ for _, column = range response.ColumnAttrs() {
```go
row := result.Row()
columns := row.Columns
attributes := row.Attributes
countItems := result.CountItems()

117
client/orm.go
View file

@ -21,7 +21,6 @@ import (
"encoding/json"
"fmt"
"math"
"sort"
"strconv"
"strings"
"sync"
@ -343,51 +342,24 @@ func OptIndexTrackExistence(trackExistence bool) IndexOption {
// OptionsOptions is used to pass an option to Option call.
type OptionsOptions struct {
columnAttrs bool
excludeColumns bool
excludeRowAttrs bool
shards []uint64
shards []uint64
}
func (oo OptionsOptions) marshal() string {
part1 := fmt.Sprintf("columnAttrs=%s,excludeColumns=%s,excludeRowAttrs=%s",
strconv.FormatBool(oo.columnAttrs),
strconv.FormatBool(oo.excludeColumns),
strconv.FormatBool(oo.excludeRowAttrs))
if oo.shards != nil {
shardsStr := make([]string, len(oo.shards))
for i, shard := range oo.shards {
shardsStr[i] = strconv.FormatUint(shard, 10)
}
return fmt.Sprintf("%s,shards=[%s]", part1, strings.Join(shardsStr, ","))
return fmt.Sprintf("shards=[%s]", strings.Join(shardsStr, ","))
}
return part1
return ""
}
// OptionsOption is an option for Index.Options call.
type OptionsOption func(options *OptionsOptions)
// OptOptionsColumnAttrs enables returning column attributes.
func OptOptionsColumnAttrs(enable bool) OptionsOption {
return func(options *OptionsOptions) {
options.columnAttrs = enable
}
}
// OptOptionsExcludeColumns enables preventing returning columns.
func OptOptionsExcludeColumns(enable bool) OptionsOption {
return func(options *OptionsOptions) {
options.excludeColumns = enable
}
}
// OptOptionsExcludeRowAttrs enables preventing returning row attributes.
func OptOptionsExcludeRowAttrs(enable bool) OptionsOption {
return func(options *OptionsOptions) {
options.excludeRowAttrs = enable
}
}
// OptOptionsShards run the query using only the data from the given shards.
// By default, the entire data set (i.e. data from all shards) is used.
func OptOptionsShards(shards ...uint64) OptionsOption {
@ -397,7 +369,7 @@ func OptOptionsShards(shards ...uint64) OptionsOption {
}
// Index is a Pilosa index. The purpose of the Index is to represent a data namespace.
// You cannot perform cross-index queries. Column-level attributes are global to the Index.
// You cannot perform cross-index queries.
type Index struct {
mu sync.RWMutex
name string
@ -582,22 +554,6 @@ func (idx *Index) All() *PQLRowQuery {
// TODO: impelement AllLimit(limit, offset uint64) *PQLRowQuery
// SetColumnAttrs creates a SetColumnAttrs query.
// SetColumnAttrs associates arbitrary key/value pairs with a column in an index.
// Following types are accepted: integer, float, string and boolean types.
func (idx *Index) SetColumnAttrs(colIDOrKey interface{}, attrs map[string]interface{}) *PQLBaseQuery {
colStr, err := formatIDKey(colIDOrKey)
if err != nil {
return NewPQLBaseQuery("", idx, err)
}
attrsString, err := createAttributesString(attrs)
if err != nil {
return NewPQLBaseQuery("", idx, err)
}
q := fmt.Sprintf("SetColumnAttrs(%s,%s)", colStr, attrsString)
return NewPQLBaseQuery(q, idx, nil)
}
// Options creates an Options query.
func (idx *Index) Options(row *PQLRowQuery, opts ...OptionsOption) *PQLBaseQuery {
oo := &OptionsOptions{}
@ -1053,7 +1009,6 @@ func OptFieldForeignIndex(index string) FieldOption {
// Field structs are used to segment and define different functional characteristics within your entire index.
// You can think of a Field as a table-like data partition within your Index.
// Row-level attributes are namespaced at the Field level.
type Field struct {
name string
createdAt int64
@ -1096,7 +1051,6 @@ func (f *Field) copy() *Field {
// Row creates a Row query.
// Row retrieves the indices of all the set columns in a row.
// It also retrieves any attributes set on that row or column.
func (f *Field) Row(rowIDOrKey interface{}) *PQLRowQuery {
rowStr, err := formatIDKeyBool(rowIDOrKey)
if err != nil {
@ -1174,32 +1128,6 @@ func (f *Field) RowTopN(n uint64, row *PQLRowQuery) *PQLRowQuery {
return q
}
// FilterAttrTopN creates a TopN query with the given item count, row, attribute name and filter values for that field
// The attrName and attrValues arguments work together to only return Rows which have the attribute specified by attrName with one of the values specified in attrValues.
func (f *Field) FilterAttrTopN(n uint64, row *PQLRowQuery, attrName string, attrValues ...interface{}) *PQLRowQuery {
return f.filterAttrTopN(n, row, attrName, attrValues...)
}
func (f *Field) filterAttrTopN(n uint64, row *PQLRowQuery, field string, values ...interface{}) *PQLRowQuery {
if err := validateLabel(field); err != nil {
return NewPQLRowQuery("", f.index, err)
}
b, err := json.Marshal(values)
if err != nil {
return NewPQLRowQuery("", f.index, err)
}
var q *PQLRowQuery
if row == nil {
q = NewPQLRowQuery(fmt.Sprintf("TopN(%s,n=%d,attrName='%s',attrValues=%s)",
f.name, n, field, string(b)), f.index, nil)
} else {
serializedRow := row.serialize()
q = NewPQLRowQuery(fmt.Sprintf("TopN(%s,%s,n=%d,attrName='%s',attrValues=%s)",
f.name, serializedRow.String(), n, field, string(b)), f.index, nil)
}
return q
}
// Range creates a Range query.
// Similar to Row, but only returns columns which were set with timestamps between the given start and end timestamps.
// *Deprecated at Pilosa 1.3*
@ -1226,24 +1154,6 @@ func (f *Field) RowRange(rowIDOrKey interface{}, start time.Time, end time.Time)
return q
}
// SetRowAttrs creates a SetRowAttrs query.
// SetRowAttrs associates arbitrary key/value pairs with a row in a field.
// Following types are accepted: integer, float, string and boolean types.
func (f *Field) SetRowAttrs(rowIDOrKey interface{}, attrs map[string]interface{}) *PQLBaseQuery {
rowStr, err := formatIDKeyBool(rowIDOrKey)
if err != nil {
return NewPQLBaseQuery("", f.index, err)
}
attrsString, err := createAttributesString(attrs)
if err != nil {
return NewPQLBaseQuery("", f.index, err)
}
text := fmt.Sprintf("SetRowAttrs(%s,%s,%s)", f.name, rowStr, attrsString)
q := NewPQLBaseQuery(text, f.index, nil)
q.hasKeys = f.options.keys || f.index.options.keys
return q
}
// Store creates a Store call.
// Store writes the result of the row query to the specified row. If the row already exists, it will be replaced. The destination field must be of field type set.
func (f *Field) Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery {
@ -1254,23 +1164,6 @@ func (f *Field) Store(row *PQLRowQuery, rowIDOrKey interface{}) *PQLBaseQuery {
return NewPQLBaseQuery(fmt.Sprintf("Store(%s,%s=%s)", row.serialize().String(), f.name, rowStr), f.index, nil)
}
func createAttributesString(attrs map[string]interface{}) (string, error) {
attrsList := make([]string, 0, len(attrs))
for k, v := range attrs {
// TODO: validate the type of v is one of string, int64, float64, bool
if err := validateLabel(k); err != nil {
return "", err
}
if vs, ok := v.(string); ok {
attrsList = append(attrsList, fmt.Sprintf("%s=%s", k, strconv.Quote(vs)))
} else {
attrsList = append(attrsList, fmt.Sprintf("%s=%v", k, v))
}
}
sort.Strings(attrsList)
return strings.Join(attrsList, ","), nil
}
func formatIDKey(idKey interface{}) (string, error) {
switch v := idKey.(type) {
case uint:

92
client/orm_test.go
View file

@ -398,12 +398,6 @@ func TestORM(t *testing.T) {
comparePQL(t,
"TopN(collaboration,Row(collaboration=3),n=10)",
collabField.RowTopN(10, collabField.Row(3)))
comparePQL(t,
"TopN(sample-field,Row(collaboration=7),n=12,attrName='category',attrValues=[80,81])",
sampleField.FilterAttrTopN(12, collabField.Row(7), "category", 80, 81))
comparePQL(t,
"TopN(sample-field,n=12,attrName='category',attrValues=[80,81])",
sampleField.FilterAttrTopN(12, nil, "category", 80, 81))
})
t.Run("FieldLT", func(t *testing.T) {
@ -511,22 +505,8 @@ func TestORM(t *testing.T) {
}
})
t.Run("FilterFieldTopNInvalidField", func(t *testing.T) {
q := sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81)
if q.Error() == nil {
t.Fatalf("should have failed")
}
})
t.Run("FilterFieldTopNInvalidValue", func(t *testing.T) {
q := sampleField.FilterAttrTopN(12, collabField.Row(7), "category", 80, func() {})
if q.Error() == nil {
t.Fatalf("should have failed")
}
})
t.Run("RowOperationInvalidArg", func(t *testing.T) {
invalid := sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81)
invalid := NewPQLRowQuery("", sampleIndex, errors.New("invalid"))
// invalid argument in pos 1
q := sampleIndex.Union(invalid, b1)
if q.Error() == nil {
@ -560,64 +540,6 @@ func TestORM(t *testing.T) {
}
})
t.Run("SetColumnAttrs", func(t *testing.T) {
attrs := map[string]interface{}{
"quote": "\"Don't worry, be happy\"",
"happy": true,
}
comparePQL(t,
"SetColumnAttrs(5,happy=true,quote=\"\\\"Don't worry, be happy\\\"\")",
projectIndex.SetColumnAttrs(5, attrs))
q := projectIndex.SetColumnAttrs(false, attrs)
if q.err == nil {
t.Fatalf("should have failed")
}
})
t.Run("SetColumnAttrsInvalidAttr", func(t *testing.T) {
attrs := map[string]interface{}{
"color": "blue",
"$invalid$": true,
}
if projectIndex.SetColumnAttrs(5, attrs).Error() == nil {
t.Fatalf("Should have failed")
}
})
t.Run("SetRowAttrs", func(t *testing.T) {
attrs := map[string]interface{}{
"quote": "\"Don't worry, be happy\"",
"active": true,
}
comparePQL(t,
`SetRowAttrs(collaboration,5,active=true,quote="\"Don't worry, be happy\"")`,
collabField.SetRowAttrs(5, attrs))
comparePQL(t,
"SetRowAttrs(collaboration,'foo',active=true,quote=\"\\\"Don't worry, be happy\\\"\")",
collabField.SetRowAttrs("foo", attrs))
q := collabField.SetRowAttrs(nil, attrs)
if q.err == nil {
t.Fatalf("should have failed")
}
})
t.Run("SetRowAttrsInvalidAttr", func(t *testing.T) {
attrs := map[string]interface{}{
"color": "blue",
"$invalid$": true,
}
if collabField.SetRowAttrs(5, attrs).Error() == nil {
t.Fatalf("Should have failed")
}
if collabField.SetRowAttrs("foo", attrs).Error() == nil {
t.Fatalf("Should have failed")
}
})
t.Run("Store", func(t *testing.T) {
comparePQL(t,
"Store(Row(collaboration=5),sample-field=10)",
@ -630,18 +552,10 @@ func TestORM(t *testing.T) {
t.Run("Options", func(t *testing.T) {
comparePQL(t,
"Options(Row(collaboration=5),columnAttrs=true,excludeColumns=true,excludeRowAttrs=true,shards=[1,3])",
"Options(Row(collaboration=5),shards=[1,3])",
sampleIndex.Options(collabField.Row(5),
OptOptionsColumnAttrs(true),
OptOptionsExcludeColumns(true),
OptOptionsExcludeRowAttrs(true),
OptOptionsShards(1, 3),
))
comparePQL(t,
"Options(Row(collaboration=5),columnAttrs=true,excludeColumns=false,excludeRowAttrs=false)",
sampleIndex.Options(collabField.Row(5),
OptOptionsColumnAttrs(true),
))
})
t.Run("BatchQuery", func(t *testing.T) {
@ -664,7 +578,7 @@ func TestORM(t *testing.T) {
t.Run("BatchQueryWithError", func(t *testing.T) {
q := sampleIndex.BatchQuery()
q.Add(sampleField.FilterAttrTopN(12, collabField.Row(7), "$invalid$", 80, 81))
q.Add(NewPQLBaseQuery("", nil, errors.New("invalid")))
if q.Error() == nil {
t.Fatalf("The error must be set")
}

109
client/response.go
View file

@ -19,7 +19,6 @@ package client
import (
"encoding/json"
"errors"
"fmt"
"github.com/pilosa/pilosa/v2/pb"
@ -45,7 +44,6 @@ const (
// QueryResponse represents the response from a Pilosa query.
type QueryResponse struct {
ResultList []QueryResult `json:"results,omitempty"`
ColumnList []ColumnItem `json:"columns,omitempty"`
ErrorMessage string `json:"error-message,omitempty"`
Success bool `json:"success,omitempty"`
}
@ -65,18 +63,9 @@ func newQueryResponseFromInternal(response *pb.QueryResponse) (*QueryResponse, e
}
results = append(results, result)
}
columns := make([]ColumnItem, 0, len(response.ColumnAttrSets))
for _, p := range response.ColumnAttrSets {
columnItem, err := newColumnItemFromInternal(p)
if err != nil {
return nil, err
}
columns = append(columns, columnItem)
}
return &QueryResponse{
ResultList: results,
ColumnList: columns,
Success: true,
}, nil
}
@ -94,26 +83,6 @@ func (qr *QueryResponse) Result() QueryResult {
return qr.ResultList[0]
}
// Columns returns all column attributes in the response.
// *DEPRECATED*
func (qr *QueryResponse) Columns() []ColumnItem {
return qr.ColumnList
}
// Column returns the attributes for first column.
// *DEPRECATED*
func (qr *QueryResponse) Column() ColumnItem {
if len(qr.ColumnList) == 0 {
return ColumnItem{}
}
return qr.ColumnList[0]
}
// ColumnAttrs returns all column attributes in the response.
func (qr *QueryResponse) ColumnAttrs() []ColumnItem {
return qr.ColumnList
}
// QueryResult represents one of the results in the response.
type QueryResult interface {
Type() uint32
@ -256,22 +225,15 @@ func (TopNResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersR
// RowResult represents a result from Row, Union, Intersect, Difference and Range PQL calls.
type RowResult struct {
Attributes map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
}
func newRowResultFromInternal(row *pb.Row) (*RowResult, error) {
attrs, err := convertInternalAttrsToMap(row.Attrs)
if err != nil {
return nil, err
}
result := &RowResult{
Attributes: attrs,
Columns: row.Columns,
Keys: row.Keys,
}
return result, nil
return &RowResult{
Columns: row.Columns,
Keys: row.Keys,
}, nil
}
// Type is the type of this result.
@ -312,13 +274,11 @@ func (b RowResult) MarshalJSON() ([]byte, error) {
keys = []string{}
}
return json.Marshal(struct {
Attributes map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys"`
}{
Attributes: b.Attributes,
Columns: columns,
Keys: keys,
Columns: columns,
Keys: keys,
})
}
@ -415,7 +375,7 @@ func (BoolResult) GroupCounts() []GroupCount { return nil }
// RowIdentifiers returns the result of a Rows call.
func (BoolResult) RowIdentifiers() RowIdentifiersResult { return RowIdentifiersResult{} }
// NilResult is returned from calls which don't return a value, such as SetRowAttrs.
// NilResult is returned from calls which don't return a value.
type NilResult struct{}
// Type is the type of this result.
@ -546,50 +506,3 @@ func groupCountsFromInternal(items *pb.GroupCounts) GroupCountResult {
}
return GroupCountResult(result)
}
const (
stringType = 1
intType = 2
boolType = 3
floatType = 4
)
func convertInternalAttrsToMap(attrs []*pb.Attr) (attrsMap map[string]interface{}, err error) {
attrsMap = make(map[string]interface{}, len(attrs))
for _, attr := range attrs {
switch attr.Type {
case stringType:
attrsMap[attr.Key] = attr.StringValue
case intType:
attrsMap[attr.Key] = attr.IntValue
case boolType:
attrsMap[attr.Key] = attr.BoolValue
case floatType:
attrsMap[attr.Key] = attr.FloatValue
default:
return nil, errors.New("Unknown attribute type")
}
}
return attrsMap, nil
}
// ColumnItem represents data about a column.
// Column data is only returned if QueryOptions.Columns was set to true.
type ColumnItem struct {
ID uint64 `json:"id,omitempty"`
Key string `json:"key,omitempty"`
Attributes map[string]interface{} `json:"attributes,omitempty"`
}
func newColumnItemFromInternal(column *pb.ColumnAttrSet) (ColumnItem, error) {
attrs, err := convertInternalAttrsToMap(column.Attrs)
if err != nil {
return ColumnItem{}, err
}
return ColumnItem{
ID: column.ID,
Key: column.Key,
Attributes: attrs,
}, nil
}

65
client/response_test.go
View file

@ -28,55 +28,25 @@ import (
)
func TestNewRowResultFromInternal(t *testing.T) {
targetAttrs := map[string]interface{}{
"name": "some string",
"age": int64(95),
"registered": true,
"height": 1.83,
}
targetColumns := []uint64{5, 10}
attrs := []*pb.Attr{
{Key: "name", StringValue: "some string", Type: 1},
{Key: "age", IntValue: 95, Type: 2},
{Key: "registered", BoolValue: true, Type: 3},
{Key: "height", FloatValue: 1.83, Type: 4},
}
row := &pb.Row{
Attrs: attrs,
Columns: []uint64{5, 10},
}
result, err := newRowResultFromInternal(row)
if err != nil {
t.Fatalf("Failed with error: %s", err)
}
// assertMapEquals(t, targetAttrs, result.Attributes)
if !reflect.DeepEqual(targetAttrs, result.Attributes) {
t.Fatal()
}
if !reflect.DeepEqual(targetColumns, result.Columns) {
t.Fatal()
}
}
func TestNewQueryResponseFromInternal(t *testing.T) {
targetAttrs := map[string]interface{}{
"name": "some string",
"age": int64(95),
"registered": true,
"height": 1.83,
}
targetColumns := []uint64{5, 10}
targetCountItems := []CountResultItem{
{ID: 10, Count: 100},
}
attrs := []*pb.Attr{
{Key: "name", StringValue: "some string", Type: 1},
{Key: "age", IntValue: 95, Type: 2},
{Key: "registered", BoolValue: true, Type: 3},
{Key: "height", FloatValue: 1.83, Type: 4},
}
row := &pb.Row{
Attrs: attrs,
Columns: []uint64{5, 10},
}
pairs := []*pb.Pair{
@ -107,9 +77,6 @@ func TestNewQueryResponseFromInternal(t *testing.T) {
if results[0] != qr.Result() {
t.Fatalf("Result() should return the first result")
}
if !reflect.DeepEqual(targetAttrs, results[0].Row().Attributes) {
t.Fatalf("The row result should contain the attributes")
}
if !reflect.DeepEqual(targetColumns, results[0].Row().Columns) {
t.Fatalf("The row result should contain the columns")
}
@ -137,29 +104,6 @@ func TestNewQueryResponseWithErrorFromInternal(t *testing.T) {
}
}
func TestNewQueryResponseFromInternalFailure(t *testing.T) {
attrs := []*pb.Attr{
{Key: "name", StringValue: "some string", Type: 99},
}
row := &pb.Row{
Attrs: attrs,
}
response := &pb.QueryResponse{
Results: []*pb.QueryResult{{Type: QueryResultTypeRow, Row: row}},
}
qr, err := newQueryResponseFromInternal(response)
if qr != nil && err == nil {
t.Fatalf("Should have failed")
}
response = &pb.QueryResponse{
ColumnAttrSets: []*pb.ColumnAttrSet{{ID: 1, Attrs: attrs}},
}
qr, err = newQueryResponseFromInternal(response)
if qr != nil && err == nil {
t.Fatalf("Should have failed")
}
}
func TestCountResultItemToString(t *testing.T) {
tests := []struct {
item *CountResultItem
@ -182,14 +126,7 @@ func TestCountResultItemToString(t *testing.T) {
}
func TestMarshalResults(t *testing.T) {
attrs := []*pb.Attr{
{Key: "name", StringValue: "some string", Type: 1},
{Key: "age", IntValue: 95, Type: 2},
{Key: "registered", BoolValue: true, Type: 3},
{Key: "height", FloatValue: 1.83, Type: 4},
}
row := &pb.Row{
Attrs: attrs,
Columns: []uint64{5, 10},
}
pairs := []*pb.Pair{
@ -212,7 +149,7 @@ func TestMarshalResults(t *testing.T) {
resultJSONStrings[i] = string(b)
}
targetJSON := []string{
`{"attrs":{"age":95,"height":1.83,"name":"some string","registered":true},"columns":[5,10],"keys":[]}`,
`{"columns":[5,10],"keys":[]}`,
`[{"id":10,"count":100}]`,
}
for i := range targetJSON {

68
ctl/backup.go
View file

@ -198,18 +198,12 @@ func (cmd *BackupCommand) backupIndex(ctx context.Context, tw *tar.Writer, ii *p
if err := cmd.backupIndexTranslateData(ctx, tw, ii.Name); err != nil {
return err
}
if err := cmd.backupIndexAttrData(ctx, tw, ii.Name); err != nil {
return err
}
// Back up field translation & attribute data.
// Back up field translation data.
for _, fi := range ii.Fields {
if err := cmd.backupFieldTranslateData(ctx, tw, ii.Name, fi.Name); err != nil {
return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err)
}
if err := cmd.backupFieldAttrData(ctx, tw, ii.Name, fi.Name); err != nil {
return fmt.Errorf("cannot backup field attr data for field %q on index %q: %w", fi.Name, ii.Name, err)
}
}
return nil
@ -294,36 +288,6 @@ func (cmd *BackupCommand) backupIndexPartitionTranslateData(ctx context.Context,
return nil
}
func (cmd *BackupCommand) backupIndexAttrData(ctx context.Context, tw *tar.Writer, name string) error {
logger := cmd.Logger()
logger.Printf("backing up index attr data: %s", name)
rc, err := cmd.client.IndexAttrDataReader(ctx, name)
if err != nil {
return fmt.Errorf("fetching index attr data reader: %w", err)
}
defer rc.Close()
// Read to buffer to determine size.
var buf bytes.Buffer
if _, err := buf.ReadFrom(rc); err != nil {
return fmt.Errorf("copying index attr data to memory: %w", err)
}
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("indexes", name, "attributes"),
Mode: 0666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
return err
} else if _, err := io.Copy(tw, &buf); err != nil {
return fmt.Errorf("copying index attr data to archive: %w", err)
}
return nil
}
func (cmd *BackupCommand) backupFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error {
logger := cmd.Logger()
logger.Printf("backing up field translation data: %s/%s", indexName, fieldName)
@ -356,36 +320,6 @@ func (cmd *BackupCommand) backupFieldTranslateData(ctx context.Context, tw *tar.
return nil
}
func (cmd *BackupCommand) backupFieldAttrData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error {
logger := cmd.Logger()
logger.Printf("backing up field attr data: %s/%s", indexName, fieldName)
rc, err := cmd.client.FieldAttrDataReader(ctx, indexName, fieldName)
if err != nil {
return fmt.Errorf("fetching field attr data reader: %w", err)
}
defer rc.Close()
// Read to buffer to determine size.
var buf bytes.Buffer
if _, err := buf.ReadFrom(rc); err != nil {
return fmt.Errorf("copying field attr data to memory: %w", err)
}
// Build header & copy data to archive.
if err = tw.WriteHeader(&tar.Header{
Name: path.Join("indexes", indexName, "fields", fieldName, "attributes"),
Mode: 0666,
Size: int64(buf.Len()),
ModTime: time.Now(),
}); err != nil {
return err
} else if _, err := io.Copy(tw, &buf); err != nil {
return fmt.Errorf("copying field attr data to archive: %w", err)
}
return nil
}
func (cmd *BackupCommand) TLSHost() string { return cmd.Host }
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }

152
encoding/proto/proto.go
View file

@ -16,7 +16,6 @@ package proto
import (
"fmt"
"sort"
"time"
"github.com/gogo/protobuf/proto"
@ -227,14 +226,6 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
s.decodeImportRoaringRequest(msg, mt)
return nil
case *pilosa.ImportColumnAttrsRequest:
msg := &pb.ImportColumnAttrsRequest{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshaling ImportColumnAttrsRequest")
}
s.decodeImportColumnAttrsRequest(msg, mt)
return nil
case *pilosa.ImportResponse:
msg := &pb.ImportResponse{}
err := proto.Unmarshal(buf, msg)
@ -385,8 +376,6 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeImportValueRequest(mt)
case *pilosa.ImportRoaringRequest:
return s.encodeImportRoaringRequest(mt)
case *pilosa.ImportColumnAttrsRequest:
return s.encodeImportColumnAttrsRequest(mt)
case *pilosa.ImportResponse:
return s.encodeImportResponse(mt)
case *pilosa.BlockDataRequest:
@ -488,27 +477,13 @@ func (s Serializer) encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *
}
}
func (s Serializer) encodeImportColumnAttrsRequest(m *pilosa.ImportColumnAttrsRequest) *pb.ImportColumnAttrsRequest {
return &pb.ImportColumnAttrsRequest{
Index: m.Index,
IndexCreatedAt: m.IndexCreatedAt,
Shard: m.Shard,
AttrKey: m.AttrKey,
AttrVals: m.AttrVals,
ColumnIDs: m.ColumnIDs,
}
}
func (s Serializer) encodeQueryRequest(m *pilosa.QueryRequest) *pb.QueryRequest {
r := &pb.QueryRequest{
Query: m.Query,
Shards: m.Shards,
ColumnAttrs: m.ColumnAttrs,
Remote: m.Remote,
ExcludeRowAttrs: m.ExcludeRowAttrs,
ExcludeColumns: m.ExcludeColumns,
PreTranslated: m.PreTranslated,
EmbeddedData: make([]*pb.Row, len(m.EmbeddedData)),
Query: m.Query,
Shards: m.Shards,
Remote: m.Remote,
PreTranslated: m.PreTranslated,
EmbeddedData: make([]*pb.Row, len(m.EmbeddedData)),
}
for i := range m.EmbeddedData {
r.EmbeddedData[i] = s.encodeRow(m.EmbeddedData[i])
@ -518,8 +493,7 @@ func (s Serializer) encodeQueryRequest(m *pilosa.QueryRequest) *pb.QueryRequest
func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *pb.QueryResponse {
resp := &pb.QueryResponse{
Results: make([]*pb.QueryResult, len(m.Results)),
ColumnAttrSets: s.encodeColumnAttrSets(m.ColumnAttrSets),
Results: make([]*pb.QueryResult, len(m.Results)),
}
for i := range m.Results {
@ -1209,10 +1183,7 @@ func (s Serializer) decodeLoadSchemaMessage(pb *pb.LoadSchemaMessage, m *pilosa.
func (s Serializer) decodeQueryRequest(pb *pb.QueryRequest, m *pilosa.QueryRequest) {
m.Query = pb.Query
m.Shards = pb.Shards
m.ColumnAttrs = pb.ColumnAttrs
m.Remote = pb.Remote
m.ExcludeRowAttrs = pb.ExcludeRowAttrs
m.ExcludeColumns = pb.ExcludeColumns
m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData))
m.PreTranslated = pb.PreTranslated
for i := range pb.EmbeddedData {
@ -1262,15 +1233,6 @@ func (s Serializer) decodeImportRoaringRequest(pb *pb.ImportRoaringRequest, m *p
m.UpdateExistence = pb.UpdateExistence
}
func (s Serializer) decodeImportColumnAttrsRequest(pb *pb.ImportColumnAttrsRequest, m *pilosa.ImportColumnAttrsRequest) {
m.Index = pb.Index
m.IndexCreatedAt = pb.IndexCreatedAt
m.Shard = pb.Shard
m.AttrKey = pb.AttrKey
m.AttrVals = pb.AttrVals
m.ColumnIDs = pb.ColumnIDs
}
func (s Serializer) decodeImportResponse(pb *pb.ImportResponse, m *pilosa.ImportResponse) {
m.Err = pb.Err
}
@ -1289,8 +1251,6 @@ func (s Serializer) decodeBlockDataResponse(pb *pb.BlockDataResponse, m *pilosa.
}
func (s Serializer) decodeQueryResponse(pb *pb.QueryResponse, m *pilosa.QueryResponse) {
m.ColumnAttrSets = make([]*pilosa.ColumnAttrSet, len(pb.ColumnAttrSets))
s.decodeColumnAttrSets(pb.ColumnAttrSets, m.ColumnAttrSets)
if pb.Err == "" {
m.Err = nil
} else {
@ -1300,19 +1260,6 @@ func (s Serializer) decodeQueryResponse(pb *pb.QueryResponse, m *pilosa.QueryRes
s.decodeQueryResults(pb.Results, m.Results)
}
func (s Serializer) decodeColumnAttrSets(pb []*pb.ColumnAttrSet, m []*pilosa.ColumnAttrSet) {
for i := range pb {
m[i] = &pilosa.ColumnAttrSet{}
s.decodeColumnAttrSet(pb[i], m[i])
}
}
func (s Serializer) decodeColumnAttrSet(pb *pb.ColumnAttrSet, m *pilosa.ColumnAttrSet) {
m.ID = pb.ID
m.Key = pb.Key
m.Attrs = s.decodeAttrs(pb.Attrs)
}
func (s Serializer) decodeQueryResults(pb []*pb.QueryResult, m []interface{}) {
for i := range pb {
m[i] = s.decodeQueryResult(pb[i])
@ -1457,7 +1404,6 @@ func (s Serializer) decodeRow(pr *pb.Row) *pilosa.Row {
r.SetBit(v)
}
}
r.Attrs = s.decodeAttrs(pr.Attrs)
r.Keys = pr.Keys
r.Index = pr.Index
r.Field = pr.Field
@ -1476,37 +1422,6 @@ func (s Serializer) decodeSignedRow(pr *pb.SignedRow) pilosa.SignedRow {
return r
}
func (s Serializer) decodeAttrs(pb []*pb.Attr) map[string]interface{} {
m := make(map[string]interface{}, len(pb))
for i := range pb {
key, value := s.decodeAttr(pb[i])
m[key] = value
}
return m
}
const (
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
func (s Serializer) decodeAttr(attr *pb.Attr) (key string, value interface{}) {
switch attr.Type {
case attrTypeString:
return attr.Key, attr.StringValue
case attrTypeInt:
return attr.Key, attr.IntValue
case attrTypeBool:
return attr.Key, attr.BoolValue
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil
}
}
func (s Serializer) decodeExtractedIDMatrix(m *pb.ExtractedIDMatrix) pilosa.ExtractedIDMatrix {
cols := make([]pilosa.ExtractedIDColumn, len(m.Columns))
for i, c := range m.Columns {
@ -1682,22 +1597,6 @@ func (s Serializer) decodeDecimalStruct(pb *pb.Decimal) *pql.Decimal {
}
}
func (s Serializer) encodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*pb.ColumnAttrSet {
other := make([]*pb.ColumnAttrSet, len(a))
for i := range a {
other[i] = s.encodeColumnAttrSet(a[i])
}
return other
}
func (s Serializer) encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *pb.ColumnAttrSet {
return &pb.ColumnAttrSet{
ID: set.ID,
Key: set.Key,
Attrs: s.encodeAttrs(set.Attrs),
}
}
func (s Serializer) encodeSignedRow(r pilosa.SignedRow) *pb.SignedRow {
ir := &pb.SignedRow{
Pos: s.encodeRow(r.Pos),
@ -1713,7 +1612,6 @@ func (s Serializer) encodeRow(r *pilosa.Row) *pb.Row {
ir := &pb.Row{
Keys: r.Keys,
Attrs: s.encodeAttrs(r.Attrs),
Index: r.Index,
Field: r.Field,
}
@ -1729,7 +1627,6 @@ func (s Serializer) encodeRowIdentifiers(r pilosa.RowIdentifiers) *pb.RowIdentif
return &pb.RowIdentifiers{
Rows: r.Rows,
Keys: r.Keys,
//Attrs: s.encodeAttrs(r.Attrs),
}
}
@ -1911,43 +1808,6 @@ func (s Serializer) encodeDecimal(p *pql.Decimal) *pb.Decimal {
}
}
func (s Serializer) encodeAttrs(m map[string]interface{}) []*pb.Attr {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
a := make([]*pb.Attr, len(keys))
for i := range keys {
a[i] = s.encodeAttr(keys[i], m[keys[i]])
}
return a
}
// s.encodeAttr converts a key/value pair into an Attr pb.representation.
func (s Serializer) encodeAttr(key string, value interface{}) *pb.Attr {
pb := &pb.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
}
func (s Serializer) encodeResizeNodeMessage(m *pilosa.ResizeNodeMessage) *pb.ResizeNodeMessage {
return &pb.ResizeNodeMessage{
NodeID: m.NodeID,

414
executor.go
View file

@ -221,44 +221,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
}
resp.Results = results
// Fill column attributes if requested.
if opt.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Row)
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs)
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if idx.Keys() {
idSet := make(map[uint64]struct{})
for _, col := range columnAttrSets {
idSet[col.ID] = struct{}{}
}
idMap, err := e.Cluster.translateIndexIDSet(ctx, index, idSet)
if err != nil {
return resp, errors.Wrap(err, "translating id set")
}
for _, col := range columnAttrSets {
col.Key, col.ID = idMap[col.ID], 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
// Translate response objects from ids to keys, if necessary.
// No need to translate a remote call.
if !opt.Remote {
@ -293,10 +255,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
// to avoid anything coming from the mmap-ed Tx storage.
func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) {
out = QueryResponse{
// not transactional, from attribute storage so no need to clone these:
ColumnAttrSets: resp.ColumnAttrSets, // []*ColumnAttrSet
Err: resp.Err, // error
Profile: resp.Profile, // *tracing.Profile
Err: resp.Err, // error
Profile: resp.Profile, // *tracing.Profile
}
// Results can contain *roaring.Bitmap, so need to copy from Tx mmap-ed memory.
for _, v := range resp.Results {
@ -355,29 +315,6 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) {
return
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
ax := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if len(attrs) == 0 {
continue
}
// Append column with attributes.
ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return ax, nil
}
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values (e.g. Distinct, UnionRows, ConstRow...).
func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
@ -538,11 +475,6 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
}
}
// Optimize handling for bulk attribute insertion.
if hasOnlySetRowAttrs(q.Calls) {
return e.executeBulkSetRowAttrs(ctx, qcx, index, q.Calls, opt, colTranslations, rowTranslations)
}
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for i, call := range q.Calls {
@ -784,12 +716,6 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
statFn()
res, err := e.executeSet(ctx, qcx, index, c, opt)
return res, errors.Wrapf(err, "executeSet %v", shardSlice(shards))
case "SetRowAttrs":
statFn()
return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs")
case "SetColumnAttrs":
statFn()
return nil, errors.Wrap(e.executeSetColumnAttrs(ctx, qcx, index, c, opt), "executeSetColumnAttrs")
case "TopK":
statFn()
res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
@ -876,27 +802,6 @@ func (e *executor) executeOptionsCall(ctx context.Context, qcx *Qcx, index strin
optCopy := &execOptions{}
*optCopy = *opt
if arg, ok := c.Args["columnAttrs"]; ok {
if value, ok := arg.(bool); ok {
opt.ColumnAttrs = value
} else {
return nil, errors.New("Query(): columnAttrs must be a bool")
}
}
if arg, ok := c.Args["excludeRowAttrs"]; ok {
if value, ok := arg.(bool); ok {
optCopy.ExcludeRowAttrs = value
} else {
return nil, errors.New("Query(): excludeRowAttrs must be a bool")
}
}
if arg, ok := c.Args["excludeColumns"]; ok {
if value, ok := arg.(bool); ok {
optCopy.ExcludeColumns = value
} else {
return nil, errors.New("Query(): excludeColumns must be a bool")
}
}
if arg, ok := c.Args["shards"]; ok {
if optShards, ok := arg.([]interface{}); ok {
shards = []uint64{}
@ -1560,46 +1465,7 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string
return nil, errors.Wrap(err, "map reduce")
}
// Attach attributes for non-BSI Row() calls.
// If the column label is used then return column attributes.
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Row" && !c.HasConditionArg() {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
if idx != nil {
if columnID, ok, err := c.UintArg("_" + columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, errors.Wrap(err, "getting column attrs")
}
row.Attrs = attrs
} else if err != nil {
return nil, err
} else {
// field, _ := c.Args["field"].(string)
fieldName, _ := c.FieldArg()
if fr := idx.Field(fieldName); fr != nil {
rowID, _, err := c.UintArg(fieldName)
if err != nil {
return nil, errors.Wrap(err, "getting row")
}
attrs, err := fr.RowAttrStore().Attrs(rowID)
if err != nil {
return nil, errors.Wrap(err, "getting row attrs")
}
row.Attrs = attrs
}
}
}
}
}
if opt.ExcludeColumns {
row.segments = []rowSegment{}
}
return row, nil
}
@ -2630,7 +2496,6 @@ func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string,
return nil, fmt.Errorf("cannot compute TopN() on integer, decimal, or timestamp field: %q", fieldName)
}
attrName, _ := c.Args["attrName"].(string)
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopNShard: %v", err)
@ -2639,7 +2504,6 @@ func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string,
if err != nil {
return nil, fmt.Errorf("executeTopNShard: %v", err)
}
attrValues, _ := c.Args["attrValues"].([]interface{})
tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold")
if err != nil {
return nil, fmt.Errorf("executeTopNShard: %v", err)
@ -2688,8 +2552,6 @@ func (e *executor) executeTopNShard(ctx context.Context, qcx *Qcx, index string,
N: int(n),
Src: src,
RowIDs: rowIDs,
FilterName: attrName,
FilterValues: attrValues,
MinThreshold: minThreshold,
TanimotoThreshold: tanimotoThreshold,
})
@ -5650,229 +5512,6 @@ func (e *executor) executeClearValueField(ctx context.Context, qcx *Qcx, index s
return ret, nil
}
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *executor) executeSetRowAttrs(ctx context.Context, qcx *Qcx, index string, c *pql.Call, opt *execOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetRowAttrs")
defer span.Finish()
fieldName, ok := c.Args["_field"].(string)
if !ok {
return errors.New("SetRowAttrs() field required")
}
// Retrieve field.
field := e.Holder.Field(index, fieldName)
if field == nil {
return newNotFoundError(ErrFieldNotFound, fieldName)
}
// Parse labels.
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return fmt.Errorf("reading SetRowAttrs() row: %v", err)
} else if !ok {
return fmt.Errorf("SetRowAttrs() row field '%v' required", rowLabel)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgsDecimalToFloat(c.Args)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Set attributes.
if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
return nil
}
// Execute on remote nodes in parallel.
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
// Return first error.
for range nodes {
if err := <-resp; err != nil {
return err
}
}
return nil
}
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
func (e *executor) executeBulkSetRowAttrs(ctx context.Context, qcx *Qcx, index string, calls []*pql.Call, opt *execOptions, colTranslations map[string]map[string]uint64, rowTranslations map[string]map[string]map[string]uint64) ([]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBulkSetRowAttrs")
defer span.Finish()
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for i, c := range calls {
if i%10 == 0 {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
}
// Apply call translation.
if !opt.Remote {
translated, err := e.translateCall(c, index, colTranslations, rowTranslations)
if err != nil {
return nil, errors.Wrap(err, "translating call")
}
if translated == nil {
continue
}
c = translated
}
field, ok := c.Args["_field"].(string)
if !ok {
return nil, errors.New("SetRowAttrs() field required")
}
// Retrieve field.
f := e.Holder.Field(index, field)
if f == nil {
return nil, newNotFoundError(ErrFieldNotFound, field)
}
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return nil, errors.Wrap(err, "reading SetRowAttrs() row")
} else if !ok {
return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgsDecimalToFloat(c.Args)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Create field group, if not exists.
fieldMap := m[field]
if fieldMap == nil {
fieldMap = make(map[uint64]map[string]interface{})
m[field] = fieldMap
}
// Set or merge attributes.
attr := fieldMap[rowID]
if attr == nil {
fieldMap[rowID] = cloneAttrs(attrs)
} else {
for k, v := range attrs {
attr[k] = v
}
}
}
// Bulk insert attributes by field.
for name, fieldMap := range m {
// Retrieve field.
field := e.Holder.Field(index, name)
if field == nil {
return nil, newNotFoundError(ErrFieldNotFound, name)
}
// Set attributes.
if err := field.RowAttrStore().SetBulkAttrs(fieldMap); err != nil {
return nil, err
}
}
if !opt.Remote {
tags := []string{"index:" + index, "bulk:true"}
e.Holder.Stats.CountWithCustomTags(MetricSetRowAttrs, int64(len(m)), 1.0, tags)
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
return make([]interface{}, len(calls)), nil
}
// Execute on remote nodes in parallel.
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: calls}, nil, nil)
resp <- err
}(node)
}
// Return first error.
for range nodes {
if err := <-resp; err != nil {
return nil, err
}
}
// Return a set of nil responses to match the non-optimized return.
return make([]interface{}, len(calls)), nil
}
// executeSetColumnAttrs executes a SetColumnAttrs() call.
func (e *executor) executeSetColumnAttrs(ctx context.Context, qcx *Qcx, index string, c *pql.Call, opt *execOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSetColumnAttrs")
defer span.Finish()
// Retrieve index.
idx := e.Holder.Index(index)
if idx == nil {
return newNotFoundError(ErrIndexNotFound, index)
}
col, okCol, errCol := c.UintArg("_" + columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgsDecimalToFloat(c.Args)
delete(attrs, "_"+columnLabel)
delete(attrs, "field")
// Set attributes.
if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil {
return err
}
// Do not forward call if this is already being forwarded.
if opt.Remote {
return nil
}
// Execute on remote nodes in parallel.
nodes := topology.Nodes(e.Cluster.noder.Nodes()).FilterID(e.Node.ID)
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *topology.Node) {
_, err := e.remoteExec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, nil)
resp <- err
}(node)
}
// Return first error.
for range nodes {
if err := <-resp; err != nil {
return err
}
}
return nil
}
// remoteExec executes a PQL query remotely for a set of shards on a node.
func (e *executor) remoteExec(ctx context.Context, node *topology.Node, index string, q *pql.Query, shards []uint64, embed []*Row) (results []interface{}, err error) { // nolint: interfacer
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeExec")
@ -6367,7 +6006,7 @@ func (e *executor) collectCallKeys(dst *keyCollector, c *pql.Call, index string)
// Handle _col.
if col, ok := c.Args["_col"].(string); ok {
switch c.Name {
case "Set", "SetColumnAttrs":
case "Set":
dst.CreateColumns(index, col)
default:
dst.FindColumns(index, col)
@ -6385,12 +6024,7 @@ func (e *executor) collectCallKeys(dst *keyCollector, c *pql.Call, index string)
return errors.Wrap(ErrFieldNotFound, "finding field for _row argument")
}
switch c.Name {
case "SetRowAttrs":
dst.CreateRows(index, field, row)
default:
dst.FindRows(index, field, row)
}
dst.FindRows(index, field, row)
}
// Handle queries that need a "column" argument.
@ -6700,7 +6334,7 @@ func (e *executor) translateCall(c *pql.Call, index string, columnKeys map[strin
c.Args["_col"] = id
} else {
switch c.Name {
case "Set", "SetColumnAttrs":
case "Set":
return nil, errors.Wrapf(ErrTranslatingKeyNotFound, "destination key not found %q in index %q", col, index)
default:
return e.callZero(c), nil
@ -6736,12 +6370,7 @@ func (e *executor) translateCall(c *pql.Call, index string, columnKeys map[strin
if translation, ok := indexRows[field][row]; ok {
c.Args["_row"] = translation
} else {
switch c.Name {
case "SetRowAttrs":
return nil, errors.Errorf("row key missing in %q", c.String())
default:
return e.callZero(c), nil
}
return e.callZero(c), nil
}
}
}
@ -7023,7 +6652,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
switch strategy {
case byCurrentIndex:
other := &Row{Attrs: result.Attrs}
other := &Row{}
for _, segment := range result.Segments() {
for _, col := range segment.Columns() {
other.Keys = append(other.Keys, idSet[col])
@ -7081,7 +6710,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
if rslt == nil {
return &SignedRow{Pos: &Row{}}, nil
}
other := &Row{Attrs: rslt.Attrs}
other := &Row{}
for _, segment := range rslt.Segments() {
keys, err := e.Cluster.translateIndexIDs(context.Background(), field.ForeignIndex(), segment.Columns())
if err != nil {
@ -7514,27 +7143,10 @@ type mapResponse struct {
// execOptions represents an execution context for a single Execute() call.
type execOptions struct {
Remote bool
Profile bool
ExcludeRowAttrs bool
ExcludeColumns bool
ColumnAttrs bool
PreTranslated bool
EmbeddedData []*Row
}
// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
func hasOnlySetRowAttrs(calls []*pql.Call) bool {
if len(calls) == 0 {
return false
}
for _, call := range calls {
if call.Name != "SetRowAttrs" {
return false
}
}
return true
Remote bool
Profile bool
PreTranslated bool
EmbeddedData []*Row
}
func needsShards(calls []*pql.Call) bool {
@ -7543,7 +7155,7 @@ func needsShards(calls []*pql.Call) bool {
}
for _, call := range calls {
switch call.Name {
case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs":
case "Clear", "Set":
continue
case "Count", "TopN", "Rows":
return true

306
executor_test.go
View file

@ -82,33 +82,11 @@ func TestExecutor(t *testing.T) {
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20) +
`SetRowAttrs(f, 10, foo="bar", baz=123)` +
`Set(1000, f=100)` +
`SetColumnAttrs(1000, foo="bar", baz=123)`
readQueries := []string{
`Row(f=10)`,
`Options(Row(f=10), excludeColumns=true)`,
`Options(Row(f=10), excludeRowAttrs=true)`,
}
`Set(1000, f=100)`
readQueries := []string{`Row(f=10)`}
responses := runCallTest(c, t, writeQuery, readQueries, nil)
if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit column attributes.
if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := responses[1].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit row attributes.
if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := responses[2].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
@ -147,7 +125,7 @@ func TestExecutor(t *testing.T) {
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldKeys())
if diff := cmp.Diff(responses[0].Results, []interface{}{
&pilosa.Row{Keys: []string{"bat", "foo"}, Attrs: map[string]interface{}{}},
&pilosa.Row{Keys: []string{"bat", "foo"}},
}, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" {
t.Fatal(diff)
}
@ -844,92 +822,6 @@ func TestExecutor(t *testing.T) {
})
t.Run("Options", func(t *testing.T) {
t.Run("excludeRowAttrs", func(t *testing.T) {
writeQuery := `
Set(100, f=10)
SetRowAttrs(f, 10, foo="bar")`
readQueries := []string{`Options(Row(f=10), excludeRowAttrs=true)`}
responses := runCallTest(c, t, writeQuery, readQueries, nil)
if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
t.Run("excludeColumns", func(t *testing.T) {
writeQuery := `
Set(100, f=10)
SetRowAttrs(f, 10, foo="bar")`
readQueries := []string{`Options(Row(f=10), excludeColumns=true)`}
responses := runCallTest(c, t, writeQuery, readQueries, nil)
if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
t.Run("columnAttrs", func(t *testing.T) {
writeQuery := `
Set(0, f=10)
SetColumnAttrs(0, foo="baz")
Set(100, f=10)
SetColumnAttrs(100, foo="bar")`
readQueries := []string{`Options(Row(f=10), columnAttrs=true)`}
responses := runCallTest(c, t, writeQuery, readQueries, nil)
targetColAttrSets := []*pilosa.ColumnAttrSet{
{ID: 0, Attrs: map[string]interface{}{"foo": "baz"}},
{ID: 100, Attrs: map[string]interface{}{"foo": "bar"}},
}
targetJSON := `[{"id":0,"attrs":{"foo":"baz"}},{"id":100,"attrs":{"foo":"bar"}}]`
if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{0, 100}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
} else {
// Ensure the JSON is marshaled correctly.
jres, err := json.Marshal(attrs)
if err != nil {
t.Fatal(err)
} else if string(jres) != targetJSON {
t.Fatalf("json marshal expected: %s, but got: %s", targetJSON, jres)
}
}
})
t.Run("columnAttrsWithKeys", func(t *testing.T) {
writeQuery := `
Set("one-hundred", f="ten")
SetColumnAttrs("one-hundred", foo="bar")`
readQueries := []string{`Options(Row(f="ten"), columnAttrs=true)`}
responses := runCallTest(c, t, writeQuery, readQueries,
&pilosa.IndexOptions{Keys: true},
pilosa.OptFieldKeys())
targetColAttrSets := []*pilosa.ColumnAttrSet{
{Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}},
}
targetJSON := `[{"key":"one-hundred","attrs":{"foo":"bar"}}]`
if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) {
t.Fatalf("unexpected keys: %+v", keys)
} else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
} else {
// Ensure the JSON is marshaled correctly.
jres, err := json.Marshal(attrs)
if err != nil {
t.Fatal(err)
} else if string(jres) != targetJSON {
t.Fatalf("json marshal expected: %s, but got: %s", targetJSON, jres)
}
}
})
t.Run("shards", func(t *testing.T) {
writeQuery := fmt.Sprintf(`
Set(100, f=10)
@ -941,26 +833,6 @@ func TestExecutor(t *testing.T) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("multipleOpt", func(t *testing.T) {
writeQuery := `
Set(100, f=10)
SetRowAttrs(f, 10, foo="bar")`
readQueries := []string{
`Options(Row(f=10), excludeColumns=true)
Options(Row(f=10), excludeRowAttrs=true)`,
}
responses := runCallTest(c, t, writeQuery, readQueries, nil)
if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
} else if bits := responses[0].Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := responses[0].Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
})
t.Run("Not", func(t *testing.T) {
@ -1816,66 +1688,6 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Ensure a SetRowAttrs() query can be executed.
func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := c.GetHolder(0)
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFieldIfNotExists("kf", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
t.Run("rowID", func(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other rows and fields to test isolation.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil {
t.Fatal(err)
}
f := hldr.Field("i", "f")
if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
t.Fatalf("unexpected row attr: %#v", m)
}
})
t.Run("rowKey", func(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other rows and fields to test isolation.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row200", YYY=1)`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", baz=123, bat=true)`}); err != nil {
t.Fatal(err)
}
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil {
t.Fatal(err)
} else if attrs := result.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
t.Fatalf("unexpected attrs: %+v", attrs)
}
})
}
func TestExecutor_Execute_TopK_Set(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
@ -2282,56 +2094,6 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
}
}
//Ensure TopN handles Attribute filters
func TestExecutor_Execute_TopN_Attr(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := c.GetHolder(0)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, ShardWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 10, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
//Ensure TopN handles Attribute filters with source row
func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := c.GetHolder(0)
hldr.SetBit("i", "f", 0, 0)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", 10, ShardWidth)
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{
Pairs: []pilosa.Pair{
{ID: 10, Count: 1},
},
Field: "f",
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure Min() and Max() queries can be executed.
func TestExecutor_Execute_MinMax(t *testing.T) {
t.Run("WithOffset", func(t *testing.T) {
@ -3695,17 +3457,6 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
}
})
t.Run("remote setrowattrs", func(t *testing.T) {
if _, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i",
Query: `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)`,
}); err != nil {
t.Fatalf("setrowattrs querying: %v", err)
} else if attrst, err := hldr0.RowAttrStore("i", "f").Attrs(10); err != nil || !attrst["bat"].(bool) || attrst["baz"].(int64) != 123 {
t.Fatalf("wrong attrs: %v", attrst)
}
})
t.Run("remote groupBy", func(t *testing.T) {
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i",
@ -3927,57 +3678,6 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
}
}
// Ensure SetColumnAttrs doesn't save `field` as an attribute
func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := c.GetHolder(0)
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("creating field: %v", err)
}
targetAttrs := map[string]interface{}{
"foo": "bar",
}
// SetColumnAttrs call should exclude the field attribute
_, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"})
if err != nil {
t.Fatal(err)
}
_, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"})
if err != nil {
t.Fatal(err)
}
attrs, err := index.ColumnAttrStore().Attrs(10)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(attrs, targetAttrs) {
t.Fatalf("%#v != %#v", targetAttrs, attrs)
}
// SetColumnAttrs call should not break if field is not specified
_, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"})
if err != nil {
t.Fatal(err)
}
_, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"})
if err != nil {
t.Fatal(err)
}
attrs, err = index.ColumnAttrStore().Attrs(20)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(attrs, targetAttrs) {
t.Fatalf("%#v != %#v", targetAttrs, attrs)
}
}
func TestExecutor_Time_Clear_Quantums(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()

20
field.go
View file

@ -95,9 +95,6 @@ type Field struct {
viewMap map[string]*view
// Row attribute storage and cache
rowAttrStore AttrStore
broadcaster broadcaster
Stats stats.StatsClient
schemator disco.Schemator
@ -377,8 +374,6 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel
viewMap: make(map[string]*view),
rowAttrStore: nopStore,
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
schemator: disco.NopSchemator,
@ -423,9 +418,6 @@ func (f *Field) TranslateStore() TranslateStore {
return f.translateStore
}
// RowAttrStore returns the attribute storage.
func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore }
// AvailableShards returns a bitmap of shards that contain data.
func (f *Field) AvailableShards(localOnly bool) *roaring.Bitmap {
f.mu.RLock()
@ -568,11 +560,6 @@ func (f *Field) Open() error {
return errors.Wrap(err, "opening views")
}
f.holder.Logger.Debugf("open row attribute store for index/field: %s/%s", f.index, f.name)
if err := f.rowAttrStore.Open(); err != nil {
return errors.Wrap(err, "opening attrstore")
}
// Apply the field-specific translateStore.
if err := f.applyTranslateStore(); err != nil {
return errors.Wrap(err, "applying translate store")
@ -753,7 +740,6 @@ func (f *Field) openViews() error {
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
view.rowAttrStore = f.rowAttrStore
f.holder.Logger.Debugf("add index/field/view to field.viewMap: %s/%s/%s", f.index, f.name, view.name)
f.viewMap[view.name] = view
}
@ -865,10 +851,6 @@ func (f *Field) Close() error {
f.wg.Wait()
f.availableShardChan = nil
}
// Close the attribute store.
if f.rowAttrStore != nil {
_ = f.rowAttrStore.Close()
}
// Close field translation store.
if f.translateStore != nil {
@ -1053,7 +1035,6 @@ func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool,
if err := view.openEmpty(); err != nil {
return nil, false, errors.Wrap(err, "opening view")
}
view.rowAttrStore = f.rowAttrStore
f.viewMap[view.name] = view
return view, true, nil
@ -1062,7 +1043,6 @@ func (f *Field) createViewIfNotExistsBase(cvm *CreateViewMessage) (*view, bool,
func (f *Field) newView(path, name string) *view {
view := newView(f.holder, path, f.index, f.name, name, f.options)
view.idx = f.idx
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats
view.broadcaster = f.broadcaster
return view

31
fragment.go
View file

@ -192,10 +192,6 @@ type fragment struct {
// Logger used for out-of-band log entries.
Logger logger.Logger
// Row attribute storage.
// This is set by the parent field unless overridden for testing.
RowAttrStore AttrStore
// mutexVector is used for mutex field types. It's checked for an
// existing value (to clear) prior to setting a new value.
mutexVector vector
@ -1830,7 +1826,6 @@ func (f *fragment) forEachBit(tx Tx, fn func(rowID, columnID uint64) error) erro
// top returns the top rows from the fragment.
// If opt.Src is specified then only rows which intersect src are returned.
// If opt.FilterValues exist then the row attribute specified by field is matched.
func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) {
// Retrieve pairs. If no row ids specified then return from cache.
pairs, err := f.topBitmapPairs(tx, opt.RowIDs)
@ -1843,15 +1838,6 @@ func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) {
opt.N = 0
}
// Create a fast lookup of filter values.
var filters map[interface{}]struct{}
if opt.FilterName != "" && len(opt.FilterValues) > 0 {
filters = make(map[interface{}]struct{})
for _, v := range opt.FilterValues {
filters[v] = struct{}{}
}
}
// Use `tanimotoThreshold > 0` to indicate whether or not we are considering Tanimoto.
var tanimotoThreshold uint64
var minTanimoto, maxTanimoto float64
@ -1886,20 +1872,6 @@ func (f *fragment) top(tx Tx, opt topOptions) ([]Pair, error) {
}
}
// Apply filter, if set.
if filters != nil {
attr, err := f.RowAttrStore.Attrs(rowID)
if err != nil {
return nil, errors.Wrap(err, "getting attrs")
} else if attr == nil {
continue
} else if attrValue := attr[opt.FilterName]; attrValue == nil {
continue
} else if _, ok := filters[attrValue]; !ok {
continue
}
}
// The initial n pairs should simply be added to the results.
if opt.N == 0 || results.Len() < opt.N {
// Calculate count and append.
@ -2030,9 +2002,6 @@ type topOptions struct {
RowIDs []uint64
MinThreshold uint64
// Filter field name & values.
FilterName string
FilterValues []interface{}
TanimotoThreshold uint64
}

43
fragment_internal_test.go
View file

@ -1289,46 +1289,6 @@ func TestFragment_Top(t *testing.T) {
}
}
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean(t)
// Set bits on the rows 100, 101, & 102.
f.mustSetBits(tx, 100, 1, 3, 200)
f.mustSetBits(tx, 101, 1)
f.mustSetBits(tx, 102, 1, 2)
f.RecalculateCache()
// Assign attributes.
err := f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)})
if err != nil {
t.Fatalf("setAttrs: %v", err)
}
err = f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)})
if err != nil {
t.Fatalf("setAttrs: %v", err)
}
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
// Retrieve top rows.
if pairs, err := f.top(tx, topOptions{
N: 2,
FilterName: "x",
FilterValues: []interface{}{int64(10), int64(15), int64(20)},
}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
} else if pairs[0] != (Pair{ID: 102, Count: 2}) {
t.Fatalf("unexpected pair(0): %v", pairs[0])
} else if pairs[1] != (Pair{ID: 101, Count: 1}) {
t.Fatalf("unexpected pair(1): %v", pairs[1])
}
}
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
@ -3637,9 +3597,6 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
})
f.CacheType = cacheType
f.RowAttrStore = &memAttrStore{
store: make(map[uint64]map[string]interface{}),
}
if err := f.Open(); err != nil {
PanicOn(err)

33
handler.go
View file

@ -37,15 +37,6 @@ type QueryRequest struct {
// If empty, all shards are included.
Shards []uint64
// Return column attributes, if true.
ColumnAttrs bool
// Do not return row attributes, if true.
ExcludeRowAttrs bool
// Do not return columns, if true.
ExcludeColumns bool
// If true, indicates that query is part of a larger distributed query.
// If false, this request is on the originating node.
Remote bool
@ -70,9 +61,6 @@ type QueryResponse struct {
// ValCount, Pair, Pairs, bool, uint64.
Results []interface{}
// Set of column attribute objects matching IDs returned in Result.
ColumnAttrSets []*ColumnAttrSet
// Error during parsing or execution.
Err error
@ -89,13 +77,11 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
}
return json.Marshal(struct {
Results []interface{} `json:"results"`
ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
Profile *tracing.Profile `json:"profile,omitempty"`
Results []interface{} `json:"results"`
Profile *tracing.Profile `json:"profile,omitempty"`
}{
Results: resp.Results,
ColumnAttrSets: resp.ColumnAttrSets,
Profile: resp.Profile,
Results: resp.Results,
Profile: resp.Profile,
})
}
@ -204,17 +190,6 @@ func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreate
return nil
}
// ImportColumnAttrsRequest describes the import request structure
// for a ColumnAttr import.
type ImportColumnAttrsRequest struct {
AttrKey string
ColumnIDs []uint64
AttrVals []string
Shard int64
Index string
IndexCreatedAt int64
}
// ImportRequest describes the import request structure
// for an import. BSIs use the ImportValueRequest instead.
type ImportRequest struct {

119
holder.go
View file

@ -36,7 +36,6 @@ import (
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -57,12 +56,6 @@ const (
// FieldsDir is the default fields directory used by each index.
FieldsDir = "fields"
// ColumnAttrsFileName is the name of the file used for the column attributes store.
ColumnAttrsFileName = "column-attributes"
// RowAttrsFileName is the name of the file used for the row attributes store.
RowAttrsFileName = "row-attributes"
)
func init() {
@ -92,8 +85,6 @@ type Holder struct {
sharder disco.Sharder
serializer Serializer
NewAttrStore func(string) AttrStore
// Close management
wg sync.WaitGroup
closing chan struct{}
@ -234,7 +225,6 @@ type HolderConfig struct {
Sharder disco.Sharder
CacheFlushInterval time.Duration
StatsClient stats.StatsClient
NewAttrStore func(string) AttrStore
Logger logger.Logger
RowcacheOn bool
@ -258,7 +248,6 @@ func DefaultHolderConfig() *HolderConfig {
Sharder: disco.InMemSharder,
CacheFlushInterval: defaultCacheFlushInterval,
StatsClient: stats.NopStatsClient,
NewAttrStore: newNopAttrStore,
Logger: logger.NopLogger,
StorageConfig: storage.NewDefaultConfig(),
RBFConfig: rbfcfg.NewDefaultConfig(),
@ -287,7 +276,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
partitionN: cfg.PartitionN,
Stats: cfg.StatsClient,
NewAttrStore: cfg.NewAttrStore,
cacheFlushInterval: cfg.CacheFlushInterval,
OpenTranslateStore: cfg.OpenTranslateStore,
OpenTranslateReader: cfg.OpenTranslateReader,
@ -1318,8 +1306,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
index.broadcaster = h.broadcaster
index.serializer = h.serializer
index.Schemator = h.schemator
index.newAttrStore = h.NewAttrStore
index.columnAttrs = h.NewAttrStore(filepath.Join(index.path, ColumnAttrsFileName))
index.OpenTranslateStore = h.OpenTranslateStore
index.translationSyncer = h.translationSyncer
return index, nil
@ -1526,11 +1512,6 @@ func (s *holderSyncer) SyncHolder() error {
return nil
}
// Sync index column attributes.
if err := s.syncIndex(di.Name); err != nil {
return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err)
}
tf := time.Now()
for _, fi := range di.Fields {
// Verify syncer has not closed.
@ -1538,11 +1519,6 @@ func (s *holderSyncer) SyncHolder() error {
return nil
}
// Sync field row attributes.
if err := s.syncField(di.Name, fi.Name); err != nil {
return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err)
}
for _, vi := range fi.Views {
// Verify syncer has not closed.
if s.IsClosing() {
@ -1578,101 +1554,6 @@ func (s *holderSyncer) SyncHolder() error {
return nil
}
// syncIndex synchronizes index attributes with the rest of the cluster.
func (s *holderSyncer) syncIndex(index string) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncIndex")
defer span.Finish()
// Retrieve index reference.
idx := s.Holder.Index(index)
if idx == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
// Read block checksums.
blks, err := idx.ColumnAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags(MetricColumnAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range topology.Nodes(s.Cluster.noder.Nodes()).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.ColumnAttrDiff(ctx, &node.URI, index, blks)
if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags(MetricColumnAttrDiff, int64(len(m)), 1.0, []string{indexTag, node.ID})
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = idx.ColumnAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "recomputing blocks")
}
}
return nil
}
// syncField synchronizes field attributes with the rest of the cluster.
func (s *holderSyncer) syncField(index, name string) error {
span, ctx := tracing.StartSpanFromContext(context.Background(), "HolderSyncer.syncField")
defer span.Finish()
// Retrieve field reference.
f := s.Holder.Field(index, name)
if f == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
fieldTag := fmt.Sprintf("field:%s", name)
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags(MetricRowAttrStoreBlocks, int64(len(blks)), 1.0, []string{indexTag, fieldTag})
// Sync with every other host.
for _, node := range topology.Nodes(s.Cluster.noder.Nodes()).FilterID(s.Node.ID) {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := s.Cluster.InternalClient.RowAttrDiff(ctx, &node.URI, index, name, blks)
if errors.Cause(err) == ErrFieldNotFound {
continue // field not created remotely yet, skip
} else if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags(MetricRowAttrDiff, int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
return errors.Wrap(err, "setting attrs")
}
// Recompute blocks.
blks, err = f.RowAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "recomputing blocks")
}
}
return nil
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) error {
// Retrieve local field.

16
holder_test.go
View file

@ -53,22 +53,6 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
h := test.MustOpenHolder(t)
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), pilosa.ColumnAttrsFileName), 2); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=test, err=opening attrstore: opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
roaringOnlyTest(t)

180
http/client.go
View file

@ -784,57 +784,6 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index
return nil
}
// ImportColumnAttrs does bulk import of column attrs
func (c *InternalClient) ImportColumnAttrs(ctx context.Context, uri *pnet.URI, index string, req *pilosa.ImportColumnAttrsRequest) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportRoaring")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
}
if uri == nil {
uri = c.defaultURI
}
url := fmt.Sprintf("%s/index/%s/import-column-attrs", uri, index)
// Marshal data to protobuf.
data, err := c.serializer.Marshal(req)
if err != nil {
return errors.Wrap(err, "marshal import-column-attrs request")
}
// Generate HTTP request.
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
if err != nil {
return errors.Wrap(err, "creating request")
}
httpReq.Header.Set("Content-Type", "application/x-protobuf")
httpReq.Header.Set("Accept", "application/x-protobuf")
httpReq.Header.Set("X-Pilosa-Row", "roaring")
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}
defer resp.Body.Close()
dec := json.NewDecoder(resp.Body)
rbody := &pilosa.ImportResponse{}
err = dec.Decode(rbody)
// Decode can return EOF when no error occurred. helpful!
if err != nil && err != io.EOF {
return errors.Wrap(err, "decoding response body")
}
if rbody.Err != "" {
return errors.Wrap(errors.New(rbody.Err), "importing roaring")
}
return nil
}
// ExportCSV bulk exports data for a single shard from a host to CSV format.
func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ExportCSV")
@ -1126,89 +1075,6 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi
return rsp.RowIDs, rsp.ColumnIDs, nil
}
// ColumnAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pnet.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ColumnAttrDiff")
defer span.Finish()
if uri == nil {
uri = c.defaultURI
}
u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/attr/diff", index))
// Encode request.
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Decode response object.
var rsp postIndexAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pnet.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RowAttrDiff")
defer span.Finish()
if uri == nil {
uri = c.defaultURI
}
u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/field/%s/attr/diff", index, field))
// Encode request.
buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, errors.Wrap(err, "marshaling")
}
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/json")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, errors.Wrap(pilosa.ErrFieldNotFound, field)
}
return nil, err
}
defer resp.Body.Close()
// Decode response object.
var rsp postFieldAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, errors.Wrap(err, "decoding")
}
return rsp.Attrs, nil
}
// SendMessage posts a message synchronously.
func (c *InternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.SendMessage")
@ -2187,29 +2053,6 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str
return resp.Body, nil
}
// IndexAttrDataReader returns a reader that provides a snapshot of column attributes data.
func (c *InternalClient) IndexAttrDataReader(ctx context.Context, index string) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.IndexAttrDataReader")
defer span.Finish()
// Build request.
u := fmt.Sprintf("%s/internal/index/%s/attr/data", c.defaultURI.String(), url.QueryEscape(index))
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/octet-stream")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, err
}
return resp.Body, nil
}
// FieldTranslateDataReader returns a reader that provides a snapshot of
// translation data for a field.
func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
@ -2239,29 +2082,6 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi
return resp.Body, nil
}
// FieldAttrDataReader returns a reader that provides a snapshot of row attributes data.
func (c *InternalClient) FieldAttrDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.FieldAttrDataReader")
defer span.Finish()
// Build request.
u := fmt.Sprintf("%s/internal/index/%s/field/%s/attr/data", c.defaultURI.String(), url.QueryEscape(index), url.QueryEscape(field))
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
req.Header.Set("Accept", "application/octet-stream")
// Execute request.
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return nil, err
}
return resp.Body, nil
}
// Status function is just a public function for this particular implementation of InternalClient.
// It's not require by pilosa.InternalClient interface.
// The function returns pilosa cluster state as a string ("NORMAL", "DEGRADED", "DOWN", "RESIZING", ...)

75
http/client_test.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
gohttp "net/http"
"reflect"
"strconv"
"strings"
"testing"
"time"
@ -414,60 +413,6 @@ func TestClient_Import(t *testing.T) {
}
}
// Ensure client can bulk import column attrs.
func TestClient_ImportColumnAttrs(t *testing.T) {
cluster := test.MustNewCluster(t, 2)
for _, c := range cluster.Nodes {
c.Config.Cluster.ReplicaN = 2
}
err := cluster.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)
}
defer cluster.Close()
ctx := context.Background()
_, err = cluster.GetNode(0).API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
_, err = cluster.GetNode(0).API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = cluster.GetNode(0).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
if err != nil {
t.Fatalf("querying: %v", err)
}
attrKey := "k"
// Send import request.
host := cluster.GetNode(0).URL()
c := MustNewClient(host, http.GetHTTPClient(nil))
colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
if err := c.ImportColumnAttrs(ctx, &cluster.GetNode(1).API.Node().URI, "i", colAttrsReq); err != nil {
t.Fatal(err)
}
// Verify data.
pql := "Options(Row(f=0), columnAttrs=true)"
res, err := cluster.GetNode(1).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
if err != nil {
t.Fatal(err)
}
if len(res.ColumnAttrSets) != 5 {
t.Fatal("incorrect number of column attrs set")
}
for _, v := range res.ColumnAttrSets {
attrVal := attrFun(v.ID)
if attrVal != v.Attrs[attrKey] {
t.Fatal(err)
}
}
}
// Ensure client can bulk import data.
func TestClient_ImportRoaring(t *testing.T) {
cluster := test.MustRunCluster(t, 3,
@ -1408,26 +1353,6 @@ func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaring
}
}
func attrFun(id uint64) string {
return strconv.FormatInt(int64(id), 10)
}
func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pilosa.ImportColumnAttrsRequest {
colIDs := make([]uint64, 0, 5)
attrVals := make([]string, 0, 5)
for n := uint64(0); n < 5; n++ {
colIDs = append(colIDs, n)
attrVals = append(attrVals, attrFun(n))
}
return &pilosa.ImportColumnAttrsRequest{
Index: index,
Shard: shard,
AttrKey: attrKey,
ColumnIDs: colIDs,
AttrVals: attrVals,
}
}
// verify that serverInfo has Backend
func TestClient_ServerInfoHasBackend(t *testing.T) {
//srcs := []string{"roaring", "rbf", "lmdb"}

164
http/handler.go
View file

@ -237,7 +237,7 @@ func (h *Handler) populateValidators() {
h.validators["PostImport"] = queryValidationSpecRequired().Optional("clear", "ignoreKeyCheck")
h.validators["PostImportAtomicRecord"] = queryValidationSpecRequired().Optional("simPowerLossAfter")
h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote", "clear")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns", "profile")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "excludeColumns", "profile")
h.validators["GetInfo"] = queryValidationSpecRequired()
h.validators["RecalculateCaches"] = queryValidationSpecRequired()
h.validators["GetSchema"] = queryValidationSpecRequired().Optional("views")
@ -249,10 +249,6 @@ func (h *Handler) populateValidators() {
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard")
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index")
h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired()
h.validators["GetIndexAttrData"] = queryValidationSpecRequired()
h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired()
h.validators["GetFieldAttrData"] = queryValidationSpecRequired()
h.validators["GetNodes"] = queryValidationSpecRequired()
h.validators["GetShardMax"] = queryValidationSpecRequired()
h.validators["GetTransactionList"] = queryValidationSpecRequired()
@ -387,7 +383,6 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex")
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/import-column-attrs", handler.handlePostImportColumnAttrs).Methods("POST").Name("PostImportColumnAttrs")
router.HandleFunc("/index/{index}/field", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/", handler.handlePostField).Methods("POST").Name("PostField")
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField")
@ -424,15 +419,11 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff")
router.HandleFunc("/internal/index/{index}/attr/data", handler.handleGetIndexAttrData).Methods("GET").Name("GetIndexAttrData")
router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData")
router.HandleFunc("/internal/translate/data", handler.handlePostTranslateData).Methods("POST").Name("PostTranslateData")
router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys")
router.HandleFunc("/internal/translate/ids", handler.handlePostTranslateIDs).Methods("POST").Name("PostTranslateIDs")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff")
router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE")
router.HandleFunc("/internal/index/{index}/field/{field}/attr/data", handler.handleGetFieldAttrData).Methods("GET").Name("GetFieldAttrData")
router.HandleFunc("/internal/index/{index}/shard/{shard}/snapshot", handler.handleGetIndexShardSnapshot).Methods("GET").Name("GetIndexShardSnapshot")
router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards")
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
@ -1155,48 +1146,6 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
}
// handlePostIndexAttrDiff handles POST /internal/index/attr/diff requests.
func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
// Decode request.
var req postIndexAttrDiffRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
attrs, err := h.api.IndexAttrDiff(r.Context(), indexName, req.Blocks)
if err != nil {
if errors.Cause(err) == pilosa.ErrIndexNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.logger.Errorf("response encoding error: %s", err)
}
}
// handleGetIndexAttrData handles GET /internal/index/{index}/attr/data requests.
func (h *Handler) handleGetIndexAttrData(w http.ResponseWriter, r *http.Request) {
if err := h.api.WriteColumnAttrDataTo(r.Context(), w, mux.Vars(r)["index"]); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) {
var rtype string
switch {
@ -1264,14 +1213,6 @@ func (h *Handler) handleGetPastQueries(w http.ResponseWriter, r *http.Request) {
}
type postIndexAttrDiffRequest struct {
Blocks []pilosa.AttrBlock `json:"blocks"`
}
type postIndexAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// handlePostField handles POST /field request.
func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -1677,58 +1618,6 @@ func (h *Handler) handleDeleteRemoteAvailableShard(w http.ResponseWriter, r *htt
resp.write(w, err)
}
// handlePostFieldAttrDiff handles POST /internal/field/attr/diff requests.
func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
}
indexName := mux.Vars(r)["index"]
fieldName := mux.Vars(r)["field"]
// Decode request.
var req postFieldAttrDiffRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
attrs, err := h.api.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks)
if err != nil {
switch errors.Cause(err) {
case pilosa.ErrFragmentNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.logger.Errorf("response encoding error: %s", err)
}
}
type postFieldAttrDiffRequest struct {
Blocks []pilosa.AttrBlock `json:"blocks"`
}
type postFieldAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// handleGetFieldAttrData handles GET /internal/index/{index}/field/{field}/attr/data requests.
func (h *Handler) handleGetFieldAttrData(w http.ResponseWriter, r *http.Request) {
if err := h.api.WriteRowAttrDataTo(r.Context(), w, mux.Vars(r)["index"], mux.Vars(r)["field"]); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// handleGetIndexShardSnapshot handles GET /internal/index/{index}/shard/{shard}/snapshot requests.
func (h *Handler) handleGetIndexShardSnapshot(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
@ -1821,12 +1710,9 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er
}
return &pilosa.QueryRequest{
Query: query,
Shards: shards,
Profile: profile,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
ExcludeColumns: q.Get("excludeColumns") == "true",
Query: query,
Shards: shards,
Profile: profile,
}, nil
}
@ -2534,48 +2420,6 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
}
// handlePostImportColumnAttrs
func (h *Handler) handlePostImportColumnAttrs(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {
http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType)
return
} else if r.Header.Get("Accept") != "application/x-protobuf" {
http.Error(w, "Not acceptable", http.StatusNotAcceptable)
return
}
opts := []pilosa.ImportOption{}
body, err := readBody(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
req := &pilosa.ImportColumnAttrsRequest{}
if err := proto.DefaultSerializer.Unmarshal(body, req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := h.api.ImportColumnAttrs(r.Context(), req, opts...); err != nil {
switch errors.Cause(err) {
case pilosa.ErrClusterDoesNotOwnShard, pilosa.ErrPreconditionFailed:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Write response.
_, err = w.Write(importOk)
if err != nil {
h.logger.Errorf("writing import-column-attrs response: %v", err)
}
}
// handlePostImportRoaring
func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.

19
index.go
View file

@ -49,11 +49,6 @@ type Index struct {
// Fields by name.
fields map[string]*Field
newAttrStore func(string) AttrStore
// Column attribute storage and cache.
columnAttrs AttrStore
broadcaster broadcaster
Schemator disco.Schemator
serializer Serializer
@ -92,9 +87,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
name: name,
fields: make(map[string]*Field),
newAttrStore: newNopAttrStore,
columnAttrs: nopStore,
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
holder: holder,
@ -161,9 +153,6 @@ func (i *Index) TranslateStore(partitionID int) TranslateStore {
// Keys returns true if the index uses string keys.
func (i *Index) Keys() bool { return i.keys }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrs }
// Options returns all options for this index.
func (i *Index) Options() IndexOptions {
i.mu.RLock()
@ -252,10 +241,6 @@ func (i *Index) open(idx *disco.Index) (err error) {
}
}
if err := i.columnAttrs.Open(); err != nil {
return errors.Wrap(err, "opening attrstore")
}
if i.keys {
i.holder.Logger.Debugf("open translate store for index: %s", i.name)
@ -459,9 +444,6 @@ func (i *Index) Close() error {
return errors.Wrap(err, "closing index")
}
// Close the attribute store.
i.columnAttrs.Close()
// Close partitioned translation stores.
for _, store := range i.translateStores {
if err := store.Close(); err != nil {
@ -792,7 +774,6 @@ func (i *Index) newField(path, name string) (*Field, error) {
f.broadcaster = i.broadcaster
f.schemator = i.Schemator
f.serializer = i.serializer
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, RowAttrsFileName))
f.OpenTranslateStore = i.OpenTranslateStore
return f, nil
}

6
metrics.go
View file

@ -28,8 +28,6 @@ const (
MetricCacheThresholdReached = "cache_threshold_reached_total"
MetricRow = "query_row_total"
MetricRowBSI = "query_row_bsi_total"
MetricSetRowAttrs = "query_setrowattrs_total"
MetricSetColumnAttrs = "query_setcolumnattrs_total"
MetricSetBit = "set_bit_total"
MetricClearBit = "clear_bit_total"
MetricImportingN = "importing_total"
@ -40,10 +38,6 @@ const (
MetricBlockRepair = "block_repair_total"
MetricSyncFieldDurationSeconds = "sync_field_duration_seconds"
MetricSyncIndexDurationSeconds = "sync_index_duration_seconds"
MetricColumnAttrStoreBlocks = "column_attr_store_blocks_total"
MetricColumnAttrDiff = "column_attr_diff_total"
MetricRowAttrStoreBlocks = "row_attr_store_blocks_total"
MetricRowAttrDiff = "row_attr_diff_total"
MetricHTTPRequest = "http_request_duration_seconds"
MetricGRPCUnaryQueryDurationSeconds = "grpc_request_pql_unary_query_duration_seconds"
MetricGRPCUnaryFormatDurationSeconds = "grpc_request_pql_unary_format_duration_seconds"

14
pb/private.pb.go
View file

@ -1,17 +1,3 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: private.proto

1830
pb/public.pb.go
View file

File diff suppressed because it is too large Load diff

33
pb/public.proto
View file

@ -5,7 +5,6 @@ package pb;
message Row {
repeated uint64 Columns = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
bytes Roaring = 4;
string Index = 5;
string Field = 6;
@ -117,32 +116,10 @@ message Decimal {
int64 Scale = 2;
}
message ColumnAttrSet {
uint64 ID = 1;
string Key = 3;
repeated Attr Attrs = 2;
}
message Attr {
string Key = 1;
uint64 Type = 2;
string StringValue = 3;
int64 IntValue = 4;
bool BoolValue = 5;
double FloatValue = 6;
}
message AttrMap {
repeated Attr Attrs = 1;
}
message QueryRequest {
string Query = 1;
repeated uint64 Shards = 2;
bool ColumnAttrs = 3;
bool Remote = 5;
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
repeated Row EmbeddedData = 8;
bool PreTranslated = 9;
}
@ -150,7 +127,6 @@ message QueryRequest {
message QueryResponse {
string Err = 1;
repeated QueryResult Results = 2;
repeated ColumnAttrSet ColumnAttrSets = 3;
}
message QueryResult {
@ -254,15 +230,6 @@ message ImportRoaringRequest {
bool UpdateExistence = 7;
}
message ImportColumnAttrsRequest {
string Index = 1;
int64 Shard = 2;
string AttrKey = 3;
repeated string AttrVals = 4;
repeated uint64 ColumnIDs = 5;
int64 IndexCreatedAt = 6;
}
message GroupCounts{
string Aggregate = 1;
repeated GroupCount Groups = 2;

30
pilosa.go
View file

@ -15,7 +15,6 @@
package pilosa
import (
"encoding/json"
"os"
"regexp"
"time"
@ -148,35 +147,6 @@ func newPreconditionFailedError(err error) PreconditionFailedError {
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,229}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
// MarshalJSON marshals the ColumnAttrSet to JSON such that
// either a Key or an ID is included.
func (cas ColumnAttrSet) MarshalJSON() ([]byte, error) {
if cas.Key != "" {
return json.Marshal(struct {
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}{
Key: cas.Key,
Attrs: cas.Attrs,
})
}
return json.Marshal(struct {
ID uint64 `json:"id"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}{
ID: cas.ID,
Attrs: cas.Attrs,
})
}
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"

29
pilosa_internal_test.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"bytes"
"io"
"reflect"
"testing"
@ -48,34 +47,6 @@ func TestValidateNameInvalid(t *testing.T) {
}
}
// memAttrStore represents an in-memory implementation of the AttrStore interface.
type memAttrStore struct {
store map[uint64]map[string]interface{}
}
func (s *memAttrStore) Path() string { return "" }
func (s *memAttrStore) Open() error { return nil }
func (s *memAttrStore) Close() error { return nil }
func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return s.store[id], nil
}
func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
s.store[id] = m
return nil
}
func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
for id, v := range m {
s.store[id] = v
}
return nil
}
func (s *memAttrStore) Blocks() ([]AttrBlock, error) { return nil, nil }
func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}
func (s *memAttrStore) WriteTo(w io.Writer) (int64, error) { return 0, nil }
func TestAPI_CombineForExistence(t *testing.T) {
bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 3), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 5), pos(2, 65537), pos(2, 65538))
buf := new(bytes.Buffer)

31
pql/ast.go
View file

@ -282,7 +282,7 @@ func (q *Query) WriteCallN() int {
var n int
for _, call := range q.Calls {
switch call.Name {
case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs", "ClearRow", "Store", "SetBit":
case "Set", "Clear", "ClearRow", "Store", "SetBit":
n++
}
}
@ -506,10 +506,7 @@ var callInfoByFunc = map[string]callInfo{
"Options": {
allowUnknown: false,
prototypes: map[string]interface{}{
"excludeRowAttrs": true,
"excludeColumns": true,
"columnAttrs": true,
"shards": nil,
"shards": nil,
},
},
"Set": {
@ -528,21 +525,6 @@ var callInfoByFunc = map[string]callInfo{
"_col": stringOrInt64,
},
},
"SetRowAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"field": "",
"_row": stringOrInt64,
},
},
"SetColumnAttrs": {
allowUnknown: true,
prototypes: map[string]interface{}{
"_field": "",
"_col": stringOrInt64,
},
},
"IncludesColumn": {
allowUnknown: false,
prototypes: map[string]interface{}{
@ -552,7 +534,7 @@ var callInfoByFunc = map[string]callInfo{
}
// We want to allow case-insensitive names, but we want to continue using
// friendly easy-to-read names like "SetRowAttrs", not "setrowattrs". So,
// friendly easy-to-read names like "SetBit", not "setbit". So,
// we make a map; put in a ToLower() string, get back the canonical
// capitalization. This might not have seemed like the best strategy if we
// didn't already have so much code relying on the exact strings.
@ -888,13 +870,10 @@ func (c *Call) HasConditionArg() bool {
// TranslateInfo returns the relevant translation fields.
func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fieldName string) {
switch c.Name {
case "Set", "Clear", "Row", "Range", "SetColumnAttrs", "ClearRow":
case "Set", "Clear", "Row", "Range", "ClearRow":
// Positional args in new PQL syntax require special handling here.
fieldName, _ = c.FieldArg()
return "_" + columnLabel, fieldName, fieldName
case "SetRowAttrs":
// Positional args in new PQL syntax require special handling here.
return "", "_" + rowLabel, c.ArgString("_field")
case "Rows":
return "column", "previous", c.ArgString("_field")
case "IncludesColumn":
@ -909,7 +888,7 @@ func (c *Call) TranslateInfo(columnLabel, rowLabel string) (colKey, rowKey, fiel
// Writable returns true if call is mutable (e.g. can write new translation keys)
func (c *Call) Writable() bool {
switch c.Name {
case "Set", "SetRowAttrs", "SetColumnAttrs", "SetBit":
case "Set", "SetBit":
return true
case "Not":
// to support queries like Not(Row(f="garbage"))

5
pql/pql.peg
View file

@ -7,8 +7,6 @@ type PQL Peg {
# All input queries consist of a sequence of calls, at the top level.
Calls <- sp (Call sp)* !.
Call <- "Set" {p.startCall("Set")} open col comma args (comma time)? close {p.endCall()}
/ "SetRowAttrs" {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
/ "SetColumnAttrs" {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
/ "Clear" {p.startCall("Clear")} open col comma args close {p.endCall()}
/ "ClearRow" {p.startCall("ClearRow")} open arg close {p.endCall()}
/ "Store" {p.startCall("Store")} open Call comma arg close {p.endCall()}
@ -63,9 +61,6 @@ posfield <- 'field='? <fieldExpr> { p.addPosStr("_field", text) }
col <- < digits > {p.addPosNum("_col", text)}
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)}
/ < '"' doublequotedstring '"' > {p.addPosStr("_col", text)}
row <- < digits > {p.addPosNum("_row", text)}
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_row", text)}
/ < '"' doublequotedstring '"' > {p.addPosStr("_row", text)}
open <- '(' sp
close <- sp ')' sp

4413
pql/pql.peg.go
View file

File diff suppressed because it is too large Load diff

138
pql/pqlpeg_test.go
View file

@ -36,16 +36,6 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
}
p.Execute()
p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
err = p.Init()
if err != nil {
t.Fatal(errors.Wrap(err, "creating parser"))
}
err = p.Parse()
if err == nil {
t.Fatalf("should have been an error because of the interior unescaped double quote")
}
q, err := ParseString("TopN(blah, Bitmap(id==other), field=f, n=0)")
if err != nil {
t.Fatalf("should have parsed: %v", err)
@ -190,38 +180,6 @@ func TestPEGWorking(t *testing.T) {
name: "single quoted args",
input: `Row(a='zm""e')`,
ncalls: 1},
{
name: "SetRowAttrs",
input: "SetRowAttrs(blah, 9, a=47)",
ncalls: 1},
{
name: "SetRowAttrs2args",
input: "SetRowAttrs(blah, 9, a=47, b=bval)",
ncalls: 1},
{
name: "SetRowAttrsWithRowKeySingleQuote",
input: "SetRowAttrs(blah, 'rowKey', a=47)",
ncalls: 1},
{
name: "SetRowAttrsWithRowKeyDoubleQuote",
input: `SetRowAttrs(blah, "rowKey", a=47)`,
ncalls: 1},
{
name: "SetColumnAttrs",
input: "SetColumnAttrs(9, a=47)",
ncalls: 1},
{
name: "SetColumnAttrs2args",
input: "SetColumnAttrs(9, a=47, b=bval)",
ncalls: 1},
{
name: "SetColumnAttrsWithColKeySingleQuote",
input: "SetColumnAttrs('colKey', a=47)",
ncalls: 1},
{
name: "SetColumnAttrsWithColKeyDoubleQuote",
input: `SetColumnAttrs("colKey", a=47)`,
ncalls: 1},
{
name: "Clear",
input: "Clear(1, a=53)",
@ -352,9 +310,6 @@ func TestPEGErrors(t *testing.T) {
{
name: "StartinCommaArb",
input: "Row(, a=4)"},
{
name: "SetRowAttrs0args",
input: "SetRowAttrs(blah, 9)"},
{
name: "Clear0args",
input: "Clear(9)"},
@ -485,91 +440,6 @@ func TestPQLDeepEquality(t *testing.T) {
"_field": "myfield",
},
}},
{
name: "SetRowAttrs",
call: "SetRowAttrs(myfield, 9, z=4)",
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": int64(9),
},
}},
{
name: "SetRowAttrsWithField=",
call: "SetRowAttrs(field=myfield, 9, z=4)",
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": int64(9),
},
}},
{
name: "SetRowAttrsWithRowKeySingleQuote",
call: "SetRowAttrs(myfield, 'rowKey', z=4)",
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": "rowKey",
},
}},
{
name: "SetRowAttrsWithRowKeyDoubleQuote",
call: `SetRowAttrs(myfield, "rowKey", z=4)`,
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": "rowKey",
},
}},
{
name: "SetRowAttrsWithUnicodeValues",
call: `SetRowAttrs(myfield, "∫", z="∀", a="∑")`,
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": "∀",
"a": "∑",
"_field": "myfield",
"_row": "∫",
},
}}, {
name: "SetColumnAttrs",
call: "SetColumnAttrs(9, z=4)",
exp: &Call{
Name: "SetColumnAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_col": int64(9),
},
}},
{
name: "SetColumnAttrsWithColKeySingleQuote",
call: "SetColumnAttrs('colKey', z=4)",
exp: &Call{
Name: "SetColumnAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_col": "colKey",
},
}},
{
name: "SetColumnAttrsWithColKeyDoubleQuote",
call: `SetColumnAttrs("colKey", z=4)`,
exp: &Call{
Name: "SetColumnAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_col": "colKey",
},
}},
{
name: "Clear",
call: "Clear(1, a=7)",
@ -839,11 +709,15 @@ func TestPQLDeepEquality(t *testing.T) {
}},
{
name: "OptionsWrapper",
call: "Options(Row(f1=123), excludeRowAttrs=true)",
call: "Options(Row(f1=123), shards=[1,2,3])",
exp: &Call{
Name: "Options",
Args: map[string]interface{}{
"excludeRowAttrs": true,
"shards": []interface{}{
int64(1),
int64(2),
int64(3),
},
},
Children: []*Call{
{

21
row.go
View file

@ -23,17 +23,13 @@ import (
"github.com/pkg/errors"
)
// Row is a set of integers (the associated columns), and attributes which are
// arbitrary key/value pairs storing metadata about what the row represents.
// Row is a set of integers (the associated columns).
type Row struct {
segments []rowSegment
// String keys translated to/from segment columns.
Keys []string
// Attributes associated with the row.
Attrs map[string]interface{}
// Index tells what index this row is from - needed for key translation.
Index string
@ -67,13 +63,8 @@ func (r *Row) Clone() (clone *Row) {
copy(keyClone, r.Keys)
}
attrClone := make(map[string]interface{})
for k, v := range r.Attrs {
attrClone[k] = v
}
clone = &Row{
Keys: keyClone,
Attrs: attrClone,
Index: r.Index,
Field: r.Field,
}
@ -474,18 +465,12 @@ func (r *Row) Count() uint64 {
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys,omitempty"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys,omitempty"`
}
o.Columns = r.Columns()
o.Keys = r.Keys
o.Attrs = r.Attrs
if o.Attrs == nil {
o.Attrs = make(map[string]interface{})
}
return json.Marshal(&o)
}

10
server.go
View file

@ -141,16 +141,6 @@ func OptServerDataDir(dir string) ServerOption {
}
}
// OptServerAttrStoreFunc is a functional option on Server
// used to provide the function to use to generate a new
// attribute store.
func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption {
return func(s *Server) error {
s.holderConfig.NewAttrStore = af
return nil
}
}
// OptServerAntiEntropyInterval is a functional option on Server
// used to set the anti-entropy interval.
func OptServerAntiEntropyInterval(interval time.Duration) ServerOption {

12
server/cluster_test.go
View file

@ -234,7 +234,7 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatal(err)
}
// exp is the expected result for the Row queries that follow.
exp := fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1,%d]}]}`, col)
exp := fmt.Sprintf(`{"results":[{"columns":[1,%d]}]}`, col)
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -277,7 +277,7 @@ func TestClusterResize_AddNode(t *testing.T) {
t.Fatal(err)
}
// exp is the expected result for the Row queries that follow.
exp := `{"results":[{"attrs":{},"columns":[1]}]}`
exp := `{"results":[{"columns":[1]}]}`
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -324,7 +324,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// exp is the expected result for the Row queries that follow.
exp := fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1,%d]}]}`, col)
exp := fmt.Sprintf(`{"results":[{"columns":[1,%d]}]}`, col)
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -411,7 +411,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
// exp is the expected result for the Row queries that follow.
exp := fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1,%d]}]}`, col)
exp := fmt.Sprintf(`{"results":[{"columns":[1,%d]}]}`, col)
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -457,7 +457,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
// exp is the expected result for the Row queries that follow.
exp := fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1,%d]}]}`, col)
exp := fmt.Sprintf(`{"results":[{"columns":[1,%d]}]}`, col)
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -501,7 +501,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
// exp is the expected result for the Row queries that follow.
exp := `{"results":[{"attrs":{},"columns":[],"keys":["col2","col1"]}]}`
exp := `{"results":[{"columns":[],"keys":["col2","col1"]}]}`
// Verify the data exists on the single node.
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)

3
server/config.go
View file

@ -83,8 +83,7 @@ type Config struct {
AdvertiseGRPC string `toml:"advertise-grpc"`
// MaxWritesPerRequest limits the number of mutating commands that can be in
// a single request to the server. This includes Set, Clear,
// SetRowAttrs & SetColumnAttrs.
// a single request to the server. This includes Set, Clear, ClearRow, Store, and SetBit.
MaxWritesPerRequest int `toml:"max-writes-per-request"`
// LogPath configures where Pilosa will write logs.

1
server/grpc.go
View file

@ -215,7 +215,6 @@ func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQ
resp, err := h.api.Query(stream.Context(), &query)
durQuery := time.Since(t)
// TODO: what about resp.CollumnAttrSets?
if err != nil {
return errToStatusError(err)
} else if len(resp.Results) != 1 {

156
server/handler_test.go
View file

@ -687,31 +687,11 @@ func TestHandler_Endpoints(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" {
} else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
f0 := i0.Field("f0")
if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.ShardWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
} else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil {
t.Fatal(err)
}
t.Run("ColumnAttrs_JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n"
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != exp {
t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp)
}
})
t.Run("Row pbuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))
@ -726,62 +706,6 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
} else if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
})
t.Run("Row columnattrs protobuf", func(t *testing.T) {
// Encode request body.
buf, err := cmd.API.Serializer.Marshal(&pilosa.QueryRequest{
Query: "Row(f0=30)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp pilosa.QueryResponse
if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{pilosa.ShardWidth + 1, pilosa.ShardWidth + 2, (3 * pilosa.ShardWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if _, ok := resp.Results[0].(*pilosa.Row); !ok {
t.Fatalf("unexpected response type: %#v", resp.Results[0])
} else if attrs := resp.Results[0].(*pilosa.Row).Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if attrs["a"] != "b" {
t.Fatalf("unexpected attr[a]: %v", attrs["a"])
} else if attrs["c"] != int64(1) {
t.Fatalf("unexpected attr[c]: %v", attrs["c"])
} else if !attrs["d"].(bool) {
t.Fatalf("unexpected attr[d]: %v", attrs["d"])
}
if a := resp.ColumnAttrSets; len(a) != 2 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != pilosa.ShardWidth+1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if a[0].Attrs["x"] != "y" {
t.Fatalf("unexpected attr[x]: %v", a[0].Attrs["x"])
}
})
@ -1091,83 +1015,7 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err := i.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("AttrStore Diff", func(t *testing.T) {
blks, err := i.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/internal/index/i/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
meta, err := i.CreateFieldIfNotExists("meta", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
if err := meta.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("field attrstore diff", func(t *testing.T) {
blks, err := meta.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/internal/index/i/field/meta/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
t.Run("Version", func(t *testing.T) {
w := httptest.NewRecorder()

1
server/server.go
View file

@ -478,7 +478,6 @@ func (m *Command) SetupServer() error {
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(c, &sync.Mutex{})),
pilosa.OptServerOpenIDAllocator(pilosa.OpenIDAllocator),
pilosa.OptServerLogger(m.logger),
pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),

131
server/server_test.go
View file

@ -91,7 +91,6 @@ func TestMain_Set_Quick(t *testing.T) {
"results": []interface{}{
map[string]interface{}{
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -118,7 +117,6 @@ func TestMain_Set_Quick(t *testing.T) {
"results": []interface{}{
map[string]interface{}{
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -133,135 +131,6 @@ func TestMain_Set_Quick(t *testing.T) {
}
}
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateField(context.Background(), "i", "x"); err != nil {
t.Fatal(err)
} else if err := client.CreateField(context.Background(), "i", "z"); err != nil {
t.Fatal(err)
} else if err := client.CreateField(context.Background(), "i", "neg"); err != nil {
t.Fatal(err)
}
// Set columns on different rows in different fields.
if _, err := m.Query(t, "i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `Set(100, neg=3)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query(t, "i", "", `SetRowAttrs(x, 1, x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `SetRowAttrs(x, 2, x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `SetRowAttrs(z, 2, x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `SetRowAttrs(neg, 3, x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query(t, "i", "", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query(t, "i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
if err := m.Reopen(); err != nil {
t.Fatal(err)
}
if err := m.AwaitState(disco.ClusterStateNormal, 10*time.Second); err != nil {
t.Fatalf("restarting cluster: %v", err)
}
// Query rows after reopening.
if res, err := m.Query(t, "i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query(t, "i", "columnAttrs=true", `Row(neg=3)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query(t, "i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateField(context.Background(), "i", "x"); err != nil {
t.Fatal(err)
}
// Set columns on row.
if _, err := m.Query(t, "i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query(t, "i", "", `Set(101, x=1)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query(t, "i", "", `SetColumnAttrs(100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query(t, "i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
if err := m.Reopen(); err != nil {
t.Fatal(err)
}
if err := m.AwaitState(disco.ClusterStateNormal, 10*time.Second); err != nil {
t.Fatalf("restarting cluster: %v", err)
}
// Query row after reopening.
if res, err := m.Query(t, "i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
func TestMain_GroupBy(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()

2
sql/mapper.go
View file

@ -33,8 +33,6 @@ var (
ErrMultipleSQLStatements = errors.New("statement contains multiple sql queries")
)
type Attributes map[string]interface{}
type MappedSQL struct {
SQLType string
Statement sqlparser.Statement

4
sql/select.go
View file

@ -178,9 +178,7 @@ type selectFunc struct {
}
type selectFeatures struct {
HasRowAttrs bool
HasColAttrs bool
funcs []selectFunc
funcs []selectFunc
}
type HavingClause struct {

68
stats/stats_test.go
View file

@ -152,74 +152,6 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
}
func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
field := hldr.Field("d", "f")
if field == nil {
t.Fatal("field not found")
}
hldr.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != pilosa.MetricSetRowAttrs {
t.Errorf("Expected %v, Results %s", pilosa.MetricSetRowAttrs, name)
}
if tags[0] != "index:d" {
t.Errorf("Expected index, Results %s", tags[0])
}
called = true
},
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
t.Fatal(err)
}
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_SetColumnAttrs(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
idx := hldr.Holder.Index("d")
if idx == nil {
t.Fatal("index not found")
}
hldr.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != pilosa.MetricSetColumnAttrs {
t.Errorf("Expected %v, Results %s", pilosa.MetricSetColumnAttrs, name)
}
if tags[0] != "index:d" {
t.Errorf("Expected index, Results %s", tags[0])
}
called = true
},
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil {
t.Fatal(err)
}
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_APICalls(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()

11
test/holder.go
View file

@ -20,7 +20,6 @@ import (
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
@ -40,7 +39,6 @@ func NewHolder(tb testing.TB) *Holder {
}
h := &Holder{Holder: pilosa.NewHolder(path, nil)}
h.Holder.NewAttrStore = boltdb.NewAttrStore
return h
}
@ -115,15 +113,6 @@ func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row {
return row.Clone()
}
func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())
if err != nil {
panic(err)
}
return f.RowAttrStore()
}
func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum string) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())

4
txfactory.go
View file

@ -643,7 +643,7 @@ func (f *TxFactory) IndexUsageDetails() (map[string]IndexUsage, uint64, error) {
fieldUsages[field] = fUsage
}
// index metadata, e.g. columnAttrs
// index metadata
indexMetaBytes, err := directoryUsage(indexPath, false)
if err != nil {
return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index metadata (%s)", index)
@ -691,7 +691,7 @@ func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error)
keysBytes = 0
}
// field metadata, e.g. rowAttrs
// field metadata
fieldPath := path.Join(indexPath, FieldsDir, field)
metaBytes, err := directoryUsage(fieldPath, false) // this includes keys
if err != nil {

7
view.go
View file

@ -60,9 +60,8 @@ type view struct {
// Fragments by shard.
fragments map[uint64]*fragment
broadcaster broadcaster
stats stats.StatsClient
rowAttrStore AttrStore
broadcaster broadcaster
stats stats.StatsClient
knownShards *roaring.Bitmap
knownShardsCopied uint32
@ -148,7 +147,6 @@ func (v *view) openWithShardSet(ss *shardSet) error {
for shard := range shards {
frag := v.newFragment(shard)
frags = append(frags, frag)
frag.RowAttrStore = v.rowAttrStore
v.fragments[frag.shard] = frag
}
@ -333,7 +331,6 @@ func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
if err := frag.Open(); err != nil {
return nil, errors.Wrap(err, "opening fragment")
}
frag.RowAttrStore = v.rowAttrStore
v.fragments[shard] = frag
v.addKnownShard(shard)

3
view_internal_test.go
View file

@ -55,9 +55,6 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
if err := v.openEmpty(); err != nil {
PanicOn(err)
}
v.rowAttrStore = &memAttrStore{
store: make(map[uint64]map[string]interface{}),
}
return v
}