From 93e5767325ab83e14484082f4d71c8c7b3a19d7d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 6 Sep 2018 16:27:10 -0500 Subject: [PATCH] cleanup #1622 --- api.go | 6 +++--- client.go | 4 ++-- http/client.go | 7 +------ http/handler.go | 3 +-- roaring/roaring.go | 4 ++-- roaring/roaring_internal_test.go | 10 +++++----- .../{ => testdata}/bitmapcontainer.roaringbitmap | Bin 7 files changed, 14 insertions(+), 20 deletions(-) rename roaring/{ => testdata}/bitmapcontainer.roaringbitmap (100%) diff --git a/api.go b/api.go index 77ce4d0a0..ffe85dd37 100644 --- a/api.go +++ b/api.go @@ -296,7 +296,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e } // ImportRoaringBytes fast loading of standard roaring format -func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64, forward bool) (err error) { +func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64 ) (err error) { if err = api.validate(apiField); err != nil { err = errors.Wrap(err, "validating api method") return @@ -316,12 +316,12 @@ func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, ind err = field.importRoaringBytes(roaringBytes, shard) wg.Done() }(node) - } else if forward { + } else { wg.Add(1) //forward it on go func(node *Node) { //execute on node - err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes, forward) + err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes) wg.Done() }(node) } diff --git a/client.go b/client.go index d0e36db6d..d6f1f959f 100644 --- a/client.go +++ b/client.go @@ -51,7 +51,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error + ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error } //=============== @@ -107,7 +107,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { +func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte ) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/http/client.go b/http/client.go index 91fd7b571..7cc941c16 100644 --- a/http/client.go +++ b/http/client.go @@ -522,7 +522,7 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui } // ImportRoaringBytes fast import of raw bits in roaring standard format -func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte, forward bool) error { +func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -533,11 +533,6 @@ func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.No // Create URL. u := nodePathToURL(node, endpoint) - if forward { - v := url.Values{} - v.Set("noforward", "y") - u.RawQuery = v.Encode() - } // Generate HTTP request. req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes)) if err != nil { diff --git a/http/handler.go b/http/handler.go index f29128bd2..f86549c3c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1444,7 +1444,6 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request indexName := mux.Vars(r)["index"] fieldName := mux.Vars(r)["field"] shardName := mux.Vars(r)["shard"] - _, noForward := r.URL.Query()["noforward"] // Read entire body. body, err := ioutil.ReadAll(r.Body) @@ -1459,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard, !noForward) + err = h.api.ImportRoaringBytes(r.Context(), body, indexName, fieldName, shard ) // Marshal response object. msg := string("") diff --git a/roaring/roaring.go b/roaring/roaring.go index 853fcade9..abf36c40e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3340,7 +3340,7 @@ func popcountAndSlice(s, m []uint64) uint64 { return cnt } -// Stuff i needed from the roaring project +// constants from github.com/RoaringBitmap/roaring const ( //taken from roaring/util.go serialCookieNoRunContainer = 12346 // only arrays and bitmaps serialCookie = 12347 // runs, arrays, and bitmaps @@ -3404,7 +3404,7 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint err = fmt.Errorf("malformed bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) return } - pos += 2 * 2 * int(size) //moving pos past keycount + pos += 2 * 2 * int(size) // moving pos past keycount return } diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 581a8040f..2469657d1 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3230,8 +3230,8 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { func TestUnmarshalStdRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays - _2arrayContainer, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") - bm, er := UnmarshalStandardRoaring(_2arrayContainer) + rbContainerWithTwoArrays, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") + bm, er := UnmarshalStandardRoaring(rbContainerWithTwoArrays) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3239,8 +3239,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 65537]", bm.Slice()) } //generated serialize image from java(clojure) with a run and array - _rle_array_container, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - bm, er = UnmarshalStandardRoaring(_rle_array_container) + rbContainerWithRLEandArray, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + bm, er = UnmarshalStandardRoaring(rbContainerWithRLEandArray) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3248,7 +3248,7 @@ func TestUnmarshalStdRoaring(t *testing.T) { t.Fatalf("unexpected bitmap %v expected bits [1 2 3 4 5 6 7 8 9 10 65537]", bm.Slice()) } //had to use an external file because emacs was barfing on the long line :() - _bitmap_array_container, _ := ioutil.ReadFile("./bitmapcontainer.roaringbitmap") + _bitmap_array_container, _ := ioutil.ReadFile("testdata/bitmapcontainer.roaringbitmap") bm, er = UnmarshalStandardRoaring(_bitmap_array_container) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) diff --git a/roaring/bitmapcontainer.roaringbitmap b/roaring/testdata/bitmapcontainer.roaringbitmap similarity index 100% rename from roaring/bitmapcontainer.roaringbitmap rename to roaring/testdata/bitmapcontainer.roaringbitmap