Merge pull request #1412 from tgruben/import-roaring-existence

Update existence field on import-roaring requests

api.go

@@ -416,6 +416,12 @@ func importWorker(importWork chan importJob) {
 					case RequestActionSet:
 						fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
 						if fileMagic == roaring.MagicNumber { // if pilosa roaring format
+							if ef := j.field.idx.existenceField(); ef != nil {
+								err = ef.importRoaring(j.ctx, tx, viewData, j.shard, "standard", false)
+								if err != nil {
+									return errors.Wrap(err, "importing pilosa roaring existence")
+								}
+							}
 							err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear)
 							if err != nil {
 								return errors.Wrap(err, "importing pilosa roaring")
@@ -425,6 +431,12 @@ func importWorker(importWork chan importJob) {
 							// field.importRoaring changes the standard roaring run format to pilosa roaring
 							data := make([]byte, len(viewData))
 							copy(data, viewData)
+							if ef := j.field.idx.existenceField(); ef != nil {
+								err = ef.importRoaring(j.ctx, tx, data, j.shard, "standard", false)
+								if err != nil {
+									return errors.Wrap(err, "importing pilosa roaring existence")
+								}
+							}
 							err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear)
 
 							if err != nil {
@@ -468,7 +480,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 	span, ctx := tracing.StartSpanFromContext(ctx, "API.ImportRoaring")
 	span.LogKV("index", indexName, "field", fieldName)
 	defer span.Finish()
-
 	if err := api.validate(apiField); err != nil {
 		return errors.Wrap(err, "validating api method")
 	}

executor.go

@@ -4569,7 +4569,6 @@ func (e *executor) executeUnionRows(ctx context.Context, qcx *Qcx, index string,
 
 // executeAllCallShard executes an All() call for a local shard.
 func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (res *Row, err0 error) {
-
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard")
 	defer span.Finish()
 
@@ -4596,7 +4595,6 @@ func (e *executor) executeAllCallShard(ctx context.Context, qcx *Qcx, index stri
 		}
 
 		defer finisher(&err0)
-
 		if existenceRow, err = existenceFrag.row(tx, 0); err != nil {
 			return nil, err
 		}

http/client_test.go

@@ -1435,3 +1435,49 @@ func TestClient_ServerInfoHasTxSrc(t *testing.T) {
 	}
 	pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid
 }
+func TestClient_ImportRoaringExists(t *testing.T) {
+	cluster := test.MustNewCluster(t, 1)
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
+	defer cluster.Close()
+
+	node := cluster.GetNode(0)
+	_, err = node.API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{TrackExistence: true})
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	_, err = node.API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+	// Send import request.
+	host := node.URL()
+	c := MustNewClient(host, http.GetHTTPClient(nil))
+	// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
+	roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
+
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+		t.Fatal(err)
+	}
+	expected := []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}
+	var qr pilosa.QueryResponse
+	qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Row(f=0)"})
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	got := qr.Results[0].(*pilosa.Row).Columns()
+	if !reflect.DeepEqual(got, expected) {
+		t.Fatalf(" Row unexpected columns: got %+v  expected: %+v", got, expected)
+	}
+	qr, err = node.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "All()"})
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	got = qr.Results[0].(*pilosa.Row).Columns()
+	if !reflect.DeepEqual(got, expected) {
+		t.Fatalf("All unexpected columns: got %+v  expected: %+v", got, expected)
+	}
+
+}

http/handler.go

@@ -2486,7 +2486,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
 		http.Error(w, error, code)
 		return
 	}
-
 	// Get index and field type to determine how to handle the
 	// import data.
 	indexName := mux.Vars(r)["index"]

index.go

@@ -482,7 +482,6 @@ func (i *Index) Fields() []*Field {
 func (i *Index) existenceField() *Field {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
-
 	return i.existenceFld
 }