Merge branch 'master' into chore/syang/remove-prodpad

.circleci/config.yml

@@ -38,25 +38,21 @@ commands:
     description: "skips the parent job if the PR includes no changes to pilosa"
     steps:
       - run: |
-          ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')"
+          ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
           echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
-          if [ -z $ROOT_CHANGED_FILES ] ; then
+          if [ -z "$ROOT_CHANGED_FILES" ] ; then
             echo "halting step"
             circleci step halt
-          else
-            true
           fi
   skip-if-lattice-unchanged:
     description: "skips the parent job if the PR includes no changes to lattice"
     steps:
       - run: |
-          LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')"
+          LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
           echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
-          if [ -z $LATTICE_CHANGED_FILES ] ; then
+          if [ -z "$LATTICE_CHANGED_FILES" ] ; then
             echo "halting step"
             circleci step halt
-          else
-            true
           fi
 
 jobs:

executor.go

@@ -3912,6 +3912,11 @@ func (e *executor) executeExternalLookup(ctx context.Context, qcx *Qcx, index st
 		return ExtractedTable{}, errors.New("too many inputs to lookup query")
 	}
 
+	write, _, err := c.BoolArg("write")
+	if err != nil {
+		return ExtractedTable{}, errors.Wrap(err, "parsing write argument")
+	}
+
 	rawArg, err := e.executeCall(ctx, qcx, index, c.Children[0], shards, opt)
 	if err != nil {
 		return ExtractedTable{}, errors.Wrapf(err, "evaluating SQL argument call %q", c.String())
@@ -3937,6 +3942,21 @@ func (e *executor) executeExternalLookup(ctx context.Context, qcx *Qcx, index st
 		arg = argRow.Columns()
 	}
 
+	if write {
+		tx, err := e.Holder.lookupDB.BeginTx(ctx, nil)
+		if err != nil {
+			return ExtractedTable{}, errors.Wrap(err, "creating postgres transaction")
+		}
+		defer tx.Rollback() //nolint:errcheck
+
+		_, err = tx.ExecContext(ctx, query, pq.Array(arg))
+		if err != nil {
+			return ExtractedTable{}, errors.Wrap(err, "executing postgres write")
+		}
+
+		return ExtractedTable{}, errors.Wrap(tx.Commit(), "committing postgres transaction")
+	}
+
 	result, err := e.Holder.lookupDB.QueryContext(ctx, query, pq.Array(arg))
 	if err != nil {
 		return ExtractedTable{}, errors.Wrapf(err, "SQL query failed")

executor_test.go

@@ -7913,7 +7913,7 @@ func TestExternalLookup(t *testing.T) {
 	defer c.Close()
 
 	// Populate a field with some data that can be used in queries.
-	c.CreateField(t, "i", pilosa.IndexOptions{}, "f")
+	c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f")
 	c.ImportBits(t, "i", "f", [][2]uint64{
 		{1, 1},
 		{1, 3},
@@ -8053,4 +8053,12 @@ func TestExternalLookup(t *testing.T) {
 			})
 		}
 	})
+	t.Run("Delete", func(t *testing.T) {
+		c.Query(t, "i", `ExternalLookup(All(), query="delete from lookup where id = ANY($1)", write=true)`)
+		res := c.Query(t, "i", `ExternalLookup(All(), query="select id from lookup where id = ANY($1)")`)
+		tbl := res.Results[0].(pilosa.ExtractedTable)
+		if len(tbl.Columns) != 0 {
+			t.Errorf("unexpected remaining records: %v", tbl)
+		}
+	})
 }

pql/ast.go

@@ -437,6 +437,7 @@ var callInfoByFunc = map[string]callInfo{
 		allowUnknown: false,
 		prototypes: map[string]interface{}{
 			"query": "",
+			"write": true,
 		},
 	},
 	"Limit": {