Sup 294 pre sort command (#2209)

* first cut at pre-sort command that works on ndjson

* finish pre_sort command for CSV and JSON and add test

* try fixing golangci-lint

* remove some dumb lint checks

* more linter disabling

* take .golangci.yml from previous repo

* go fmt (facepalm)

* remove ioutil to fix lint

.github/workflows/ci.yml

@@ -22,6 +22,22 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           dry: true
 
+  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          # version: v1.29
+          args: --timeout=5m
+
   validate:
     name: Code Checks
     runs-on: ubuntu-latest
@@ -40,10 +56,5 @@ jobs:
       - name: go vet
         run: go vet ./...
 
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --timeout=5m
-
       - name: test
         run: go test ./...

api.go

@@ -26,8 +26,8 @@ import (
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
 	"github.com/featurebasedb/featurebase/v3/dax/storage"
-	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/rbf"
 	"github.com/prometheus/client_golang/prometheus"
 

apply.go

@@ -14,10 +14,10 @@ import (
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 
 	ivy "robpike.io/ivy/arrow"
@@ -95,7 +95,7 @@ func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx con
 			col := value.ToArrowColumn(accumulator, pool)
 			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
 		}
-		// only acutally reduce on the initiating node i hate the network
+		// only actually reduce on the initiating node i hate the network
 		// over head but oh well
 		ctxIvy.AssignGlobal("_", accumulator)
 		ok, err := runIvyString(ctxIvy, reduceCode)
@@ -541,7 +541,7 @@ func (sf *ShardFile) Save(name string) error {
 		if sf.table != nil {
 			// we append if there was existing file
 			column := sf.table.Column(col)
-			// if primative type
+			// if primitive type
 			switch column.DataType() {
 			case arrow.BinaryTypes.String:
 				chunks = sf.buildFromStrings(col, mem)

arrow.go

@@ -17,9 +17,9 @@ import (
 	"github.com/apache/arrow/go/v10/parquet"
 	"github.com/apache/arrow/go/v10/parquet/file"
 	"github.com/apache/arrow/go/v10/parquet/pqarrow"
-	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 )
 

client/client.go

@@ -20,7 +20,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/client/types"
 	fbproto "github.com/featurebasedb/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import
@@ -30,6 +29,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"

cmd/pre_sort.go

@@ -0,0 +1,29 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+func newPreSortCommand(logdest logger.Logger) *cobra.Command {
+	cmd := ctl.NewPreSortCommand(logdest)
+	ccmd := &cobra.Command{
+		Use:   "pre_sort",
+		Short: "Sort records within files into files by FB partition for more efficient ingest",
+		Long: `
+Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
+`,
+		RunE: usageErrorWrapper(cmd),
+	}
+
+	flags := ccmd.Flags()
+	flags.StringVarP(&cmd.File, "file", "", "", "Input file or directory.")
+	flags.StringVarP(&cmd.Table, "table", "", "", "Name of table (used to hash keys to determine partition).")
+	flags.StringVarP(&cmd.Type, "type", "", cmd.Type, "Input file type (csv or ndjson).")
+	flags.StringSliceVar(&cmd.PrimaryKeyFields, "primary-key-fields", []string{}, "Names of primary key fields. For CSV there must be a header row and these pulled from there.")
+	flags.IntVar(&cmd.PartitionN, "partition-n", cmd.PartitionN, "Number of partitions.")
+	flags.StringVarP(&cmd.OutputDir, "output-dir", "", cmd.OutputDir, "Directory name to write output to.")
+	return ccmd
+}

cmd/root.go

@@ -107,6 +107,7 @@ at https://docs.featurebase.com/.
 	rc.AddCommand(newCLICommand(logdest))
 	rc.AddCommand(newDAXCommand(stderr))
 	rc.AddCommand(newDataframeCsvLoaderCommand(logdest))
+	rc.AddCommand(newPreSortCommand(logdest))
 
 	rc.SetOutput(stderr)
 	return rc

cmd/server.go

@@ -8,7 +8,6 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 
-
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/server"
 	"github.com/featurebasedb/featurebase/v3/tracing"

cmd/server_test.go

@@ -9,11 +9,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/felixge/fgprof"
 	"github.com/featurebasedb/featurebase/v3/cmd"
 	_ "github.com/featurebasedb/featurebase/v3/test"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/featurebasedb/featurebase/v3/toml"
+	"github.com/felixge/fgprof"
 	"github.com/pkg/errors"
 )
 

ctl/keygen.go

@@ -8,8 +8,8 @@ import (
 	"io"
 	"os"
 
-	"github.com/gorilla/securecookie"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/gorilla/securecookie"
 )
 
 // Keygen represents a command for generating a cryptographic key.

ctl/pre_sort.go

@@ -0,0 +1,245 @@
+package ctl
+
+import (
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"hash/fnv"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/pkg/errors"
+)
+
+type PreSortCommand struct {
+	// Optional Index filter
+	File string `json:"file"`
+
+	Type string `json:"type"`
+
+	Table string `json:"table"`
+
+	PrimaryKeyFields []string `json:"primary-key-fields"`
+
+	PartitionN int `json:"partition-n"`
+
+	// Path to write sorted files to
+	OutputDir string
+
+	// Standard input/output
+	logDest logger.Logger
+
+	outputFiles map[int]*os.File
+}
+
+// NewPreSortCommand returns a new instance of PreSortCommand.
+func NewPreSortCommand(logdest logger.Logger) *PreSortCommand {
+	return &PreSortCommand{
+		logDest:     logdest,
+		Type:        "ndjson",
+		OutputDir:   "presorted_files",
+		PartitionN:  256,
+		outputFiles: make(map[int]*os.File),
+	}
+}
+
+// Run executes the main program execution.
+func (cmd *PreSortCommand) Run(ctx context.Context) error {
+	if cmd.File == "" {
+		return errors.New("must set a file or directory to read from")
+	}
+	if len(cmd.PrimaryKeyFields) == 0 {
+		return errors.New("must define primary-key-fields")
+	}
+	if cmd.Table == "" {
+		return errors.New("must set a table name")
+	}
+	filelist := []string{}
+	walkfunc := func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
+			filelist = append(filelist, path)
+		}
+		return nil
+	}
+	err := filepath.WalkDir(cmd.File, walkfunc)
+	if err != nil {
+		return errors.Wrapf(err, "wakling %s", cmd.File)
+	}
+	if len(filelist) > 3 {
+		cmd.logDest.Printf("%d files: %v...", len(filelist), filelist[:3])
+	} else {
+		cmd.logDest.Printf("Files: %v", filelist)
+	}
+
+	if err := os.MkdirAll(cmd.OutputDir, 0755); err != nil {
+		return errors.Wrapf(err, "creating output directory: '%s'", cmd.OutputDir)
+	}
+
+	for i := 0; i < cmd.PartitionN; i++ {
+		name := filepath.Join(cmd.OutputDir, fmt.Sprintf("output_%d", i))
+		f, err := os.Create(name)
+		if err != nil {
+			return errors.Wrapf(err, "couldn't open output file %s", name)
+		}
+		cmd.outputFiles[i] = f
+	}
+	defer func() {
+		for _, f := range cmd.outputFiles {
+			_ = f.Close()
+		}
+	}()
+
+	switch cmd.Type {
+	case "ndjson":
+		if err := cmd.RunNDJSON(filelist); err != nil {
+			return errors.Wrap(err, "running ndjson")
+		}
+	case "csv":
+		if err := cmd.RunCSV(filelist); err != nil {
+			return errors.Wrap(err, "running csv")
+		}
+	default:
+		return errors.Errorf("unsupported type %s, try ndjson or csv", cmd.Type)
+	}
+
+	return nil
+}
+
+func (cmd *PreSortCommand) RunNDJSON(filelist []string) error {
+	outputEncoders := make(map[int]*json.Encoder)
+	for i, f := range cmd.outputFiles {
+		outputEncoders[i] = json.NewEncoder(f)
+		outputEncoders[i].SetIndent("", "")
+		outputEncoders[i].SetEscapeHTML(false)
+	}
+
+	for _, fname := range filelist {
+		f, err := os.Open(fname)
+		if err != nil {
+			cmd.logDest.Warnf("Could not open %s, skipping", fname)
+		}
+		dec := json.NewDecoder(f)
+		rec := map[string]interface{}{}
+		for err = dec.Decode(&rec); err == nil; err = dec.Decode(&rec) {
+			partition, err := cmd.ndjsonPartition(rec)
+			if err != nil {
+				return err
+			}
+			err = outputEncoders[partition].Encode(rec)
+			if err != nil {
+				return errors.Wrapf(err, "writing record to partition %d", partition)
+			}
+		}
+		if err != io.EOF && err != nil {
+			return errors.Wrapf(err, "error attempting to decode json from %s", fname)
+		}
+	}
+	return nil
+}
+
+func (cmd *PreSortCommand) ndjsonPartition(rec map[string]interface{}) (int, error) {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(cmd.Table))
+
+	for i, name := range cmd.PrimaryKeyFields {
+		val, ok := rec[name]
+		if !ok {
+			return 0, errors.Errorf("couldn't find primary key part '%s' in record: %+v", name, rec)
+		}
+		h.Write([]byte(toString(val)))
+		if i < len(cmd.PrimaryKeyFields)-1 {
+			h.Write([]byte{'|'})
+		}
+	}
+	return int(h.Sum64() % uint64(cmd.PartitionN)), nil
+}
+
+func (cmd *PreSortCommand) RunCSV(filelist []string) error {
+	outputWriters := make(map[int]*csv.Writer)
+	for i, f := range cmd.outputFiles {
+		outputWriters[i] = csv.NewWriter(f)
+	}
+
+	for _, fname := range filelist {
+		f, err := os.Open(fname)
+		if err != nil {
+			cmd.logDest.Warnf("Could not open %s, skipping", fname)
+		}
+		reader := csv.NewReader(f)
+		reader.ReuseRecord = true
+		rec, err := reader.Read()
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		header := make(map[string]int)
+		for i, key := range rec {
+			header[key] = i
+		}
+		for rec, err = reader.Read(); err == nil; rec, err = reader.Read() {
+			partition, err := cmd.csvPartition(header, rec)
+			if err != nil {
+				return err
+			}
+			err = outputWriters[partition].Write(rec)
+			if err != nil {
+				return errors.Wrapf(err, "writing record to partition %d", partition)
+			}
+		}
+		if err != io.EOF && err != nil {
+			return errors.Wrapf(err, "error attempting to decode json from %s", fname)
+		}
+	}
+
+	for _, w := range outputWriters {
+		w.Flush()
+	}
+	return nil
+}
+
+func (cmd *PreSortCommand) csvPartition(header map[string]int, rec []string) (int, error) {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(cmd.Table))
+
+	for i, name := range cmd.PrimaryKeyFields {
+		pos, ok := header[name]
+		if !ok {
+			return 0, errors.Errorf("couldn't find primary key part '%s' in header: %+v", name, header)
+		}
+		val := rec[pos]
+		h.Write([]byte(toString(val)))
+		if i < len(cmd.PrimaryKeyFields)-1 {
+			h.Write([]byte{'|'})
+		}
+	}
+	return int(h.Sum64() % uint64(cmd.PartitionN)), nil
+}
+
+func toString(val interface{}) string {
+	switch valt := val.(type) {
+	case string:
+		return valt
+	case float64, int, int64, uint64, float32, uint:
+		return fmt.Sprintf("%d", valt)
+	default:
+		return fmt.Sprintf("%v", valt)
+	}
+}
+
+func (cmd *PreSortCommand) partition(key string) int {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(cmd.Table))
+	_, _ = h.Write([]byte(key))
+	return int(h.Sum64() % uint64(cmd.PartitionN))
+}
+
+func partition(key string) int {
+	indexName := "karambit"
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(indexName))
+	_, _ = h.Write([]byte(key))
+	return int(h.Sum64() % uint64(256))
+}

ctl/pre_sort_test.go

@@ -0,0 +1,136 @@
+package ctl_test
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+func TestPreSort(t *testing.T) {
+	td := t.TempDir()
+	testFile(t, "sample.ndjson", td, sampleNDJSON)
+	testFile(t, "sample.csv", td, sampleCSV)
+
+	com := ctl.NewPreSortCommand(logger.StderrLogger)
+	com.Type = "ndjson"
+	com.OutputDir = "ndjson_out"
+	com.File = "sample.ndjson"
+	com.Table = "blah"
+	com.PartitionN = 5
+	com.PrimaryKeyFields = []string{"url"}
+
+	os.Chdir(td)
+
+	err := com.Run(context.Background())
+	if err != nil {
+		t.Fatalf("running ndjson: %v", err)
+	}
+
+	entries, err := os.ReadDir(filepath.Join(td, "ndjson_out"))
+	if err != nil {
+		t.Fatalf("reading dir: %v", err)
+	}
+	totalSize := 0
+	numWithData := 0
+	totalSizeRead := 0
+	if len(entries) != 5 {
+		t.Errorf("expected 5 output files, but got: %d", len(entries))
+	}
+	for _, entry := range entries {
+		stuff, err := os.ReadFile(filepath.Join(com.OutputDir, entry.Name()))
+		if err != nil {
+			t.Fatalf("reading: %v", err)
+		}
+		totalSizeRead += len(stuff)
+		t.Logf("%s\n", stuff)
+		if size(t, entry) > 0 {
+			totalSize += int(size(t, entry))
+			numWithData++
+		}
+		t.Log(entry.Name(), size(t, entry))
+	}
+	if totalSize < len(sampleNDJSON) || totalSize > len(sampleNDJSON) {
+		t.Errorf("unexpected total data size orig: %d, got: %d", len(sampleNDJSON), totalSize)
+	}
+
+	com = ctl.NewPreSortCommand(logger.StderrLogger)
+	com.Type = "csv"
+	com.OutputDir = "csv_out"
+	com.File = "sample.csv"
+	com.Table = "blah"
+	com.PartitionN = 5
+	com.PrimaryKeyFields = []string{"a", "b"}
+
+	err = com.Run(context.Background())
+	if err != nil {
+		t.Fatalf("running ndjson: %v", err)
+	}
+
+	entries, err = os.ReadDir(filepath.Join(td, com.OutputDir))
+	if err != nil {
+		t.Fatalf("reading dir: %v", err)
+	}
+	totalSize = 0
+	numWithData = 0
+	totalSizeRead = 0
+	if len(entries) != 5 {
+		t.Errorf("expected 5 output files, but got: %d", len(entries))
+	}
+	for _, entry := range entries {
+		stuff, err := os.ReadFile(filepath.Join(com.OutputDir, entry.Name()))
+		if err != nil {
+			t.Fatalf("reading: %v", err)
+		}
+		totalSizeRead += len(stuff)
+		t.Logf("%s\n", stuff)
+		if size(t, entry) > 0 {
+			totalSize += int(size(t, entry))
+			numWithData++
+		}
+		t.Log(entry.Name(), size(t, entry))
+	}
+	// -14 is removing the header
+	if totalSize < len(sampleCSV)-14 || totalSize > len(sampleCSV)-14 {
+		t.Errorf("unexpected total data size orig: %d, got: %d", len(sampleCSV), totalSize)
+	}
+
+}
+
+func size(t *testing.T, e fs.DirEntry) int64 {
+	info, err := e.Info()
+	if err != nil {
+		t.Fatalf(": %v", err)
+	}
+	return info.Size()
+}
+
+func testFile(t *testing.T, name, dir, contents string) {
+	f, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("creating temp file: %v", err)
+	}
+	_, err = f.WriteString(contents)
+	if err != nil {
+		t.Fatalf("writing temp file: %v", err)
+	}
+}
+
+var sampleNDJSON string = `{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=0","result":{"extractorData":{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=0","data":[{"group":[{"Business":[{"href":"https://www.yelp.com/biz/milk-and-wood-san-jose?osq=Desserts","text":"Milk & Wood"}]},{"Business":[{"href":"https://www.yelp.com/biz/dzuis-cakes-and-desserts-san-jose?osq=Desserts","text":"Dzui’s Cakes & Desserts"}]},{"Business":[{"href":"https://www.yelp.com/biz/recess-italian-ice-and-desserts-san-jose?osq=Desserts","text":"Recess Italian Ice and Desserts"}]},{"Business":[{"href":"https://www.yelp.com/biz/icicles-san-jose-7?osq=Desserts","text":"ICICLES"}]},{"Business":[{"href":"https://www.yelp.com/biz/sweet-rendezvous-san-jose?osq=Desserts","text":"Sweet Rendezvous"}]},{"Business":[{"href":"https://www.yelp.com/biz/passion-t-snacks-and-desserts-san-jose?osq=Desserts","text":"Passion-T Snacks and Desserts"}]},{"Business":[{"href":"https://www.yelp.com/biz/vampire-penguin-featuring-jastea-san-jose?osq=Desserts","text":"Vampire Penguin featuring Jastea"}]},{"Business":[{"href":"https://www.yelp.com/biz/sweet-gelato-tea-lounge-san-jose?osq=Desserts","text":"Sweet Gelato Tea Lounge"}]},{"Business":[{"href":"https://www.yelp.com/biz/matcha-love-san-jose-6?osq=Desserts","text":"Matcha Love"}]},{"Business":[{"href":"https://www.yelp.com/biz/anton-sv-p%C3%A2tisserie-san-jose-2?osq=Desserts","text":"Anton SV Pâtisserie"}]}]}]},"pageData":{"statusCode":200,"timestamp":1513286383006},"timestamp":1513286383006,"sequenceNumber":0}}
+{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=10","result":{"extractorData":{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=10","data":[{"group":[{"Business":[{"href":"https://www.yelp.com/biz/oooh-san-jose-4?osq=Desserts","text":"Oooh"}]},{"Business":[{"href":"https://www.yelp.com/biz/hannah-san-jose?osq=Desserts","text":"Hannah"}]},{"Business":[{"href":"https://www.yelp.com/biz/chocatoo-san-jose?osq=Desserts","text":"Chocatoo"}]},{"Business":[{"href":"https://www.yelp.com/biz/nox-cookie-bar-san-jose?osq=Desserts","text":"Nox Cookie Bar"}]},{"Business":[{"href":"https://www.yelp.com/biz/sweet-fix-creamery-san-jose?osq=Desserts","text":"Sweet Fix Creamery"}]},{"Business":[{"href":"https://www.yelp.com/biz/my-milkshake-san-jose?osq=Desserts","text":"My Milkshake"}]},{"Business":[{"href":"https://www.yelp.com/biz/matcha-love-san-jose-6?osq=Desserts","text":"Matcha Love"}]},{"Business":[{"href":"https://www.yelp.com/biz/banana-cr%C3%AApe-san-jose-2?osq=Desserts","text":"Banana Crêpe"}]},{"Business":[{"href":"https://www.yelp.com/biz/marco-polo-italian-ice-cream-san-jose-4?osq=Desserts","text":"Marco Polo Italian Ice Cream"}]},{"Business":[{"href":"https://www.yelp.com/biz/blackball-desserts-san-jose-san-jose?osq=Desserts","text":"BlackBall Desserts San Jose"}]}]}]},"pageData":{"statusCode":200,"timestamp":1513286384917},"timestamp":1513286384917,"sequenceNumber":1}}
+{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=20","result":{"extractorData":{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=20","data":[{"group":[{"Business":[{"href":"https://www.yelp.com/biz/anton-sv-p%C3%A2tisserie-san-jose-2?osq=Desserts","text":"Anton SV Pâtisserie"}]},{"Business":[{"href":"https://www.yelp.com/biz/soyful-desserts-san-jose-8?osq=Desserts","text":"Soyful Desserts"}]},{"Business":[{"href":"https://www.yelp.com/biz/cocola-bakery-san-jose?osq=Desserts","text":"Cocola Bakery"}]},{"Business":[{"href":"https://www.yelp.com/biz/charlies-cheesecake-works-san-jose?osq=Desserts","text":"Charlie’s Cheesecake Works"}]},{"Business":[{"href":"https://www.yelp.com/biz/jt-express-san-jose-2?osq=Desserts","text":"JT Express"}]},{"Business":[{"href":"https://www.yelp.com/biz/nox-cookie-bar-san-jose?osq=Desserts","text":"Nox Cookie Bar"}]},{"Business":[{"href":"https://www.yelp.com/biz/shuei-do-manju-shop-san-jose?osq=Desserts","text":"Shuei-Do Manju Shop"}]},{"Business":[{"href":"https://www.yelp.com/biz/churros-el-guero-san-jose?osq=Desserts","text":"Churros El Guero"}]},{"Business":[{"href":"https://www.yelp.com/biz/sno-crave-tea-house-san-jose-4?osq=Desserts","text":"Sno-Crave Tea House"}]},{"Business":[{"href":"https://www.yelp.com/biz/j-sweets-san-jose-3?osq=Desserts","text":"J.Sweets"}]}]}]},"pageData":{"statusCode":200,"timestamp":1513286395948},"timestamp":1513286395948,"sequenceNumber":2}}
+{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=30","result":{"extractorData":{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=30","data":[{"group":[{"Business":[{"href":"https://www.yelp.com/biz/treatbot-san-jose-2?osq=Desserts","text":"Treatbot"}]},{"Business":[{"href":"https://www.yelp.com/biz/cream-san-jose?osq=Desserts","text":"CREAM"}]},{"Business":[{"href":"https://www.yelp.com/biz/my-milkshake-san-jose?osq=Desserts","text":"My Milkshake"}]},{"Business":[{"href":"https://www.yelp.com/biz/peters-bakery-san-jose?osq=Desserts","text":"Peters’ Bakery"}]},{"Business":[{"href":"https://www.yelp.com/biz/the-charming-kitchen-san-jose-5?osq=Desserts","text":"The Charming Kitchen"}]},{"Business":[{"href":"https://www.yelp.com/biz/sweet-fix-creamery-san-jose?osq=Desserts","text":"Sweet Fix Creamery"}]},{"Business":[{"href":"https://www.yelp.com/biz/california-mochi-santa-clara-4?osq=Desserts","text":"California Mochi"}]},{"Business":[{"href":"https://www.yelp.com/biz/raw-sugar-milpitas-2?osq=Desserts","text":"Raw Sugar"}]},{"Business":[{"href":"https://www.yelp.com/biz/chola-desserts-san-jose?osq=Desserts","text":"Chola Desserts"}]},{"Business":[{"href":"https://www.yelp.com/biz/san-jose-tofu-company-san-jose?osq=Desserts","text":"San Jose Tofu Company"}]}]}]},"pageData":{"statusCode":200,"timestamp":1513286386420},"timestamp":1513286386420,"sequenceNumber":3}}
+{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=40","result":{"extractorData":{"url":"https://www.yelp.com/search?find_desc=Desserts&find_loc=San+Jose,+CA&start=40","data":[{"group":[{"Business":[{"href":"https://www.yelp.com/biz/the-sweet-corner-san-jose?osq=Desserts","text":"The Sweet Corner"}]},{"Business":[{"href":"https://www.yelp.com/biz/marco-polo-italian-ice-cream-san-jose-4?osq=Desserts","text":"Marco Polo Italian Ice Cream"}]},{"Business":[{"href":"https://www.yelp.com/biz/honeyberry-san-jose-9?osq=Desserts","text":"Honeyberry"}]},{"Business":[{"href":"https://www.yelp.com/biz/my-ch%C3%A8-san-jose?osq=Desserts","text":"My Chè"}]},{"Business":[{"href":"https://www.yelp.com/biz/creme-paris-san-jose-3?osq=Desserts","text":"Cre’Me Paris"}]},{"Business":[{"href":"https://www.yelp.com/biz/snowflake-san-jose?osq=Desserts","text":"Snowflake"}]},{"Business":[{"href":"https://www.yelp.com/biz/willow-glen-creamery-san-jose-4?osq=Desserts","text":"Willow Glen Creamery"}]},{"Business":[{"href":"https://www.yelp.com/biz/vans-bakery-san-jose?osq=Desserts","text":"Van’s Bakery"}]},{"Business":[{"href":"https://www.yelp.com/biz/happiness-cafe-san-jose?osq=Desserts","text":"Happiness Cafe"}]},{"Business":[{"href":"https://www.yelp.com/biz/la-original-paleteria-y-neveria-san-jose?osq=Desserts","text":"La Original Paleteria Y Neveria"}]}]}]},"pageData":{"statusCode":200,"timestamp":1513286387763},"timestamp":1513286387763,"sequenceNumber":4}}
+`
+
+var sampleCSV string = `a,b,c,d,e,f,g
+1,2,3,4,5,6,7
+2,3,4,5,6,7,8
+3,4,5,6,7,8,9
+4,5,6,7,8,9,0
+0,1,2,3,4,5,6
+`

ctl/util.go

@@ -8,8 +8,8 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/felixge/fgprof"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/felixge/fgprof"
 	"github.com/pkg/errors"
 )
 

dax/computer/service/computer.go

@@ -7,14 +7,14 @@ import (
 	"net/http"
 
 	featurebase "github.com/featurebasedb/featurebase/v3"
-        "github.com/featurebasedb/featurebase/v3/dax"
-        "github.com/featurebasedb/featurebase/v3/dax/computer"
-        mdsclient "github.com/featurebasedb/featurebase/v3/dax/mds/client"
-        "github.com/featurebasedb/featurebase/v3/dax/snapshotter"
-        "github.com/featurebasedb/featurebase/v3/dax/writelogger"
-        "github.com/featurebasedb/featurebase/v3/errors"
-        "github.com/featurebasedb/featurebase/v3/logger"
-        fbserver "github.com/featurebasedb/featurebase/v3/server"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	mdsclient "github.com/featurebasedb/featurebase/v3/dax/mds/client"
+	"github.com/featurebasedb/featurebase/v3/dax/snapshotter"
+	"github.com/featurebasedb/featurebase/v3/dax/writelogger"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	fbserver "github.com/featurebasedb/featurebase/v3/server"
 )
 
 // Ensure type implements interface.

dax/mds/http/handler.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/mds"
+	"github.com/gorilla/mux"
 )
 
 func Handler(mds *mds.MDS) http.Handler {

dax/queryer/http/handler.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/queryer"
+	"github.com/gorilla/mux"
 )
 
 func Handler(q *queryer.Queryer) http.Handler {

dax/queryer/orchestrator.go

@@ -11,11 +11,11 @@ import (
 
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
 )
 

dax/service_manager.go

@@ -5,9 +5,9 @@ import (
 	"net/http"
 	"sync"
 
-	"github.com/gorilla/mux"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/gorilla/mux"
 )
 
 // ServiceKey is a unique key used to identify one service managed by the

encoding/proto/proto.go

@@ -15,8 +15,6 @@ import (
 	"github.com/apache/arrow/go/v10/parquet"
 	"github.com/apache/arrow/go/v10/parquet/file"
 	"github.com/apache/arrow/go/v10/parquet/pqarrow"
-	"github.com/gogo/protobuf/proto"
-	"github.com/gomem/gomem/pkg/dataframe"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	pnet "github.com/featurebasedb/featurebase/v3/net"
@@ -24,6 +22,8 @@ import (
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/gogo/protobuf/proto"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 )
 

encoding/proto/proto_test.go

@@ -9,9 +9,9 @@ import (
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/gomem/gomem/pkg/dataframe"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/pb"
+	"github.com/gomem/gomem/pkg/dataframe"
 )
 
 func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) {

executor.go

@@ -17,8 +17,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/gomem/gomem/pkg/dataframe"
-	"github.com/lib/pq"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/pql"
@@ -28,6 +26,8 @@ import (
 	"github.com/featurebasedb/featurebase/v3/task"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/lib/pq"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"

executor_test.go

@@ -26,6 +26,7 @@ import (
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/proto"
 	"github.com/featurebasedb/featurebase/v3/server"
@@ -33,7 +34,6 @@ import (
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
 	"github.com/google/go-cmp/cmp"
-	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )

hack.go

@@ -6,9 +6,9 @@ import (
 	"math"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/featurebasedb/featurebase/v3/pb"
 	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/gogo/protobuf/proto"
 )
 
 func UnmarshalIndexOptions(name string, createdAt int64, buf []byte) (*IndexOptions, error) {

http_handler.go

@@ -28,9 +28,6 @@ import (
 	"time"
 
 	"github.com/apache/arrow/go/v10/arrow"
-	"github.com/felixge/fgprof"
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
 	"github.com/featurebasedb/featurebase/v3/authn"
 	"github.com/featurebasedb/featurebase/v3/authz"
 	fbcontext "github.com/featurebasedb/featurebase/v3/context"
@@ -43,10 +40,13 @@ import (
 	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
 	"github.com/featurebasedb/featurebase/v3/storage"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/featurebasedb/featurebase/v3/wireprotocol"
+	"github.com/felixge/fgprof"
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/featurebasedb/featurebase/v3/wireprotocol"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/prom2json"
 	uuid "github.com/satori/go.uuid"

idk/bankgen/put.go

@@ -10,11 +10,11 @@ import (
 	"strings"
 	"sync/atomic"
 
-	liavro "github.com/linkedin/goavro/v2"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/idk/common"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	liavro "github.com/linkedin/goavro/v2"
 	"github.com/pkg/errors"
 
 	confluent "github.com/confluentinc/confluent-kafka-go/kafka"

idk/cmd/bankgen/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/bankgen"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/bankgen/main_test.go

@@ -4,9 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/bankgen"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/bankgen"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/datagen/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/datagen"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 	"gopkg.in/DataDog/dd-trace-go.v1/profiler"
 )
 

idk/cmd/datagen/main_test.go

@@ -5,9 +5,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/datagen"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/datagen"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/ingester/main.go

@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"runtime/debug"
 
-	"github.com/gorilla/mux"
-	"github.com/jaffee/commandeer/pflag"
 	pilosaclient "github.com/featurebasedb/featurebase/v3/client"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/idk/api"
+	"github.com/gorilla/mux"
+	"github.com/jaffee/commandeer/pflag"
 	"github.com/pkg/errors"
 )
 

idk/cmd/kafkagen/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafkagen"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/kafkagen/main_test.go

@@ -4,9 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/kafkagen"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/kafkagen"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/kafkaput/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/kafkaput/main_test.go

@@ -4,9 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/molecula-consumer-csv/main.go

@@ -5,9 +5,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/csv"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-csv/main_test.go

@@ -4,9 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/csv"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/csv"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/molecula-consumer-kafka-delete/main.go

@@ -3,8 +3,8 @@ package main
 import (
 	"log"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-kafka-delete/main_test.go

@@ -5,9 +5,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/molecula-consumer-kafka-sasl/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka_sasl"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-kafka-static/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-kafka/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-kafka/main_test.go

@@ -5,10 +5,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/jaffee/commandeer"
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka"
+	"github.com/jaffee/commandeer"
+	"github.com/jaffee/commandeer/pflag"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/cmd/molecula-consumer-kinesis/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/kinesis"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func logError(m *kinesis.Main, err error) {

idk/cmd/molecula-consumer-sql/main.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"os"
 
-	"github.com/jaffee/commandeer/pflag"
 	"github.com/featurebasedb/featurebase/v3/idk/sql"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 )
 
 func main() {

idk/cmd/molecula-consumer-sql/main_test.go

@@ -4,9 +4,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/featurebasedb/featurebase/v3/idk/sql"
 	"github.com/jaffee/commandeer"
 	"github.com/jaffee/commandeer/pflag"
-	"github.com/featurebasedb/featurebase/v3/idk/sql"
 	pflag13 "github.com/spf13/pflag"
 )
 

idk/datagen/cmd.go

@@ -11,11 +11,11 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/glycerine/vprint"
 	pilosaclient "github.com/featurebasedb/featurebase/v3/client"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/glycerine/vprint"
 	"github.com/pkg/errors"
 
 	"github.com/featurebasedb/featurebase/v3/idk/common"

idk/ingest.go

@@ -23,7 +23,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/felixge/fgprof"
 	pilosacore "github.com/featurebasedb/featurebase/v3"
 	pilosagrpc "github.com/featurebasedb/featurebase/v3/api/client"
 	pilosabatch "github.com/featurebasedb/featurebase/v3/batch"
@@ -35,6 +34,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	proto "github.com/featurebasedb/featurebase/v3/proto"
+	"github.com/felixge/fgprof"
 	"github.com/pkg/errors"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"

idk/ingest_test.go

@@ -14,17 +14,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/featurebasedb/featurebase/v3/authn"
-	pilosaclient "github.com/featurebasedb/featurebase/v3/client"
-	"github.com/featurebasedb/featurebase/v3/idk/idktest"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/golang-jwt/jwt"
 	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/authn"
 	batch "github.com/featurebasedb/featurebase/v3/batch"
+	pilosaclient "github.com/featurebasedb/featurebase/v3/client"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	mdsclient "github.com/featurebasedb/featurebase/v3/dax/mds/client"
+	"github.com/featurebasedb/featurebase/v3/idk/idktest"
 	"github.com/featurebasedb/featurebase/v3/idk/mds"
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/golang-jwt/jwt"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )

idk/kafka/cmd_delete_test.go

@@ -13,10 +13,10 @@ import (
 	"time"
 
 	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
-	"github.com/jaffee/commandeer/pflag"
 	pilosaclient "github.com/featurebasedb/featurebase/v3/client"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/jaffee/commandeer/pflag"
 	"github.com/pkg/errors"
 )
 

idk/kafka/putsource.go

@@ -15,13 +15,13 @@ import (
 	"time"
 
 	confluent "github.com/confluentinc/confluent-kafka-go/kafka"
-	"github.com/go-avro/avro"
-	liavro "github.com/linkedin/goavro/v2"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/idk/common"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka/csrc"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/go-avro/avro"
+	liavro "github.com/linkedin/goavro/v2"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )

idk/kafka/putsource_test.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/go-avro/avro"
 	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/go-avro/avro"
 )
 
 func TestIdkSchemaToAvroRecordSchema(t *testing.T) {

internal/clustertests/cluster_test.go

@@ -14,12 +14,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang-jwt/jwt"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/golang-jwt/jwt"
 	"github.com/pkg/errors"
 )
 

internal_client.go

@@ -20,13 +20,13 @@ import (
 
 	fbcontext "github.com/featurebasedb/featurebase/v3/context"
 
-	"github.com/hashicorp/go-retryablehttp"
 	"github.com/featurebasedb/featurebase/v3/authn"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	pnet "github.com/featurebasedb/featurebase/v3/net"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )

iterator.go

@@ -4,7 +4,6 @@ package pilosa
 
 import (
 	"fmt"
-
 )
 
 // iterator is an interface for looping over row/column pairs.

prometheus/prometheus_test.go

@@ -5,7 +5,6 @@ package prometheus_test
 import (
 	"testing"
 
-
 	"github.com/featurebasedb/featurebase/v3/test"
 	"github.com/prometheus/client_golang/prometheus"
 	io_prometheus_client "github.com/prometheus/client_model/go"

proto/vdsm/vdsm.pb.go

@@ -6,8 +6,8 @@ package vdsm
 import (
 	context "context"
 	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
 	proto1 "github.com/featurebasedb/featurebase/v3/proto"
+	proto "github.com/golang/protobuf/proto"
 	grpc "google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"

roaring/roaring.go

@@ -4517,7 +4517,7 @@ func intersectionCountArrayArray(a, b *Container) (n int32) {
 	na, nb := len(ca), len(cb)
 	if na > nb {
 		ca, cb = cb, ca
-		na, nb = nb, na // nolint: staticcheck, ineffassign
+		na, nb = nb, na //nolint: staticcheck, ineffassign
 	}
 	j := 0
 	for _, va := range ca {
@@ -4619,7 +4619,7 @@ func intersectionCallbackArrayArray(a, b *Container, fn func(uint16)) {
 	na, nb := len(ca), len(cb)
 	if na > nb {
 		ca, cb = cb, ca
-		na, nb = nb, na // nolint: staticcheck, ineffassign
+		na, nb = nb, na //nolint: staticcheck, ineffassign
 	}
 	if (na << 2) < nb {
 		for _, va := range ca {

server.go

@@ -19,17 +19,17 @@ import (
 	uuid "github.com/satori/go.uuid"
 
 	daxstorage "github.com/featurebasedb/featurebase/v3/dax/storage"
-        "github.com/featurebasedb/featurebase/v3/disco"
-        "github.com/featurebasedb/featurebase/v3/logger"
-        pnet "github.com/featurebasedb/featurebase/v3/net"
-        rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg"
-        "github.com/featurebasedb/featurebase/v3/roaring"
-        "github.com/featurebasedb/featurebase/v3/sql3"
-        "github.com/featurebasedb/featurebase/v3/sql3/parser"
-        planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types"
-        "github.com/featurebasedb/featurebase/v3/storage"
-        "github.com/pkg/errors"
-        "golang.org/x/sync/errgroup"
+	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	pnet "github.com/featurebasedb/featurebase/v3/net"
+	rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg"
+	"github.com/featurebasedb/featurebase/v3/roaring"
+	"github.com/featurebasedb/featurebase/v3/sql3"
+	"github.com/featurebasedb/featurebase/v3/sql3/parser"
+	planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+	"github.com/featurebasedb/featurebase/v3/storage"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 
 	_ "github.com/lib/pq"
 )

server/server.go

@@ -27,27 +27,27 @@ import (
 	"time"
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
-        "github.com/featurebasedb/featurebase/v3/authn"
-        "github.com/featurebasedb/featurebase/v3/authz"
-        "github.com/featurebasedb/featurebase/v3/dax"
-        "github.com/featurebasedb/featurebase/v3/dax/computer"
-        "github.com/featurebasedb/featurebase/v3/dax/storage"
-        "github.com/featurebasedb/featurebase/v3/disco"
-        "github.com/featurebasedb/featurebase/v3/encoding/proto"
-        petcd "github.com/featurebasedb/featurebase/v3/etcd"
-        "github.com/featurebasedb/featurebase/v3/gcnotify"
-        "github.com/featurebasedb/featurebase/v3/gopsutil"
-        "github.com/featurebasedb/featurebase/v3/logger"
-        pnet "github.com/featurebasedb/featurebase/v3/net"
-        "github.com/featurebasedb/featurebase/v3/sql3"
-        "github.com/featurebasedb/featurebase/v3/sql3/planner"
-        "github.com/featurebasedb/featurebase/v3/statik"
-        "github.com/featurebasedb/featurebase/v3/systemlayer"
-        "github.com/featurebasedb/featurebase/v3/syswrap"
-        "github.com/featurebasedb/featurebase/v3/testhook"
-        "github.com/pelletier/go-toml"
-        "github.com/pkg/errors"
-        "golang.org/x/sync/errgroup"
+	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/authz"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
+	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/featurebasedb/featurebase/v3/encoding/proto"
+	petcd "github.com/featurebasedb/featurebase/v3/etcd"
+	"github.com/featurebasedb/featurebase/v3/gcnotify"
+	"github.com/featurebasedb/featurebase/v3/gopsutil"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	pnet "github.com/featurebasedb/featurebase/v3/net"
+	"github.com/featurebasedb/featurebase/v3/sql3"
+	"github.com/featurebasedb/featurebase/v3/sql3/planner"
+	"github.com/featurebasedb/featurebase/v3/statik"
+	"github.com/featurebasedb/featurebase/v3/systemlayer"
+	"github.com/featurebasedb/featurebase/v3/syswrap"
+	"github.com/featurebasedb/featurebase/v3/testhook"
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 type loggerLogger interface {

sql3/planner/compilebulkinsert.go

@@ -15,9 +15,6 @@ import (
 	"github.com/featurebasedb/featurebase/v3/sql3/parser"
 	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
 	goerrors "github.com/pkg/errors"
-
-
-
 )
 
 // compileBulkInsertStatement compiles a BULK INSERT statement into a

sql3/planner/opdistinct.go

@@ -8,8 +8,8 @@ import (
 	"fmt"
 
 	"github.com/featurebasedb/featurebase/v3/bufferpool"
-        "github.com/featurebasedb/featurebase/v3/extendiblehash"
-        "github.com/featurebasedb/featurebase/v3/sql3"
+	"github.com/featurebasedb/featurebase/v3/extendiblehash"
+	"github.com/featurebasedb/featurebase/v3/sql3"
 	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
 )
 

sql3/sql_complex_test.go

@@ -11,14 +11,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	sql_test "github.com/featurebasedb/featurebase/v3/sql3/test"
 	"github.com/featurebasedb/featurebase/v3/test"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
-	)
+)
 
 func TestPlanner_Misc(t *testing.T) {
 	d, err := pql.ParseDecimal("12.345678")

wireprotocol/wireprimitives_test.go

@@ -5,11 +5,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/sql3/parser"
 	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
 	"github.com/featurebasedb/featurebase/v3/wireprotocol"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestWireProtocol_Schema(t *testing.T) {