mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 02:44:59 +00:00
eb6c6e3105
* Add kafka support to CLI (fbsql) This commit adds the ability to provide a `--kafka-config` command line argument referncing a toml file to configure kafka. * Move "Molecula Consumer" message to the logger; hide it in basic mode * Fold decimal(scale) into kafka.source-type * Build fbsql with cgo in docker for CI * Re-organize the fbsql kafka config and setup. Allow field config to use the table schema if no fields provided. * Display timestamp fields with format RFC3339Nano * remove kafkaRunner (no longer used) * Fix cli/batch test (and make sure it's not excluded from CI) The logic in our Makefile was exluding from tests any package with `/batch` in the package name. This excluded `/cli/batch`, which is not good. This commit changes the exclusion logic to include the `/v3` portion of the package name, so `/v3/batch`. * Rename Basic() to SetBasic()
32 lines
1.8 KiB
Go
32 lines
1.8 KiB
Go
// Copyright 2021 Molecula Corp. All rights reserved.
|
|
package cmd
|
|
|
|
import (
|
|
"github.com/featurebasedb/featurebase/v3/ctl"
|
|
"github.com/featurebasedb/featurebase/v3/logger"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func newPreSortCommand(logdest logger.Logger) *cobra.Command {
|
|
cmd := ctl.NewPreSortCommand(logdest)
|
|
ccmd := &cobra.Command{
|
|
Use: "presort",
|
|
Short: "Sort records within files into files by FB partition for more efficient ingest",
|
|
Long: `
|
|
Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
|
|
`,
|
|
RunE: UsageErrorWrapper(cmd),
|
|
}
|
|
|
|
flags := ccmd.Flags()
|
|
flags.StringVarP(&cmd.File, "file", "", "", "Input file or directory.")
|
|
flags.StringVarP(&cmd.Table, "table", "", "", "Name of table (used to hash keys to determine partition).")
|
|
flags.StringVarP(&cmd.Type, "type", "", cmd.Type, "Input file type (csv or ndjson).")
|
|
flags.StringSliceVar(&cmd.PrimaryKeyFields, "primary-key-fields", []string{}, "Names of primary key fields. For CSV there must be a header row and these are pulled from there.")
|
|
flags.IntVar(&cmd.PartitionN, "partition-n", cmd.PartitionN, "Number of partitions.")
|
|
flags.StringVarP(&cmd.OutputDir, "output-dir", "", cmd.OutputDir, "Directory name to write output to.")
|
|
flags.StringVarP(&cmd.PrimaryKeySeparator, "primary-key-separator", "", cmd.PrimaryKeySeparator, "Separator to write in between primary key fields, can be empty.")
|
|
flags.IntVar(&cmd.JobSize, "job-size", cmd.JobSize, "Number of lines to put into each job (purely a performance tuning parameter, only supported by ndjson mode).")
|
|
flags.IntVar(&cmd.NumWorkers, "num-workers", cmd.NumWorkers, "Number of parallel worker routines doing decode->hash->encode. Only supported by ndjson mode.")
|
|
return ccmd
|
|
}
|