mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
123 lines
3.8 KiB
Go
123 lines
3.8 KiB
Go
package kafka
|
|
|
|
import (
|
|
"io"
|
|
"time"
|
|
|
|
fbbatch "github.com/featurebasedb/featurebase/v3/batch"
|
|
"github.com/featurebasedb/featurebase/v3/errors"
|
|
"github.com/featurebasedb/featurebase/v3/idk"
|
|
"github.com/featurebasedb/featurebase/v3/idk/common"
|
|
"github.com/featurebasedb/featurebase/v3/idk/kafka"
|
|
"github.com/featurebasedb/featurebase/v3/idk/kafka_sasl"
|
|
"github.com/featurebasedb/featurebase/v3/logger"
|
|
)
|
|
|
|
// Runner is a CLI-specific kafka consumer. It's similar to
|
|
// idk.kafka_static.Main in that it embeds idk.Main and contains additional
|
|
// functionality specific to its use case.
|
|
type Runner struct {
|
|
idk.Main `flag:"!embed"`
|
|
idk.ConfluentCommand `flag:"!embed"`
|
|
Hosts []string `help:"Comma separated list of host:port pairs for Kafka."`
|
|
Group string `help:"Kafka group."`
|
|
Topics []string `help:"Kafka topics to read from."`
|
|
Timeout time.Duration `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
|
|
Header []idk.RawField `help:"Header configuration."`
|
|
}
|
|
|
|
// Configure and return *runner with common elements of all kafka runners
|
|
func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
|
|
idkMain := idk.NewMain()
|
|
idkMain.IDField = cfg.IDField
|
|
idkMain.PrimaryKeyFields = cfg.PrimaryKeys
|
|
idkMain.Index = cfg.Table
|
|
idkMain.Batcher = batcher
|
|
idkMain.BatchSize = cfg.BatchSize
|
|
idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
|
|
idkMain.MaxMsgs = uint64(cfg.MaxMessages)
|
|
idkMain.SetBasic()
|
|
idkMain.SetLog(logger.NewStandardLogger(logWriter))
|
|
idkMain.OffsetMode = true
|
|
idkMain.Namespace = "sql_kafka_runner"
|
|
idkMain.Pprof = "" // don't initialize pprof until we actually use it in tests
|
|
|
|
kr := &Runner{
|
|
Main: *idkMain,
|
|
Hosts: cfg.Hosts,
|
|
Group: cfg.Group,
|
|
Topics: cfg.Topics,
|
|
Header: cfg.Fields,
|
|
Timeout: cfg.Timeout,
|
|
}
|
|
|
|
kr.KafkaBootstrapServers = cfg.Hosts
|
|
kr.SchemaRegistryURL = cfg.SchemaRegistryURL
|
|
kr.SchemaRegistryUsername = cfg.SchemaRegistryUsername
|
|
kr.SchemaRegistryPassword = cfg.SchemaRegistryPassword
|
|
|
|
// NewSource should be set based on the encoding of the source (e.g. JSON, Avro)
|
|
switch cfg.Encode {
|
|
case encodingTypeAvro:
|
|
kr.GetAvroNewSource(cfg)
|
|
case encodingTypeAvroDelete:
|
|
kr.GetAvroNewSource(cfg)
|
|
kr.Delete = true
|
|
default:
|
|
kr.GetJSONNewSource(cfg)
|
|
}
|
|
return kr
|
|
}
|
|
|
|
func (r *Runner) GetJSONNewSource(cfg ConfigForIDK) {
|
|
|
|
r.NewSource = func() (idk.Source, error) {
|
|
source := kafka_sasl.NewSource()
|
|
//source.KafkaBootstrapServers = r.Hosts
|
|
source.Group = r.Group
|
|
source.Topics = r.Topics
|
|
source.Log = r.Main.Log()
|
|
source.Timeout = r.Timeout
|
|
source.HeaderFields = r.Header
|
|
source.AllowMissingFields = cfg.AllowMissingFields
|
|
source.KafkaConfiguration = cfg.ConfluentConfig
|
|
confluentCfg, err := common.SetupConfluent(&r.ConfluentCommand)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
source.ConfigMap = confluentCfg
|
|
|
|
err = source.Open()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "opening source")
|
|
}
|
|
return source, nil
|
|
}
|
|
}
|
|
|
|
func (r *Runner) GetAvroNewSource(cfg ConfigForIDK) {
|
|
|
|
r.NewSource = func() (idk.Source, error) {
|
|
source := kafka.NewSource()
|
|
source.KafkaBootstrapServers = r.Hosts
|
|
source.SchemaRegistryURL = cfg.SchemaRegistryURL
|
|
source.SchemaRegistryUsername = cfg.SchemaRegistryUsername
|
|
source.SchemaRegistryPassword = cfg.SchemaRegistryPassword
|
|
source.Group = r.Group
|
|
source.Topics = r.Topics
|
|
source.Log = r.Main.Log()
|
|
source.Timeout = r.Timeout
|
|
source.KafkaConfiguration = cfg.ConfluentConfig
|
|
confluentCfg, err := common.SetupConfluent(&r.ConfluentCommand)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
source.ConfigMap = confluentCfg
|
|
|
|
err = source.Open()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "opening source")
|
|
}
|
|
return source, nil
|
|
}
|
|
}
|